Files
it_cost_dashboard/.github/workflows/cicd.yaml
T
2026-09-21 18:42:32 +08:00

325 lines
14 KiB
YAML

name: CI/CD (dev on branch, prod on tag)
on:
push:
branches:
- dev # build & deploy for dev only
tags:
- "*" # build & deploy for prod only when a tag is pushed
jobs:
setup:
runs-on:
group: trend-emu-intranet
labels:
- "ubuntu-latest-intranet"
outputs:
matrix: ${{ steps.pick.outputs.matrix }}
steps:
- name: Pick matrix config from ref type
id: pick
shell: bash
run: |
if [[ "${{ github.ref_type }}" == "tag" ]]; then
echo 'matrix={"include":[{"env":"prod","image_name_server":"it-cost-dashboard-server-prod","image_name_db":"it-cost-dashboard-db-prod","latest_tag":"latest-prod","release":"it-cost-dashboard-prod","namespace":"it-cost-dashboard-prod","chart":"./helm","values_file":"./helm/values-prod.yaml"}]}' >> "$GITHUB_OUTPUT"
else
echo 'matrix={"include":[{"env":"dev","image_name_server":"it-cost-dashboard-server-dev","image_name_db":"it-cost-dashboard-db-dev","latest_tag":"latest-dev","release":"it-cost-dashboard-dev","namespace":"it-cost-dashboard-dev","chart":"./helm","values_file":"./helm/values-dev.yaml"}]}' >> "$GITHUB_OUTPUT"
fi
build-and-deploy:
needs: setup
runs-on:
group: trend-emu-intranet
labels:
- "ubuntu-latest-intranet"
strategy:
matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
# Per-environment group: a dev push must never cancel an in-flight prod
# deploy (the original used a hardcoded "prod" group for both refs).
concurrency:
group: deploy-${{ matrix.env }}
cancel-in-progress: true
env:
# host/repo, e.g. jfrog.trendmicro.com/ameait-it-cost-dashboard
JFROG_REGISTRY: ${{ secrets.JFROG_REGISTRY }}
DOCKER_BUILDKIT: "1"
COMPOSE_DOCKER_CLI_BUILD: "1"
steps:
- name: Checkout (with tags)
uses: actions/checkout@v4
with:
fetch-depth: 0 # IMPORTANT to access tags
- name: Verify tag is on prod branch (main)
id: guard
if: github.ref_type == 'tag'
shell: bash
run: |
git fetch origin main --quiet
if git merge-base --is-ancestor "${{ github.sha }}" origin/main; then
echo "Tag commit is on main. Proceeding with prod deploy."
else
echo "::error::Tag ${GITHUB_REF#refs/tags/} points at a commit not on main. Refusing prod deploy."
exit 1
fi
- name: Set up kubeconfig
run: |
echo "KUBECONFIG=${RUNNER_TEMP}/kubeconfig" >> "$GITHUB_ENV"
echo "${{ secrets.KUBE_CONFIG_DATA }}" | base64 -d > "${RUNNER_TEMP}/kubeconfig"
chmod 600 "${RUNNER_TEMP}/kubeconfig"
# Temporary debug step: we cannot shell into this self-hosted runner
# directly, so this proves (or disproves) network reachability to the
# k8s API from inside the runner itself, ahead of the image build/push
# so we fail fast instead of waiting ~10 min for the helm timeout.
# Remove once the ACL/connectivity issue is confirmed fixed.
- name: "[DEBUG] Network path to k8s API (6443)"
run: |
set +e
echo "===== Runner identity / network info ====="
hostname -f 2>/dev/null || hostname
echo "--- Interfaces ---"
ip -brief addr show 2>/dev/null || ifconfig
echo "--- Default route ---"
ip route show 2>/dev/null || route -n
echo
echo "===== K8s API endpoint (parsed from kubeconfig) ====="
KUBE_HOST=$(grep -m1 'server:' "${KUBECONFIG}" | sed -E 's#.*https?://([^:/]+).*#\1#')
KUBE_PORT=$(grep -m1 'server:' "${KUBECONFIG}" | sed -E 's#.*:([0-9]+)/?.*#\1#')
KUBE_PORT="${KUBE_PORT:-6443}"
echo "Host: ${KUBE_HOST}"
echo "Port: ${KUBE_PORT}"
echo
echo "===== DNS resolution ====="
getent hosts "${KUBE_HOST}" || nslookup "${KUBE_HOST}" || dig +short "${KUBE_HOST}"
RESOLVED_IPS=$(getent hosts "${KUBE_HOST}" | awk '{print $1}')
echo
echo "===== Route to each resolved IP ====="
for ip in ${RESOLVED_IPS}; do
echo "-- ip route get ${ip} --"
ip route get "${ip}" 2>/dev/null || echo "(ip route not available)"
done
echo
echo "===== TCP connect test per resolved IP (port ${KUBE_PORT}) ====="
for ip in ${RESOLVED_IPS}; do
if timeout 5 bash -c "cat < /dev/null > /dev/tcp/${ip}/${KUBE_PORT}" 2>/dev/null; then
echo "OK ${ip}:${KUBE_PORT} - TCP connect succeeded"
else
echo "FAIL ${ip}:${KUBE_PORT} - TCP connect timed out or refused"
fi
done
echo
echo "===== TCP connect test via hostname (through NLB, port ${KUBE_PORT}) ====="
if timeout 5 bash -c "cat < /dev/null > /dev/tcp/${KUBE_HOST}/${KUBE_PORT}" 2>/dev/null; then
echo "OK ${KUBE_HOST}:${KUBE_PORT} - TCP connect succeeded"
else
echo "FAIL ${KUBE_HOST}:${KUBE_PORT} - TCP connect timed out or refused"
fi
echo
echo "===== curl -v (expect HTTP 401/403 = reachable; a hang/timeout = blocked) ====="
curl -v -k --max-time 8 "https://${KUBE_HOST}:${KUBE_PORT}/version"
echo
echo "===== traceroute (best effort, may be blocked itself) ====="
if command -v traceroute >/dev/null 2>&1; then
traceroute -T -p "${KUBE_PORT}" -w 2 -m 15 "${KUBE_HOST}" || true
else
echo "traceroute not installed on runner, skipping"
fi
set -e
- name: Login to JFrog
run: |
# JFROG_REGISTRY is host/repo but `docker login` takes only the host.
JFROG_HOST=$(echo "${JFROG_REGISTRY}" | cut -d'/' -f1)
echo "Logging in to ${JFROG_HOST}"
echo "${{ secrets.JFROG_IDENTITY_TOKEN }}" | docker login "${JFROG_HOST}" \
-u "${{ secrets.JFROG_USERNAME }}" \
--password-stdin
- name: Determine ref type and release tag
id: refinfo
shell: bash
run: |
echo "GITHUB_REF=${GITHUB_REF}"
echo "GITHUB_REF_TYPE=${GITHUB_REF_TYPE:-${{ github.ref_type }}}"
if [[ "${{ github.ref_type }}" == "tag" ]]; then
echo "ref_type=tag" >> $GITHUB_OUTPUT
echo "release_tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
elif [[ "${{ github.ref_type }}" == "branch" ]]; then
echo "ref_type=branch" >> $GITHUB_OUTPUT
echo "release_tag=" >> $GITHUB_OUTPUT
else
echo "ref_type=unknown" >> $GITHUB_OUTPUT
echo "release_tag=" >> $GITHUB_OUTPUT
fi
- name: Pre-build diagnostics
run: |
echo "Ref: ${GITHUB_REF}"
echo "Ref type: ${{ steps.refinfo.outputs.ref_type }}"
echo "Release tag: ${{ steps.refinfo.outputs.release_tag }}"
echo "Environment: ${{ matrix.env }}"
echo "Server image: ${JFROG_REGISTRY}/${{ matrix.image_name_server }}"
echo "DB image: ${JFROG_REGISTRY}/${{ matrix.image_name_db }}"
docker version
helm version
kubectl version --client
- name: Build & Push server image
run: |
REF_TYPE="${{ steps.refinfo.outputs.ref_type }}"
RELEASE_TAG="${{ steps.refinfo.outputs.release_tag }}"
IMAGE="${JFROG_REGISTRY}/${{ matrix.image_name_server }}"
BUILD_TAG_ARGS=(
-t "${IMAGE}:${{ matrix.latest_tag }}"
-t "${IMAGE}:${{ github.sha }}"
)
if [[ "${REF_TYPE}" == "tag" && -n "${RELEASE_TAG}" ]]; then
BUILD_TAG_ARGS+=( -t "${IMAGE}:${RELEASE_TAG}" )
fi
# AUTH_SECRET is build-time only (Auth.js v5 initializes NextAuth() at
# module load during `next build`). The real secret comes from k8s.
docker build \
-f Dockerfile \
--build-arg AUTH_SECRET=build-placeholder-not-used-at-runtime \
"${BUILD_TAG_ARGS[@]}" \
.
docker push "${IMAGE}:${{ matrix.latest_tag }}"
docker push "${IMAGE}:${{ github.sha }}"
if [[ "${REF_TYPE}" == "tag" && -n "${RELEASE_TAG}" ]]; then
docker push "${IMAGE}:${RELEASE_TAG}"
fi
- name: Build & Push db image
run: |
REF_TYPE="${{ steps.refinfo.outputs.ref_type }}"
RELEASE_TAG="${{ steps.refinfo.outputs.release_tag }}"
IMAGE="${JFROG_REGISTRY}/${{ matrix.image_name_db }}"
BUILD_TAG_ARGS=(
-t "${IMAGE}:${{ matrix.latest_tag }}"
-t "${IMAGE}:${{ github.sha }}"
)
if [[ "${REF_TYPE}" == "tag" && -n "${RELEASE_TAG}" ]]; then
BUILD_TAG_ARGS+=( -t "${IMAGE}:${RELEASE_TAG}" )
fi
docker build \
-f Dockerfile.db \
"${BUILD_TAG_ARGS[@]}" \
.
docker push "${IMAGE}:${{ matrix.latest_tag }}"
docker push "${IMAGE}:${{ github.sha }}"
if [[ "${REF_TYPE}" == "tag" && -n "${RELEASE_TAG}" ]]; then
docker push "${IMAGE}:${RELEASE_TAG}"
fi
- name: Ensure namespace exists
run: |
# `helm --atomic` against a missing namespace fails outright.
kubectl create namespace "${{ matrix.namespace }}" \
--dry-run=client -o yaml | kubectl apply -f -
- name: Ensure JFrog pull secret exists
run: |
kubectl create secret docker-registry jfrog-registry-creds \
--docker-server="$(echo "${JFROG_REGISTRY}" | cut -d'/' -f1)" \
--docker-username="${{ secrets.JFROG_USERNAME }}" \
--docker-password="${{ secrets.JFROG_IDENTITY_TOKEN }}" \
--namespace "${{ matrix.namespace }}" \
--dry-run=client -o yaml | kubectl apply -f -
# ---- DEV DEPLOY (branch dev only) ----
# Deployed by commit SHA, not latest-dev: an immutable tag sidesteps the
# moving-tag + IfNotPresent stale-cache trap and makes `kubectl describe
# pod` show exactly which commit is live.
- name: Deploy DEV with Helm
if: github.ref_type == 'branch' && github.ref == 'refs/heads/dev'
run: |
kubectl get nodes
kubectl get ns
helm upgrade --install "${{ matrix.release }}" "${{ matrix.chart }}" \
-f "${{ matrix.values_file }}" \
--namespace "${{ matrix.namespace }}" \
--set server.image.repository="${JFROG_REGISTRY}/${{ matrix.image_name_server }}" \
--set server.image.tag="${{ github.sha }}" \
--set db.image.repository="${JFROG_REGISTRY}/${{ matrix.image_name_db }}" \
--set db.image.tag="${{ github.sha }}" \
--set-string secrets.authSecret="${{ secrets.AUTH_SECRET_DEV }}" \
--set-string secrets.dbRootPassword="${{ secrets.DB_PASSWORD_DEV }}" \
--set-string secrets.localAdminPassword="${{ secrets.LOCAL_ADMIN_PASSWORD_DEV }}" \
--set-string secrets.entraClientId="${{ secrets.ENTRA_CLIENT_ID_DEV }}" \
--set-string secrets.entraClientSecret="${{ secrets.ENTRA_CLIENT_SECRET_DEV }}" \
--set-string secrets.entraIssuer="${{ secrets.ENTRA_ISSUER_DEV }}" \
--wait --timeout 10m --atomic --debug
# ---- PROD DEPLOY (tag push only) ----
- name: Deploy PROD with Helm
if: github.ref_type == 'tag'
run: |
kubectl get nodes
kubectl get ns
TAG="${GITHUB_REF#refs/tags/}"
if [[ -z "${TAG}" ]]; then
echo "No tag detected; aborting prod deploy."
exit 1
fi
helm upgrade --install "${{ matrix.release }}" "${{ matrix.chart }}" \
-f "${{ matrix.values_file }}" \
--namespace "${{ matrix.namespace }}" \
--set server.image.repository="${JFROG_REGISTRY}/${{ matrix.image_name_server }}" \
--set server.image.tag="${TAG}" \
--set db.image.repository="${JFROG_REGISTRY}/${{ matrix.image_name_db }}" \
--set db.image.tag="${TAG}" \
--set-string secrets.authSecret="${{ secrets.AUTH_SECRET_PROD }}" \
--set-string secrets.dbRootPassword="${{ secrets.DB_PASSWORD_PROD }}" \
--set-string secrets.localAdminPassword="${{ secrets.LOCAL_ADMIN_PASSWORD_PROD }}" \
--set-string secrets.entraClientId="${{ secrets.ENTRA_CLIENT_ID_PROD }}" \
--set-string secrets.entraClientSecret="${{ secrets.ENTRA_CLIENT_SECRET_PROD }}" \
--set-string secrets.entraIssuer="${{ secrets.ENTRA_ISSUER_PROD }}" \
--wait --timeout 10m --atomic --debug
# Strictly redundant with immutable image tags: Helm already rolled the
# pods and --atomic already verified health. Retained as belt-and-braces.
# Scoped to component=server so it does not bounce MySQL every push.
- name: Rollout restart (server only)
if: success() && (github.ref_type == 'branch' || github.ref_type == 'tag')
run: |
kubectl rollout restart deployment \
-l "app.kubernetes.io/instance=${{ matrix.release }},app.kubernetes.io/component=server" \
-n "${{ matrix.namespace }}"
- name: Post-deploy diagnostics (if failure)
if: failure()
run: |
echo "----- Diagnostics -----"
kubectl get deploy,sts,po,svc,ing -n "${{ matrix.namespace }}" -o wide || true
echo "----- PVCs (a Pending PVC = missing gp3 StorageClass; invisible in pod logs) -----"
kubectl get pvc -n "${{ matrix.namespace }}" -o wide || true
echo "----- Events -----"
kubectl get events -n "${{ matrix.namespace }}" --sort-by=.lastTimestamp || true
for p in $(kubectl get po -n "${{ matrix.namespace }}" -o name); do
echo "### $p"
kubectl describe "$p" -n "${{ matrix.namespace }}" || true
kubectl logs "$p" -n "${{ matrix.namespace }}" --all-containers --tail=200 || true
done