name: CI/CD (dev on branch, prod on tag) on: push: branches: - dev # build & deploy for dev only tags: - "*" # build & deploy for prod only when a tag is pushed jobs: setup: runs-on: group: trend-emu-intranet labels: - "ubuntu-latest-intranet" outputs: matrix: ${{ steps.pick.outputs.matrix }} steps: - name: Pick matrix config from ref type id: pick shell: bash run: | if [[ "${{ github.ref_type }}" == "tag" ]]; then echo 'matrix={"include":[{"env":"prod","image_name_server":"it-cost-dashboard-server-prod","image_name_db":"it-cost-dashboard-db-prod","latest_tag":"latest-prod","release":"it-cost-dashboard-prod","namespace":"it-cost-dashboard-prod","chart":"./helm","values_file":"./helm/values-prod.yaml"}]}' >> "$GITHUB_OUTPUT" else echo 'matrix={"include":[{"env":"dev","image_name_server":"it-cost-dashboard-server-dev","image_name_db":"it-cost-dashboard-db-dev","latest_tag":"latest-dev","release":"it-cost-dashboard-dev","namespace":"it-cost-dashboard-dev","chart":"./helm","values_file":"./helm/values-dev.yaml"}]}' >> "$GITHUB_OUTPUT" fi build-and-deploy: needs: setup runs-on: group: trend-emu-intranet labels: - "ubuntu-latest-intranet" strategy: matrix: ${{ fromJSON(needs.setup.outputs.matrix) }} # Per-environment group: a dev push must never cancel an in-flight prod # deploy (the original used a hardcoded "prod" group for both refs). concurrency: group: deploy-${{ matrix.env }} cancel-in-progress: true env: # host/repo, e.g. jfrog.trendmicro.com/ameait-it-cost-dashboard JFROG_REGISTRY: ${{ secrets.JFROG_REGISTRY }} DOCKER_BUILDKIT: "1" COMPOSE_DOCKER_CLI_BUILD: "1" steps: - name: Checkout (with tags) uses: actions/checkout@v4 with: fetch-depth: 0 # IMPORTANT to access tags - name: Verify tag is on prod branch (main) id: guard if: github.ref_type == 'tag' shell: bash run: | git fetch origin main --quiet if git merge-base --is-ancestor "${{ github.sha }}" origin/main; then echo "Tag commit is on main. Proceeding with prod deploy." else echo "::error::Tag ${GITHUB_REF#refs/tags/} points at a commit not on main. Refusing prod deploy." exit 1 fi - name: Set up kubeconfig run: | echo "KUBECONFIG=${RUNNER_TEMP}/kubeconfig" >> "$GITHUB_ENV" echo "${{ secrets.KUBE_CONFIG_DATA }}" | base64 -d > "${RUNNER_TEMP}/kubeconfig" chmod 600 "${RUNNER_TEMP}/kubeconfig" # Temporary debug step: we cannot shell into this self-hosted runner # directly, so this proves (or disproves) network reachability to the # k8s API from inside the runner itself, ahead of the image build/push # so we fail fast instead of waiting ~10 min for the helm timeout. # Remove once the ACL/connectivity issue is confirmed fixed. - name: "[DEBUG] Network path to k8s API (6443)" run: | set +e echo "===== Runner identity / network info =====" hostname -f 2>/dev/null || hostname echo "--- Interfaces ---" ip -brief addr show 2>/dev/null || ifconfig echo "--- Default route ---" ip route show 2>/dev/null || route -n echo echo "===== K8s API endpoint (parsed from kubeconfig) =====" KUBE_HOST=$(grep -m1 'server:' "${KUBECONFIG}" | sed -E 's#.*https?://([^:/]+).*#\1#') KUBE_PORT=$(grep -m1 'server:' "${KUBECONFIG}" | sed -E 's#.*:([0-9]+)/?.*#\1#') KUBE_PORT="${KUBE_PORT:-6443}" echo "Host: ${KUBE_HOST}" echo "Port: ${KUBE_PORT}" echo echo "===== DNS resolution =====" getent hosts "${KUBE_HOST}" || nslookup "${KUBE_HOST}" || dig +short "${KUBE_HOST}" RESOLVED_IPS=$(getent hosts "${KUBE_HOST}" | awk '{print $1}') echo echo "===== Route to each resolved IP =====" for ip in ${RESOLVED_IPS}; do echo "-- ip route get ${ip} --" ip route get "${ip}" 2>/dev/null || echo "(ip route not available)" done echo echo "===== TCP connect test per resolved IP (port ${KUBE_PORT}) =====" for ip in ${RESOLVED_IPS}; do if timeout 5 bash -c "cat < /dev/null > /dev/tcp/${ip}/${KUBE_PORT}" 2>/dev/null; then echo "OK ${ip}:${KUBE_PORT} - TCP connect succeeded" else echo "FAIL ${ip}:${KUBE_PORT} - TCP connect timed out or refused" fi done echo echo "===== TCP connect test via hostname (through NLB, port ${KUBE_PORT}) =====" if timeout 5 bash -c "cat < /dev/null > /dev/tcp/${KUBE_HOST}/${KUBE_PORT}" 2>/dev/null; then echo "OK ${KUBE_HOST}:${KUBE_PORT} - TCP connect succeeded" else echo "FAIL ${KUBE_HOST}:${KUBE_PORT} - TCP connect timed out or refused" fi echo echo "===== curl -v (expect HTTP 401/403 = reachable; a hang/timeout = blocked) =====" curl -v -k --max-time 8 "https://${KUBE_HOST}:${KUBE_PORT}/version" echo echo "===== traceroute (best effort, may be blocked itself) =====" if command -v traceroute >/dev/null 2>&1; then traceroute -T -p "${KUBE_PORT}" -w 2 -m 15 "${KUBE_HOST}" || true else echo "traceroute not installed on runner, skipping" fi set -e - name: Login to JFrog run: | # JFROG_REGISTRY is host/repo but `docker login` takes only the host. JFROG_HOST=$(echo "${JFROG_REGISTRY}" | cut -d'/' -f1) echo "Logging in to ${JFROG_HOST}" echo "${{ secrets.JFROG_IDENTITY_TOKEN }}" | docker login "${JFROG_HOST}" \ -u "${{ secrets.JFROG_USERNAME }}" \ --password-stdin - name: Determine ref type and release tag id: refinfo shell: bash run: | echo "GITHUB_REF=${GITHUB_REF}" echo "GITHUB_REF_TYPE=${GITHUB_REF_TYPE:-${{ github.ref_type }}}" if [[ "${{ github.ref_type }}" == "tag" ]]; then echo "ref_type=tag" >> $GITHUB_OUTPUT echo "release_tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT elif [[ "${{ github.ref_type }}" == "branch" ]]; then echo "ref_type=branch" >> $GITHUB_OUTPUT echo "release_tag=" >> $GITHUB_OUTPUT else echo "ref_type=unknown" >> $GITHUB_OUTPUT echo "release_tag=" >> $GITHUB_OUTPUT fi - name: Pre-build diagnostics run: | echo "Ref: ${GITHUB_REF}" echo "Ref type: ${{ steps.refinfo.outputs.ref_type }}" echo "Release tag: ${{ steps.refinfo.outputs.release_tag }}" echo "Environment: ${{ matrix.env }}" echo "Server image: ${JFROG_REGISTRY}/${{ matrix.image_name_server }}" echo "DB image: ${JFROG_REGISTRY}/${{ matrix.image_name_db }}" docker version helm version kubectl version --client - name: Build & Push server image run: | REF_TYPE="${{ steps.refinfo.outputs.ref_type }}" RELEASE_TAG="${{ steps.refinfo.outputs.release_tag }}" IMAGE="${JFROG_REGISTRY}/${{ matrix.image_name_server }}" BUILD_TAG_ARGS=( -t "${IMAGE}:${{ matrix.latest_tag }}" -t "${IMAGE}:${{ github.sha }}" ) if [[ "${REF_TYPE}" == "tag" && -n "${RELEASE_TAG}" ]]; then BUILD_TAG_ARGS+=( -t "${IMAGE}:${RELEASE_TAG}" ) fi # AUTH_SECRET is build-time only (Auth.js v5 initializes NextAuth() at # module load during `next build`). The real secret comes from k8s. docker build \ -f Dockerfile \ --build-arg AUTH_SECRET=build-placeholder-not-used-at-runtime \ "${BUILD_TAG_ARGS[@]}" \ . docker push "${IMAGE}:${{ matrix.latest_tag }}" docker push "${IMAGE}:${{ github.sha }}" if [[ "${REF_TYPE}" == "tag" && -n "${RELEASE_TAG}" ]]; then docker push "${IMAGE}:${RELEASE_TAG}" fi - name: Build & Push db image run: | REF_TYPE="${{ steps.refinfo.outputs.ref_type }}" RELEASE_TAG="${{ steps.refinfo.outputs.release_tag }}" IMAGE="${JFROG_REGISTRY}/${{ matrix.image_name_db }}" BUILD_TAG_ARGS=( -t "${IMAGE}:${{ matrix.latest_tag }}" -t "${IMAGE}:${{ github.sha }}" ) if [[ "${REF_TYPE}" == "tag" && -n "${RELEASE_TAG}" ]]; then BUILD_TAG_ARGS+=( -t "${IMAGE}:${RELEASE_TAG}" ) fi docker build \ -f Dockerfile.db \ "${BUILD_TAG_ARGS[@]}" \ . docker push "${IMAGE}:${{ matrix.latest_tag }}" docker push "${IMAGE}:${{ github.sha }}" if [[ "${REF_TYPE}" == "tag" && -n "${RELEASE_TAG}" ]]; then docker push "${IMAGE}:${RELEASE_TAG}" fi - name: Ensure namespace exists run: | # `helm --atomic` against a missing namespace fails outright. kubectl create namespace "${{ matrix.namespace }}" \ --dry-run=client -o yaml | kubectl apply -f - # ---- DEV DEPLOY (branch dev only) ---- # Deployed by commit SHA, not latest-dev: an immutable tag sidesteps the # moving-tag + IfNotPresent stale-cache trap and makes `kubectl describe # pod` show exactly which commit is live. - name: Deploy DEV with Helm if: github.ref_type == 'branch' && github.ref == 'refs/heads/dev' run: | kubectl get nodes kubectl get ns helm upgrade --install "${{ matrix.release }}" "${{ matrix.chart }}" \ -f "${{ matrix.values_file }}" \ --namespace "${{ matrix.namespace }}" \ --set server.image.repository="${JFROG_REGISTRY}/${{ matrix.image_name_server }}" \ --set server.image.tag="${{ github.sha }}" \ --set db.image.repository="${JFROG_REGISTRY}/${{ matrix.image_name_db }}" \ --set db.image.tag="${{ github.sha }}" \ --set-string secrets.authSecret="${{ secrets.AUTH_SECRET_DEV }}" \ --set-string secrets.dbRootPassword="${{ secrets.DB_PASSWORD_DEV }}" \ --set-string secrets.localAdminPassword="${{ secrets.LOCAL_ADMIN_PASSWORD_DEV }}" \ --wait --timeout 10m --debug # ---- PROD DEPLOY (tag push only) ---- - name: Deploy PROD with Helm if: github.ref_type == 'tag' run: | kubectl get nodes kubectl get ns TAG="${GITHUB_REF#refs/tags/}" if [[ -z "${TAG}" ]]; then echo "No tag detected; aborting prod deploy." exit 1 fi helm upgrade --install "${{ matrix.release }}" "${{ matrix.chart }}" \ -f "${{ matrix.values_file }}" \ --namespace "${{ matrix.namespace }}" \ --set server.image.repository="${JFROG_REGISTRY}/${{ matrix.image_name_server }}" \ --set server.image.tag="${TAG}" \ --set db.image.repository="${JFROG_REGISTRY}/${{ matrix.image_name_db }}" \ --set db.image.tag="${TAG}" \ --set-string secrets.authSecret="${{ secrets.AUTH_SECRET_PROD }}" \ --set-string secrets.dbRootPassword="${{ secrets.DB_PASSWORD_PROD }}" \ --set-string secrets.localAdminPassword="${{ secrets.LOCAL_ADMIN_PASSWORD_PROD }}" \ --wait --timeout 10m --atomic --debug # Strictly redundant with immutable image tags: Helm already rolled the # pods and --atomic already verified health. Retained as belt-and-braces. # Scoped to component=server so it does not bounce MySQL every push. - name: Rollout restart (server only) if: success() && (github.ref_type == 'branch' || github.ref_type == 'tag') run: | kubectl rollout restart deployment \ -l "app.kubernetes.io/instance=${{ matrix.release }},app.kubernetes.io/component=server" \ -n "${{ matrix.namespace }}" - name: Post-deploy diagnostics (if failure) if: failure() run: | echo "----- Diagnostics -----" kubectl get deploy,sts,po,svc,ing -n "${{ matrix.namespace }}" -o wide || true echo "----- PVCs (a Pending PVC = missing gp3 StorageClass; invisible in pod logs) -----" kubectl get pvc -n "${{ matrix.namespace }}" -o wide || true echo "----- Events -----" kubectl get events -n "${{ matrix.namespace }}" --sort-by=.lastTimestamp || true for p in $(kubectl get po -n "${{ matrix.namespace }}" -o name); do echo "### $p" kubectl describe "$p" -n "${{ matrix.namespace }}" || true kubectl logs "$p" -n "${{ matrix.namespace }}" --all-containers --tail=200 || true done