310 lines
13 KiB
YAML
310 lines
13 KiB
YAML
name: CI/CD (dev on branch, prod on tag)
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- dev # build & deploy for dev only
|
|
tags:
|
|
- "*" # build & deploy for prod only when a tag is pushed
|
|
|
|
jobs:
|
|
setup:
|
|
runs-on:
|
|
group: trend-emu-intranet
|
|
labels:
|
|
- "ubuntu-latest-intranet"
|
|
outputs:
|
|
matrix: ${{ steps.pick.outputs.matrix }}
|
|
steps:
|
|
- name: Pick matrix config from ref type
|
|
id: pick
|
|
shell: bash
|
|
run: |
|
|
if [[ "${{ github.ref_type }}" == "tag" ]]; then
|
|
echo 'matrix={"include":[{"env":"prod","image_name_server":"it-cost-dashboard-server-prod","image_name_db":"it-cost-dashboard-db-prod","latest_tag":"latest-prod","release":"it-cost-dashboard-prod","namespace":"it-cost-dashboard-prod","chart":"./helm","values_file":"./helm/values-prod.yaml"}]}' >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo 'matrix={"include":[{"env":"dev","image_name_server":"it-cost-dashboard-server-dev","image_name_db":"it-cost-dashboard-db-dev","latest_tag":"latest-dev","release":"it-cost-dashboard-dev","namespace":"it-cost-dashboard-dev","chart":"./helm","values_file":"./helm/values-dev.yaml"}]}' >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
build-and-deploy:
|
|
needs: setup
|
|
runs-on:
|
|
group: trend-emu-intranet
|
|
labels:
|
|
- "ubuntu-latest-intranet"
|
|
strategy:
|
|
matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
|
|
|
|
# Per-environment group: a dev push must never cancel an in-flight prod
|
|
# deploy (the original used a hardcoded "prod" group for both refs).
|
|
concurrency:
|
|
group: deploy-${{ matrix.env }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
# host/repo, e.g. jfrog.trendmicro.com/ameait-it-cost-dashboard
|
|
JFROG_REGISTRY: ${{ secrets.JFROG_REGISTRY }}
|
|
DOCKER_BUILDKIT: "1"
|
|
COMPOSE_DOCKER_CLI_BUILD: "1"
|
|
|
|
steps:
|
|
- name: Checkout (with tags)
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0 # IMPORTANT to access tags
|
|
|
|
- name: Verify tag is on prod branch (main)
|
|
id: guard
|
|
if: github.ref_type == 'tag'
|
|
shell: bash
|
|
run: |
|
|
git fetch origin main --quiet
|
|
if git merge-base --is-ancestor "${{ github.sha }}" origin/main; then
|
|
echo "Tag commit is on main. Proceeding with prod deploy."
|
|
else
|
|
echo "::error::Tag ${GITHUB_REF#refs/tags/} points at a commit not on main. Refusing prod deploy."
|
|
exit 1
|
|
fi
|
|
|
|
- name: Set up kubeconfig
|
|
run: |
|
|
echo "KUBECONFIG=${RUNNER_TEMP}/kubeconfig" >> "$GITHUB_ENV"
|
|
echo "${{ secrets.KUBE_CONFIG_DATA }}" | base64 -d > "${RUNNER_TEMP}/kubeconfig"
|
|
chmod 600 "${RUNNER_TEMP}/kubeconfig"
|
|
|
|
# Temporary debug step: we cannot shell into this self-hosted runner
|
|
# directly, so this proves (or disproves) network reachability to the
|
|
# k8s API from inside the runner itself, ahead of the image build/push
|
|
# so we fail fast instead of waiting ~10 min for the helm timeout.
|
|
# Remove once the ACL/connectivity issue is confirmed fixed.
|
|
- name: "[DEBUG] Network path to k8s API (6443)"
|
|
run: |
|
|
set +e
|
|
echo "===== Runner identity / network info ====="
|
|
hostname -f 2>/dev/null || hostname
|
|
echo "--- Interfaces ---"
|
|
ip -brief addr show 2>/dev/null || ifconfig
|
|
echo "--- Default route ---"
|
|
ip route show 2>/dev/null || route -n
|
|
|
|
echo
|
|
echo "===== K8s API endpoint (parsed from kubeconfig) ====="
|
|
KUBE_HOST=$(grep -m1 'server:' "${KUBECONFIG}" | sed -E 's#.*https?://([^:/]+).*#\1#')
|
|
KUBE_PORT=$(grep -m1 'server:' "${KUBECONFIG}" | sed -E 's#.*:([0-9]+)/?.*#\1#')
|
|
KUBE_PORT="${KUBE_PORT:-6443}"
|
|
echo "Host: ${KUBE_HOST}"
|
|
echo "Port: ${KUBE_PORT}"
|
|
|
|
echo
|
|
echo "===== DNS resolution ====="
|
|
getent hosts "${KUBE_HOST}" || nslookup "${KUBE_HOST}" || dig +short "${KUBE_HOST}"
|
|
|
|
RESOLVED_IPS=$(getent hosts "${KUBE_HOST}" | awk '{print $1}')
|
|
|
|
echo
|
|
echo "===== Route to each resolved IP ====="
|
|
for ip in ${RESOLVED_IPS}; do
|
|
echo "-- ip route get ${ip} --"
|
|
ip route get "${ip}" 2>/dev/null || echo "(ip route not available)"
|
|
done
|
|
|
|
echo
|
|
echo "===== TCP connect test per resolved IP (port ${KUBE_PORT}) ====="
|
|
for ip in ${RESOLVED_IPS}; do
|
|
if timeout 5 bash -c "cat < /dev/null > /dev/tcp/${ip}/${KUBE_PORT}" 2>/dev/null; then
|
|
echo "OK ${ip}:${KUBE_PORT} - TCP connect succeeded"
|
|
else
|
|
echo "FAIL ${ip}:${KUBE_PORT} - TCP connect timed out or refused"
|
|
fi
|
|
done
|
|
|
|
echo
|
|
echo "===== TCP connect test via hostname (through NLB, port ${KUBE_PORT}) ====="
|
|
if timeout 5 bash -c "cat < /dev/null > /dev/tcp/${KUBE_HOST}/${KUBE_PORT}" 2>/dev/null; then
|
|
echo "OK ${KUBE_HOST}:${KUBE_PORT} - TCP connect succeeded"
|
|
else
|
|
echo "FAIL ${KUBE_HOST}:${KUBE_PORT} - TCP connect timed out or refused"
|
|
fi
|
|
|
|
echo
|
|
echo "===== curl -v (expect HTTP 401/403 = reachable; a hang/timeout = blocked) ====="
|
|
curl -v -k --max-time 8 "https://${KUBE_HOST}:${KUBE_PORT}/version"
|
|
|
|
echo
|
|
echo "===== traceroute (best effort, may be blocked itself) ====="
|
|
if command -v traceroute >/dev/null 2>&1; then
|
|
traceroute -T -p "${KUBE_PORT}" -w 2 -m 15 "${KUBE_HOST}" || true
|
|
else
|
|
echo "traceroute not installed on runner, skipping"
|
|
fi
|
|
set -e
|
|
|
|
- name: Login to JFrog
|
|
run: |
|
|
# JFROG_REGISTRY is host/repo but `docker login` takes only the host.
|
|
JFROG_HOST=$(echo "${JFROG_REGISTRY}" | cut -d'/' -f1)
|
|
echo "Logging in to ${JFROG_HOST}"
|
|
echo "${{ secrets.JFROG_IDENTITY_TOKEN }}" | docker login "${JFROG_HOST}" \
|
|
-u "${{ secrets.JFROG_USERNAME }}" \
|
|
--password-stdin
|
|
|
|
- name: Determine ref type and release tag
|
|
id: refinfo
|
|
shell: bash
|
|
run: |
|
|
echo "GITHUB_REF=${GITHUB_REF}"
|
|
echo "GITHUB_REF_TYPE=${GITHUB_REF_TYPE:-${{ github.ref_type }}}"
|
|
if [[ "${{ github.ref_type }}" == "tag" ]]; then
|
|
echo "ref_type=tag" >> $GITHUB_OUTPUT
|
|
echo "release_tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
|
|
elif [[ "${{ github.ref_type }}" == "branch" ]]; then
|
|
echo "ref_type=branch" >> $GITHUB_OUTPUT
|
|
echo "release_tag=" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "ref_type=unknown" >> $GITHUB_OUTPUT
|
|
echo "release_tag=" >> $GITHUB_OUTPUT
|
|
fi
|
|
|
|
- name: Pre-build diagnostics
|
|
run: |
|
|
echo "Ref: ${GITHUB_REF}"
|
|
echo "Ref type: ${{ steps.refinfo.outputs.ref_type }}"
|
|
echo "Release tag: ${{ steps.refinfo.outputs.release_tag }}"
|
|
echo "Environment: ${{ matrix.env }}"
|
|
echo "Server image: ${JFROG_REGISTRY}/${{ matrix.image_name_server }}"
|
|
echo "DB image: ${JFROG_REGISTRY}/${{ matrix.image_name_db }}"
|
|
docker version
|
|
helm version
|
|
kubectl version --client
|
|
|
|
- name: Build & Push server image
|
|
run: |
|
|
REF_TYPE="${{ steps.refinfo.outputs.ref_type }}"
|
|
RELEASE_TAG="${{ steps.refinfo.outputs.release_tag }}"
|
|
IMAGE="${JFROG_REGISTRY}/${{ matrix.image_name_server }}"
|
|
|
|
BUILD_TAG_ARGS=(
|
|
-t "${IMAGE}:${{ matrix.latest_tag }}"
|
|
-t "${IMAGE}:${{ github.sha }}"
|
|
)
|
|
if [[ "${REF_TYPE}" == "tag" && -n "${RELEASE_TAG}" ]]; then
|
|
BUILD_TAG_ARGS+=( -t "${IMAGE}:${RELEASE_TAG}" )
|
|
fi
|
|
|
|
# AUTH_SECRET is build-time only (Auth.js v5 initializes NextAuth() at
|
|
# module load during `next build`). The real secret comes from k8s.
|
|
docker build \
|
|
-f Dockerfile \
|
|
--build-arg AUTH_SECRET=build-placeholder-not-used-at-runtime \
|
|
"${BUILD_TAG_ARGS[@]}" \
|
|
.
|
|
|
|
docker push "${IMAGE}:${{ matrix.latest_tag }}"
|
|
docker push "${IMAGE}:${{ github.sha }}"
|
|
if [[ "${REF_TYPE}" == "tag" && -n "${RELEASE_TAG}" ]]; then
|
|
docker push "${IMAGE}:${RELEASE_TAG}"
|
|
fi
|
|
|
|
- name: Build & Push db image
|
|
run: |
|
|
REF_TYPE="${{ steps.refinfo.outputs.ref_type }}"
|
|
RELEASE_TAG="${{ steps.refinfo.outputs.release_tag }}"
|
|
IMAGE="${JFROG_REGISTRY}/${{ matrix.image_name_db }}"
|
|
|
|
BUILD_TAG_ARGS=(
|
|
-t "${IMAGE}:${{ matrix.latest_tag }}"
|
|
-t "${IMAGE}:${{ github.sha }}"
|
|
)
|
|
if [[ "${REF_TYPE}" == "tag" && -n "${RELEASE_TAG}" ]]; then
|
|
BUILD_TAG_ARGS+=( -t "${IMAGE}:${RELEASE_TAG}" )
|
|
fi
|
|
|
|
docker build \
|
|
-f Dockerfile.db \
|
|
"${BUILD_TAG_ARGS[@]}" \
|
|
.
|
|
|
|
docker push "${IMAGE}:${{ matrix.latest_tag }}"
|
|
docker push "${IMAGE}:${{ github.sha }}"
|
|
if [[ "${REF_TYPE}" == "tag" && -n "${RELEASE_TAG}" ]]; then
|
|
docker push "${IMAGE}:${RELEASE_TAG}"
|
|
fi
|
|
|
|
- name: Ensure namespace exists
|
|
run: |
|
|
# `helm --atomic` against a missing namespace fails outright.
|
|
kubectl create namespace "${{ matrix.namespace }}" \
|
|
--dry-run=client -o yaml | kubectl apply -f -
|
|
|
|
# ---- DEV DEPLOY (branch dev only) ----
|
|
# Deployed by commit SHA, not latest-dev: an immutable tag sidesteps the
|
|
# moving-tag + IfNotPresent stale-cache trap and makes `kubectl describe
|
|
# pod` show exactly which commit is live.
|
|
- name: Deploy DEV with Helm
|
|
if: github.ref_type == 'branch' && github.ref == 'refs/heads/dev'
|
|
run: |
|
|
kubectl get nodes
|
|
kubectl get ns
|
|
|
|
helm upgrade --install "${{ matrix.release }}" "${{ matrix.chart }}" \
|
|
-f "${{ matrix.values_file }}" \
|
|
--namespace "${{ matrix.namespace }}" \
|
|
--set server.image.repository="${JFROG_REGISTRY}/${{ matrix.image_name_server }}" \
|
|
--set server.image.tag="${{ github.sha }}" \
|
|
--set db.image.repository="${JFROG_REGISTRY}/${{ matrix.image_name_db }}" \
|
|
--set db.image.tag="${{ github.sha }}" \
|
|
--set-string secrets.authSecret="${{ secrets.AUTH_SECRET_DEV }}" \
|
|
--set-string secrets.dbRootPassword="${{ secrets.DB_ROOT_PASSWORD_DEV }}" \
|
|
--set-string secrets.localAdminPassword="${{ secrets.LOCAL_ADMIN_PASSWORD_DEV }}" \
|
|
--wait --timeout 10m --atomic --debug
|
|
|
|
# ---- PROD DEPLOY (tag push only) ----
|
|
- name: Deploy PROD with Helm
|
|
if: github.ref_type == 'tag'
|
|
run: |
|
|
kubectl get nodes
|
|
kubectl get ns
|
|
|
|
TAG="${GITHUB_REF#refs/tags/}"
|
|
if [[ -z "${TAG}" ]]; then
|
|
echo "No tag detected; aborting prod deploy."
|
|
exit 1
|
|
fi
|
|
|
|
helm upgrade --install "${{ matrix.release }}" "${{ matrix.chart }}" \
|
|
-f "${{ matrix.values_file }}" \
|
|
--namespace "${{ matrix.namespace }}" \
|
|
--set server.image.repository="${JFROG_REGISTRY}/${{ matrix.image_name_server }}" \
|
|
--set server.image.tag="${TAG}" \
|
|
--set db.image.repository="${JFROG_REGISTRY}/${{ matrix.image_name_db }}" \
|
|
--set db.image.tag="${TAG}" \
|
|
--set-string secrets.authSecret="${{ secrets.AUTH_SECRET_PROD }}" \
|
|
--set-string secrets.dbRootPassword="${{ secrets.DB_ROOT_PASSWORD_PROD }}" \
|
|
--set-string secrets.localAdminPassword="${{ secrets.LOCAL_ADMIN_PASSWORD_PROD }}" \
|
|
--wait --timeout 10m --atomic --debug
|
|
|
|
# Strictly redundant with immutable image tags: Helm already rolled the
|
|
# pods and --atomic already verified health. Retained as belt-and-braces.
|
|
# Scoped to component=server so it does not bounce MySQL every push.
|
|
- name: Rollout restart (server only)
|
|
if: success() && (github.ref_type == 'branch' || github.ref_type == 'tag')
|
|
run: |
|
|
kubectl rollout restart deployment \
|
|
-l "app.kubernetes.io/instance=${{ matrix.release }},app.kubernetes.io/component=server" \
|
|
-n "${{ matrix.namespace }}"
|
|
|
|
- name: Post-deploy diagnostics (if failure)
|
|
if: failure()
|
|
run: |
|
|
echo "----- Diagnostics -----"
|
|
kubectl get deploy,sts,po,svc,ing -n "${{ matrix.namespace }}" -o wide || true
|
|
echo "----- PVCs (a Pending PVC = missing gp3 StorageClass; invisible in pod logs) -----"
|
|
kubectl get pvc -n "${{ matrix.namespace }}" -o wide || true
|
|
echo "----- Events -----"
|
|
kubectl get events -n "${{ matrix.namespace }}" --sort-by=.lastTimestamp || true
|
|
for p in $(kubectl get po -n "${{ matrix.namespace }}" -o name); do
|
|
echo "### $p"
|
|
kubectl describe "$p" -n "${{ matrix.namespace }}" || true
|
|
kubectl logs "$p" -n "${{ matrix.namespace }}" --all-containers --tail=200 || true
|
|
done
|