diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1bfee24..ba8cf24 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,106 +1,88 @@ -deploy_test: - stage: deploy_test - image: ubuntu:22.04 - before_script: - - | - set -euo pipefail - apt-get update && apt-get install -y curl ca-certificates jq +stages: [build, deploy] - echo "Installing kubectl ${KUBECTL_VERSION}..." - curl -LO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" - chmod +x kubectl && mv kubectl /usr/local/bin/ - kubectl version --client +variables: + IMAGE_REPO: "$CI_REGISTRY_IMAGE" + TAG: "${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" + RELEASE: "open-meteo-service-gitlab" + NAMESPACE: "sandbox-gitlab" + CHART_PATH: "./open-meteo-service" + VALUES_FILE: "./open-meteo-service/values.yaml" + SERVICE_NAME: "open-meteo-service-open-meteo-service" # <-- אם אצלך שם אחר, תשנה פה - echo "Installing kind ${KIND_VERSION}..." - curl -Lo kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" - chmod +x kind && mv kind /usr/local/bin/ - kind version - - echo "Installing Helm ${HELM_VERSION}..." - curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -s -- --version ${HELM_VERSION} - helm version +build: + stage: build + tags: [homelab] script: - - | - set -euo pipefail + - docker login "$CI_REGISTRY" -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" + - docker build -t "$IMAGE_REPO:$TAG" . + - docker push "$IMAGE_REPO:$TAG" + - docker tag "$IMAGE_REPO:$TAG" "$IMAGE_REPO:latest" + - docker push "$IMAGE_REPO:latest" + - echo "Built and pushed $IMAGE_REPO:$TAG and $IMAGE_REPO:latest" - echo "Creating kind cluster..." - kind create cluster --name ci-cluster --wait 5m - kubectl get nodes - kubectl cluster-info +deploy: + stage: deploy + tags: [homelab] + rules: + - if: '$CI_COMMIT_MESSAGE =~ /^ci: bump image tag/' + when: never + - when: on_success + script: |- + set -euo pipefail - echo "Creating imagePullSecret for GitLab Registry..." - kubectl create secret docker-registry gitlab-registry \ - --docker-server="${CI_REGISTRY}" \ - --docker-username="${CI_REGISTRY_USER}" \ - --docker-password="${CI_REGISTRY_PASSWORD}" \ - --namespace="${NAMESPACE}" \ - --dry-run=client -o yaml | kubectl apply -f - + # Install yq if missing + if ! command -v yq >/dev/null 2>&1; then + YQ_VERSION="v4.35.1" + wget -qO /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" + chmod +x /usr/local/bin/yq + fi - echo "Deploying Helm chart..." - helm upgrade --install ${HELM_RELEASE_NAME} ${HELM_CHART_PATH} \ - --namespace ${NAMESPACE} --create-namespace \ - --set image.repository="${CI_REGISTRY_IMAGE}" \ - --set image.tag="${CI_COMMIT_SHORT_SHA}" \ - --set image.pullPolicy=Always \ - --set imagePullSecrets[0].name=gitlab-registry \ - --set prometheus.enabled=false \ - --set grafana.enabled=false \ - --wait --timeout 5m + # Update values.yaml with the new tag + yq e ".image.tag = \"${TAG}\"" -i "${VALUES_FILE}" + echo "Updated ${VALUES_FILE} with image.tag=${TAG}" - echo "Cluster state after deploy:" - kubectl get pods -n ${NAMESPACE} - kubectl get svc -n ${NAMESPACE} + # Configure git identity + git config user.email "gitlab-ci@dvirlabs.com" + git config user.name "GitLab CI" - echo "Waiting for API pod to become Ready..." - if ! kubectl wait --for=condition=ready pod \ - -l app.kubernetes.io/name=open-meteo-service \ - -n ${NAMESPACE} --timeout=180s; then - echo "ERROR: Pod not ready. Debug info:" - kubectl get pods -n ${NAMESPACE} - kubectl describe pods -l app.kubernetes.io/name=open-meteo-service -n ${NAMESPACE} - kubectl logs -l app.kubernetes.io/name=open-meteo-service -n ${NAMESPACE} --tail=200 - exit 1 - fi + # Commit & push values bump (skip-ci to avoid loop) + git add "${VALUES_FILE}" + git commit -m "ci: bump image tag to ${TAG} [skip ci]" || echo "No changes to commit" - echo "Starting port-forward in background..." - kubectl port-forward -n ${NAMESPACE} svc/${SERVICE_NAME} 8000:8000 >/tmp/pf.log 2>&1 & - PF_PID=$! - sleep 5 + git remote set-url origin "https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" + git push origin "HEAD:${CI_COMMIT_REF_NAME}" || echo "Nothing to push" - echo "Testing /healthz..." - for i in $(seq 1 10); do - if curl -fsS --max-time 5 http://localhost:8000/healthz >/dev/null; then - echo "SUCCESS: /healthz endpoint is healthy" - break - fi - echo "Retry $i..." - sleep 3 - if [ "$i" -eq 10 ]; then - echo "ERROR: /healthz failed after 10 attempts" - echo "--- port-forward logs ---" - cat /tmp/pf.log || true - exit 1 - fi - done + echo "Deploying with Helm..." + helm upgrade --install "${RELEASE}" "${CHART_PATH}" \ + -n "${NAMESPACE}" \ + --create-namespace \ + --wait --timeout 5m - echo "Testing /coordinates..." - COORD_RESPONSE="$(curl -fsS --max-time 15 http://localhost:8000/coordinates)" - echo "$COORD_RESPONSE" | head -50 - if ! echo "$COORD_RESPONSE" | grep -q '"data"'; then - echo "ERROR: missing data field in response" - exit 1 - fi - echo "SUCCESS: /coordinates endpoint working" + echo "Waiting for deployment to be ready..." + kubectl -n "${NAMESPACE}" rollout status deploy -l app.kubernetes.io/name=open-meteo-service --timeout=180s - echo "Testing /metrics..." - curl -fsS --max-time 10 http://localhost:8000/metrics | head -30 - echo "SUCCESS: /metrics endpoint working" + echo "Port-forward for tests..." + kubectl -n "${NAMESPACE}" port-forward "svc/${SERVICE_NAME}" 8000:8000 >/tmp/pf.log 2>&1 & + PF_PID=$! + sleep 5 - echo "SUCCESS: All tests passed" + echo "Testing /healthz..." + curl -fsS http://localhost:8000/healthz >/dev/null + echo "OK /healthz" + + echo "Testing /coordinates..." + COORDS_RESPONSE="$(curl -fsS http://localhost:8000/coordinates)" + echo "$COORDS_RESPONSE" | head -20 + if ! echo "$COORDS_RESPONSE" | grep -q '"data"'; then + echo "ERROR: /coordinates response missing 'data' field" kill $PF_PID || true - after_script: - - | - echo "Final cluster state:" - kubectl get all -n ${NAMESPACE} || true - echo "Deleting kind cluster..." - kind delete cluster --name ci-cluster || true + exit 1 + fi + echo "OK /coordinates" + + echo "Testing /metrics..." + curl -fsS http://localhost:8000/metrics | head -20 + echo "OK /metrics" + + kill $PF_PID || true + echo "All tests passed!" diff --git a/README.md b/README.md index b8a38b6..cbc0ca0 100644 --- a/README.md +++ b/README.md @@ -99,69 +99,76 @@ helm install open-meteo-service ./helm \ --set grafana.persistence.enabled=true ``` -## CI Pipeline +## GitLab CI Workflow -This project includes a GitLab CI/CD pipeline that automatically builds, deploys, and tests the application without requiring Docker-in-Docker. +This project uses a GitLab CI/CD pipeline that automates building, versioning, deployment, and testing on an external k3s cluster (homelab). -### Pipeline Stages +### Pipeline Overview -1. **Build Stage (Kaniko)** - - Builds Docker image using Kaniko (secure, daemonless container builder) - - Pushes to GitLab Container Registry with commit-based tags - - Tags `latest` for main branch builds - - Uses layer caching for faster builds +The pipeline consists of two stages: **build** and **deploy**. -2. **Deploy Stage (kind + Helm)** - - Creates a local Kubernetes cluster using kind (Kubernetes in Docker) - - Installs kubectl, kind, and Helm - - Configures image pull secrets for GitLab registry access - - Deploys app using Helm with the newly built image - - Waits for deployment readiness with automatic rollback on failure +#### 1. Build Stage +- **Docker Image Build**: Builds the application Docker image using the project Dockerfile +- **Tagging Strategy**: Tags each image with a unique identifier combining branch and commit SHA: + - Format: `-` (e.g., `main-a1b2c3d`, `feature-api-xyz1234`) + - Uses `CI_COMMIT_REF_SLUG` (sanitized branch name) and `CI_COMMIT_SHORT_SHA` +- **Multi-Tag Push**: Pushes two tags to GitLab Container Registry: + - `$IMAGE_REPO:-` - Specific version tag + - `$IMAGE_REPO:latest` - Always points to the most recent build +- **Registry Authentication**: Uses GitLab's built-in CI variables (`CI_REGISTRY`, `CI_REGISTRY_USER`, `CI_REGISTRY_PASSWORD`) -3. **Test Stage (Health Checks)** - - Validates deployment with port-forwarding - - Tests `/healthz` endpoint for service health - - Tests `/coordinates` endpoint for data retrieval - - Tests `/help` endpoint for API documentation - - Provides detailed debugging info if tests fail - - Cleans up kind cluster after tests +#### 2. Deploy Stage +- **Values File Update**: + - Uses `yq` to update `open-meteo-service/values.yaml` + - Sets `image.tag` to the newly built tag (`-`) + - Keeps `image.repository` constant in the values file + - Commits the change with message: `ci: bump image tag to [skip ci]` + - Pushes back to the repository using `CI_JOB_TOKEN` for authentication +- **Helm Deployment**: + - Deploys to namespace: `sandbox-gitlab` + - Release name: `open-meteo-service-gitlab` + - Uses the updated values.yaml file from the workspace + - Waits for deployment to be ready (5-minute timeout) +- **Automated Tests**: + - Port-forwards the service to localhost:8000 + - `/healthz` - Checks service health + - `/coordinates` - Validates JSON response contains `"data"` field + - `/metrics` - Ensures metrics endpoint is accessible + - All tests must pass for the pipeline to succeed -### Image Tagging Strategy -- **Commit builds**: `$CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA` -- **Main branch**: Also tagged as `latest` +### Preventing Infinite Loops + +The pipeline includes safeguards to prevent infinite pipeline triggers: +- The deploy job includes a rule: `if: '$CI_COMMIT_MESSAGE =~ /^ci: bump image tag/'` with `when: never` +- The values bump commit includes `[skip ci]` to prevent triggering a new pipeline +- The deploy job uses the already-updated values.yaml in the current workspace + +### Deployment Target + +- **Cluster**: External k3s homelab cluster (not ephemeral) +- **Runner**: Self-hosted GitLab runner tagged with `homelab` +- **Namespace**: `sandbox-gitlab` (separate from ArgoCD/Harbor setups) +- **Ingress**: Configured for `open-meteo-gitlab.dvirlabs.com` (see values.yaml) ### Viewing Pipeline Status -1. Go to your GitLab project -2. Navigate to **CI/CD > Pipelines** -3. Click on a pipeline run to see individual job logs -4. Each stage shows real-time logs and can be expanded for details -### Debugging Failed Pipelines -If a pipeline fails: -- Check the **build** job logs for Docker build errors -- Check the **deploy** job logs for Kubernetes/Helm issues - - Pod status and events are automatically logged - - Container logs are shown if deployment fails -- Check the **test** job logs for endpoint test failures - - Response bodies are printed for debugging +1. Navigate to **CI/CD > Pipelines** in your GitLab project +2. Click on any pipeline run to see job details +3. Expand jobs to view real-time logs +4. The deploy job shows Helm output, rollout status, and test results -### Running Locally -To test the pipeline behavior locally: +### Manual Deployment + +To deploy manually with a specific tag: ```bash -# Build with Kaniko (requires Docker) -docker run --rm -v $(pwd):/workspace gcr.io/kaniko-project/executor:latest \ - --dockerfile /workspace/Dockerfile \ - --context /workspace \ - --no-push +# Option 1: Deploy using updated values.yaml +helm upgrade --install open-meteo-service-gitlab ./open-meteo-service \ + -n sandbox-gitlab --create-namespace -# Test with kind -kind create cluster -kubectl create secret docker-registry gitlab-registry \ - --docker-server=your-registry \ - --docker-username=your-user \ - --docker-password=your-token -helm install open-meteo-service ./open-meteo-service \ - --set imagePullSecrets[0].name=gitlab-registry +# Option 2: Override tag via --set +helm upgrade --install open-meteo-service-gitlab ./open-meteo-service \ + -n sandbox-gitlab --create-namespace \ + --set image.tag=main-a1b2c3d ``` ## API Documentation diff --git a/open-meteo-service/values.yaml b/open-meteo-service/values.yaml index c68f521..47626fa 100644 --- a/open-meteo-service/values.yaml +++ b/open-meteo-service/values.yaml @@ -1,12 +1,12 @@ replicaCount: 1 image: - repository: harbor.dvirlabs.com/library/open-meteo-service - tag: "1.0.3" + repository: registry.gitlab.com/dvirlabs/open-meteo-service + tag: "latest" pullPolicy: IfNotPresent imagePullSecrets: - - name: harbor-regcred + - name: gitlab-registry service: type: ClusterIP @@ -14,9 +14,9 @@ service: ingress: enabled: false - className: "traefik" + className: "" hosts: - - host: open-meteo.dvirlabs.com + - host: open-meteo-gitlab.dvirlabs.com paths: - path: / pathType: Prefix @@ -35,7 +35,7 @@ persistence: accessModes: - ReadWriteOnce size: 1Gi - storageClassName: "standard" + storageClassName: "" resources: {} @@ -54,9 +54,9 @@ prometheus: port: 9090 ingress: enabled: false - className: "traefik" + className: "" hosts: - - host: open-meteo-prometheus.dvirlabs.com + - host: prometheus.local paths: - path: / pathType: Prefix @@ -66,7 +66,7 @@ prometheus: accessModes: - ReadWriteOnce size: 5Gi - storageClassName: "standard" + storageClassName: "" resources: {} extraArgs: [] @@ -81,9 +81,9 @@ grafana: adminPassword: admin ingress: enabled: false - className: "traefik" + className: "" hosts: - - host: open-meteo-grafana.dvirlabs.com + - host: grafana.local paths: - path: / pathType: Prefix @@ -93,5 +93,5 @@ grafana: accessModes: - ReadWriteOnce size: 5Gi - storageClassName: "standard" - resources: {} \ No newline at end of file + storageClassName: "" + resources: {} diff --git a/values.yaml b/values.yaml new file mode 100644 index 0000000..174b350 --- /dev/null +++ b/values.yaml @@ -0,0 +1,97 @@ +replicaCount: 1 + +image: + repository: registry.gitlab.com/dvirlabs/open-meteo-service + tag: "latest" + pullPolicy: IfNotPresent + +imagePullSecrets: +- name: gitlab-registry + +service: + type: ClusterIP + port: 8000 + +ingress: + enabled: true + className: "traefik" + hosts: + - host: open-meteo-gitlab.dvirlabs.com + paths: + - path: / + pathType: Prefix + tls: [] + +podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/path: /metrics + prometheus.io/port: "8000" + +env: + cacheFile: /data/coordinates_cache.json + +persistence: + enabled: false + accessModes: + - ReadWriteOnce + size: 1Gi + storageClassName: "nfs-client" + +resources: {} + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +prometheus: + enabled: true + replicaCount: 1 + image: prom/prometheus:latest + service: + type: ClusterIP + port: 9090 + ingress: + enabled: true + className: "traefik" + hosts: + - host: open-meteo-prometheus-gitlab.dvirlabs.com + paths: + - path: / + pathType: Prefix + tls: [] + persistence: + enabled: true + accessModes: + - ReadWriteOnce + size: 5Gi + storageClassName: "nfs-client" + resources: {} + extraArgs: [] + +grafana: + enabled: true + replicaCount: 1 + image: grafana/grafana:latest + service: + type: ClusterIP + port: 3000 + adminUser: admin + adminPassword: admin + ingress: + enabled: true + className: "traefik" + hosts: + - host: open-meteo-grafana-gitlab.dvirlabs.com + paths: + - path: / + pathType: Prefix + tls: [] + persistence: + enabled: true + accessModes: + - ReadWriteOnce + size: 5Gi + storageClassName: "nfs-client" + resources: {}