Update the pipeline to deploy on my k3s cluster andupdate the values

2026-02-18 04:04:35 +02:00 · 2026-02-18 04:04:35 +02:00 · 76d3359896
commit 76d3359896
parent 572e14d1dd
4 changed files with 242 additions and 156 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -1,106 +1,88 @@
-deploy_test:
-  stage: deploy_test
-  image: ubuntu:22.04
-  before_script:
-    - |
-      set -euo pipefail
-      apt-get update && apt-get install -y curl ca-certificates jq
+stages: [build, deploy]

-      echo "Installing kubectl ${KUBECTL_VERSION}..."
-      curl -LO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
-      chmod +x kubectl && mv kubectl /usr/local/bin/
-      kubectl version --client
+variables:
+  IMAGE_REPO: "$CI_REGISTRY_IMAGE"
+  TAG: "${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}"
+  RELEASE: "open-meteo-service-gitlab"
+  NAMESPACE: "sandbox-gitlab"
+  CHART_PATH: "./open-meteo-service"
+  VALUES_FILE: "./open-meteo-service/values.yaml"
+  SERVICE_NAME: "open-meteo-service-open-meteo-service"  # <-- אם אצלך שם אחר, תשנה פה

-      echo "Installing kind ${KIND_VERSION}..."
-      curl -Lo kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64"
-      chmod +x kind && mv kind /usr/local/bin/
-      kind version
-
-      echo "Installing Helm ${HELM_VERSION}..."
-      curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -s -- --version ${HELM_VERSION}
-      helm version
+build:
+  stage: build
+  tags: [homelab]
  script:
-    - |
+    - docker login "$CI_REGISTRY" -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD"
+    - docker build -t "$IMAGE_REPO:$TAG" .
+    - docker push "$IMAGE_REPO:$TAG"
+    - docker tag "$IMAGE_REPO:$TAG" "$IMAGE_REPO:latest"
+    - docker push "$IMAGE_REPO:latest"
+    - echo "Built and pushed $IMAGE_REPO:$TAG and $IMAGE_REPO:latest"
+
+deploy:
+  stage: deploy
+  tags: [homelab]
+  rules:
+    - if: '$CI_COMMIT_MESSAGE =~ /^ci: bump image tag/'
+      when: never
+    - when: on_success
+  script: |-
    set -euo pipefail

-      echo "Creating kind cluster..."
-      kind create cluster --name ci-cluster --wait 5m
-      kubectl get nodes
-      kubectl cluster-info
-
-      echo "Creating imagePullSecret for GitLab Registry..."
-      kubectl create secret docker-registry gitlab-registry \
-        --docker-server="${CI_REGISTRY}" \
-        --docker-username="${CI_REGISTRY_USER}" \
-        --docker-password="${CI_REGISTRY_PASSWORD}" \
-        --namespace="${NAMESPACE}" \
-        --dry-run=client -o yaml | kubectl apply -f -
-
-      echo "Deploying Helm chart..."
-      helm upgrade --install ${HELM_RELEASE_NAME} ${HELM_CHART_PATH} \
-        --namespace ${NAMESPACE} --create-namespace \
-        --set image.repository="${CI_REGISTRY_IMAGE}" \
-        --set image.tag="${CI_COMMIT_SHORT_SHA}" \
-        --set image.pullPolicy=Always \
-        --set imagePullSecrets[0].name=gitlab-registry \
-        --set prometheus.enabled=false \
-        --set grafana.enabled=false \
-        --wait --timeout 5m
-
-      echo "Cluster state after deploy:"
-      kubectl get pods -n ${NAMESPACE}
-      kubectl get svc -n ${NAMESPACE}
-
-      echo "Waiting for API pod to become Ready..."
-      if ! kubectl wait --for=condition=ready pod \
-        -l app.kubernetes.io/name=open-meteo-service \
-        -n ${NAMESPACE} --timeout=180s; then
-        echo "ERROR: Pod not ready. Debug info:"
-        kubectl get pods -n ${NAMESPACE}
-        kubectl describe pods -l app.kubernetes.io/name=open-meteo-service -n ${NAMESPACE}
-        kubectl logs -l app.kubernetes.io/name=open-meteo-service -n ${NAMESPACE} --tail=200
-        exit 1
+    # Install yq if missing
+    if ! command -v yq >/dev/null 2>&1; then
+      YQ_VERSION="v4.35.1"
+      wget -qO /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64"
+      chmod +x /usr/local/bin/yq
    fi

-      echo "Starting port-forward in background..."
-      kubectl port-forward -n ${NAMESPACE} svc/${SERVICE_NAME} 8000:8000 >/tmp/pf.log 2>&1 &
+    # Update values.yaml with the new tag
+    yq e ".image.tag = \"${TAG}\"" -i "${VALUES_FILE}"
+    echo "Updated ${VALUES_FILE} with image.tag=${TAG}"
+
+    # Configure git identity
+    git config user.email "gitlab-ci@dvirlabs.com"
+    git config user.name "GitLab CI"
+
+    # Commit & push values bump (skip-ci to avoid loop)
+    git add "${VALUES_FILE}"
+    git commit -m "ci: bump image tag to ${TAG} [skip ci]" || echo "No changes to commit"
+
+    git remote set-url origin "https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git"
+    git push origin "HEAD:${CI_COMMIT_REF_NAME}" || echo "Nothing to push"
+
+    echo "Deploying with Helm..."
+    helm upgrade --install "${RELEASE}" "${CHART_PATH}" \
+      -n "${NAMESPACE}" \
+      --create-namespace \
+      --wait --timeout 5m
+
+    echo "Waiting for deployment to be ready..."
+    kubectl -n "${NAMESPACE}" rollout status deploy -l app.kubernetes.io/name=open-meteo-service --timeout=180s
+
+    echo "Port-forward for tests..."
+    kubectl -n "${NAMESPACE}" port-forward "svc/${SERVICE_NAME}" 8000:8000 >/tmp/pf.log 2>&1 &
    PF_PID=$!
    sleep 5

    echo "Testing /healthz..."
-      for i in $(seq 1 10); do
-        if curl -fsS --max-time 5 http://localhost:8000/healthz >/dev/null; then
-          echo "SUCCESS: /healthz endpoint is healthy"
-          break
-        fi
-        echo "Retry $i..."
-        sleep 3
-        if [ "$i" -eq 10 ]; then
-          echo "ERROR: /healthz failed after 10 attempts"
-          echo "--- port-forward logs ---"
-          cat /tmp/pf.log || true
-          exit 1
-        fi
-      done
+    curl -fsS http://localhost:8000/healthz >/dev/null
+    echo "OK /healthz"

    echo "Testing /coordinates..."
-      COORD_RESPONSE="$(curl -fsS --max-time 15 http://localhost:8000/coordinates)"
-      echo "$COORD_RESPONSE" | head -50
-      if ! echo "$COORD_RESPONSE" | grep -q '"data"'; then
-        echo "ERROR: missing data field in response"
+    COORDS_RESPONSE="$(curl -fsS http://localhost:8000/coordinates)"
+    echo "$COORDS_RESPONSE" | head -20
+    if ! echo "$COORDS_RESPONSE" | grep -q '"data"'; then
+      echo "ERROR: /coordinates response missing 'data' field"
+      kill $PF_PID || true
      exit 1
    fi
-      echo "SUCCESS: /coordinates endpoint working"
+    echo "OK /coordinates"

    echo "Testing /metrics..."
-      curl -fsS --max-time 10 http://localhost:8000/metrics | head -30
-      echo "SUCCESS: /metrics endpoint working"
+    curl -fsS http://localhost:8000/metrics | head -20
+    echo "OK /metrics"

-      echo "SUCCESS: All tests passed"
    kill $PF_PID || true
-  after_script:
-    - |
-      echo "Final cluster state:"
-      kubectl get all -n ${NAMESPACE} || true
-      echo "Deleting kind cluster..."
-      kind delete cluster --name ci-cluster || true
+    echo "All tests passed!"
--- a/README.md
+++ b/README.md
@ -99,69 +99,76 @@ helm install open-meteo-service ./helm \
  --set grafana.persistence.enabled=true
 ```

-## CI Pipeline
+## GitLab CI Workflow

-This project includes a GitLab CI/CD pipeline that automatically builds, deploys, and tests the application without requiring Docker-in-Docker.
+This project uses a GitLab CI/CD pipeline that automates building, versioning, deployment, and testing on an external k3s cluster (homelab).

-### Pipeline Stages
+### Pipeline Overview

-1. **Build Stage (Kaniko)**
-   - Builds Docker image using Kaniko (secure, daemonless container builder)
-   - Pushes to GitLab Container Registry with commit-based tags
-   - Tags `latest` for main branch builds
-   - Uses layer caching for faster builds
+The pipeline consists of two stages: **build** and **deploy**.

-2. **Deploy Stage (kind + Helm)**
-   - Creates a local Kubernetes cluster using kind (Kubernetes in Docker)
-   - Installs kubectl, kind, and Helm
-   - Configures image pull secrets for GitLab registry access
-   - Deploys app using Helm with the newly built image
-   - Waits for deployment readiness with automatic rollback on failure
+#### 1. Build Stage
+- **Docker Image Build**: Builds the application Docker image using the project Dockerfile
+- **Tagging Strategy**: Tags each image with a unique identifier combining branch and commit SHA:
+  - Format: `<branch>-<shortsha>` (e.g., `main-a1b2c3d`, `feature-api-xyz1234`)
+  - Uses `CI_COMMIT_REF_SLUG` (sanitized branch name) and `CI_COMMIT_SHORT_SHA`
+- **Multi-Tag Push**: Pushes two tags to GitLab Container Registry:
+  - `$IMAGE_REPO:<branch>-<shortsha>` - Specific version tag
+  - `$IMAGE_REPO:latest` - Always points to the most recent build
+- **Registry Authentication**: Uses GitLab's built-in CI variables (`CI_REGISTRY`, `CI_REGISTRY_USER`, `CI_REGISTRY_PASSWORD`)

-3. **Test Stage (Health Checks)**
-   - Validates deployment with port-forwarding
-   - Tests `/healthz` endpoint for service health
-   - Tests `/coordinates` endpoint for data retrieval
-   - Tests `/help` endpoint for API documentation
-   - Provides detailed debugging info if tests fail
-   - Cleans up kind cluster after tests
+#### 2. Deploy Stage
+- **Values File Update**: 
+  - Uses `yq` to update `open-meteo-service/values.yaml`
+  - Sets `image.tag` to the newly built tag (`<branch>-<shortsha>`)
+  - Keeps `image.repository` constant in the values file
+  - Commits the change with message: `ci: bump image tag to <tag> [skip ci]`
+  - Pushes back to the repository using `CI_JOB_TOKEN` for authentication
+- **Helm Deployment**:
+  - Deploys to namespace: `sandbox-gitlab`
+  - Release name: `open-meteo-service-gitlab`
+  - Uses the updated values.yaml file from the workspace
+  - Waits for deployment to be ready (5-minute timeout)
+- **Automated Tests**:
+  - Port-forwards the service to localhost:8000
+  - `/healthz` - Checks service health
+  - `/coordinates` - Validates JSON response contains `"data"` field
+  - `/metrics` - Ensures metrics endpoint is accessible
+  - All tests must pass for the pipeline to succeed

-### Image Tagging Strategy
- **Commit builds**: `$CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA`
- **Main branch**: Also tagged as `latest`
+### Preventing Infinite Loops
+
+The pipeline includes safeguards to prevent infinite pipeline triggers:
+- The deploy job includes a rule: `if: '$CI_COMMIT_MESSAGE =~ /^ci: bump image tag/'` with `when: never`
+- The values bump commit includes `[skip ci]` to prevent triggering a new pipeline
+- The deploy job uses the already-updated values.yaml in the current workspace
+
+### Deployment Target
+
+- **Cluster**: External k3s homelab cluster (not ephemeral)
+- **Runner**: Self-hosted GitLab runner tagged with `homelab`
+- **Namespace**: `sandbox-gitlab` (separate from ArgoCD/Harbor setups)
+- **Ingress**: Configured for `open-meteo-gitlab.dvirlabs.com` (see values.yaml)

 ### Viewing Pipeline Status
-1. Go to your GitLab project
-2. Navigate to **CI/CD > Pipelines**
-3. Click on a pipeline run to see individual job logs
-4. Each stage shows real-time logs and can be expanded for details

-### Debugging Failed Pipelines
-If a pipeline fails:
- Check the **build** job logs for Docker build errors
- Check the **deploy** job logs for Kubernetes/Helm issues
-  - Pod status and events are automatically logged
-  - Container logs are shown if deployment fails
- Check the **test** job logs for endpoint test failures
-  - Response bodies are printed for debugging
+1. Navigate to **CI/CD > Pipelines** in your GitLab project
+2. Click on any pipeline run to see job details
+3. Expand jobs to view real-time logs
+4. The deploy job shows Helm output, rollout status, and test results

-### Running Locally
-To test the pipeline behavior locally:
+### Manual Deployment
+
+To deploy manually with a specific tag:
 ```bash
-# Build with Kaniko (requires Docker)
-docker run --rm -v $(pwd):/workspace gcr.io/kaniko-project/executor:latest \
-  --dockerfile /workspace/Dockerfile \
-  --context /workspace \
-  --no-push
+# Option 1: Deploy using updated values.yaml
+helm upgrade --install open-meteo-service-gitlab ./open-meteo-service \
+  -n sandbox-gitlab --create-namespace

-# Test with kind
-kind create cluster
-kubectl create secret docker-registry gitlab-registry \
-  --docker-server=your-registry \
-  --docker-username=your-user \
-  --docker-password=your-token
-helm install open-meteo-service ./open-meteo-service \
-  --set imagePullSecrets[0].name=gitlab-registry
+# Option 2: Override tag via --set
+helm upgrade --install open-meteo-service-gitlab ./open-meteo-service \
+  -n sandbox-gitlab --create-namespace \
+  --set image.tag=main-a1b2c3d
 ```

 ## API Documentation
--- a/open-meteo-service/values.yaml
+++ b/open-meteo-service/values.yaml
@ -1,12 +1,12 @@
 replicaCount: 1

 image:
-  repository: harbor.dvirlabs.com/library/open-meteo-service
-  tag: "1.0.3"
+  repository: registry.gitlab.com/dvirlabs/open-meteo-service
+  tag: "latest"
  pullPolicy: IfNotPresent

 imagePullSecrets:
-  - name: harbor-regcred
+  - name: gitlab-registry

 service:
  type: ClusterIP
@ -14,9 +14,9 @@ service:

 ingress:
  enabled: false
-  className: "traefik"
+  className: ""
  hosts:
-    - host: open-meteo.dvirlabs.com
+    - host: open-meteo-gitlab.dvirlabs.com
      paths:
        - path: /
          pathType: Prefix
@ -35,7 +35,7 @@ persistence:
  accessModes:
    - ReadWriteOnce
  size: 1Gi
-  storageClassName: "standard"
+  storageClassName: ""

 resources: {}

@ -54,9 +54,9 @@ prometheus:
    port: 9090
  ingress:
    enabled: false
-    className: "traefik"
+    className: ""
    hosts:
-      - host: open-meteo-prometheus.dvirlabs.com
+      - host: prometheus.local
        paths:
          - path: /
            pathType: Prefix
@ -66,7 +66,7 @@ prometheus:
    accessModes:
      - ReadWriteOnce
    size: 5Gi
-    storageClassName: "standard"
+    storageClassName: ""
  resources: {}
  extraArgs: []

@ -81,9 +81,9 @@ grafana:
  adminPassword: admin
  ingress:
    enabled: false
-    className: "traefik"
+    className: ""
    hosts:
-      - host: open-meteo-grafana.dvirlabs.com
+      - host: grafana.local
        paths:
          - path: /
            pathType: Prefix
@ -93,5 +93,5 @@ grafana:
    accessModes:
      - ReadWriteOnce
    size: 5Gi
-    storageClassName: "standard"
+    storageClassName: ""
  resources: {}
--- a/values.yaml
+++ b/values.yaml
@ -0,0 +1,97 @@
+replicaCount: 1
+
+image:
+  repository: registry.gitlab.com/dvirlabs/open-meteo-service
+  tag: "latest"
+  pullPolicy: IfNotPresent
+
+imagePullSecrets:
+- name: gitlab-registry
+
+service:
+  type: ClusterIP
+  port: 8000
+
+ingress:
+  enabled: true
+  className: "traefik"
+  hosts:
+    - host: open-meteo-gitlab.dvirlabs.com
+      paths:
+        - path: /
+          pathType: Prefix
+  tls: []
+
+podAnnotations:
+  prometheus.io/scrape: "true"
+  prometheus.io/path: /metrics
+  prometheus.io/port: "8000"
+
+env:
+  cacheFile: /data/coordinates_cache.json
+
+persistence:
+  enabled: false
+  accessModes:
+    - ReadWriteOnce
+  size: 1Gi
+  storageClassName: "nfs-client"
+
+resources: {}
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+prometheus:
+  enabled: true
+  replicaCount: 1
+  image: prom/prometheus:latest
+  service:
+    type: ClusterIP
+    port: 9090
+  ingress:
+    enabled: true
+    className: "traefik"
+    hosts:
+      - host: open-meteo-prometheus-gitlab.dvirlabs.com
+        paths:
+          - path: /
+            pathType: Prefix
+    tls: []
+  persistence:
+    enabled: true
+    accessModes:
+      - ReadWriteOnce
+    size: 5Gi
+    storageClassName: "nfs-client"
+  resources: {}
+  extraArgs: []
+
+grafana:
+  enabled: true
+  replicaCount: 1
+  image: grafana/grafana:latest
+  service:
+    type: ClusterIP
+    port: 3000
+  adminUser: admin
+  adminPassword: admin
+  ingress:
+    enabled: true
+    className: "traefik"
+    hosts:
+      - host: open-meteo-grafana-gitlab.dvirlabs.com
+        paths:
+          - path: /
+            pathType: Prefix
+    tls: []
+  persistence:
+    enabled: true
+    accessModes:
+      - ReadWriteOnce
+    size: 5Gi
+    storageClassName: "nfs-client"
+  resources: {}