From 9505f9fafce6484b9d1797bba63943b3d7512c5d Mon Sep 17 00:00:00 2001 From: dvirlabs Date: Tue, 7 Apr 2026 18:22:49 +0300 Subject: [PATCH] Redeploy harbor --- HARBOR-CERT-FIX.md | 292 ----------------------------------- argocd-apps/harbor.yaml | 46 +++--- fix-harbor-cert.sh | 66 -------- manifests/harbor/values.yaml | 8 +- 4 files changed, 28 insertions(+), 384 deletions(-) delete mode 100644 HARBOR-CERT-FIX.md delete mode 100644 fix-harbor-cert.sh diff --git a/HARBOR-CERT-FIX.md b/HARBOR-CERT-FIX.md deleted file mode 100644 index 4f2aca5..0000000 --- a/HARBOR-CERT-FIX.md +++ /dev/null @@ -1,292 +0,0 @@ -# Harbor + cert-manager Fix - Complete Solution - -## Root Cause Analysis - -### The Problem -Your Harbor deployment had a **certificate issuer conflict**: - -1. **Harbor chart with `certSource: auto`** (default): - - Harbor generates its own self-signed CA certificate - - Creates the `harbor-ingress` TLS secret with this self-signed cert - - Secret managed by Harbor Helm chart - -2. **cert-manager annotation added to ingress**: - - `cert-manager.io/cluster-issuer: letsencrypt` tells cert-manager to manage the cert - - cert-manager tries to manage the `harbor-ingress` secret - - **Conflict**: cert-manager detects the secret was created by a different issuer - - Error: `IncorrectIssuer - Secret was previously issued by "Issuer.cert-manager.io/"` - -3. **Nginx annotations on Traefik ingress**: - - Old/irrelevant annotations like `nginx.ingress.kubernetes.io/*` present - - Causes confusion and is not best practice - -### Result -- ACME order fails: `403 urn:ietf:params:acme:error:orderNotReady` -- Certificate stuck in `False` (not Ready) state -- Harbor accessible but with Harbor's self-signed cert, not Let's Encrypt - ---- - -## The Solution - -### Key Changes in `manifests/harbor/values.yaml` - -```yaml -expose: - type: ingress - tls: - enabled: true - # Changed from "auto" to "secret" - # This tells Harbor: "Don't generate your own cert, use an external secret" - certSource: secret - secret: - # Reference the secret that cert-manager will create - secretName: "harbor-ingress" - - ingress: - className: traefik - annotations: - # cert-manager will create the Certificate and Secret automatically - cert-manager.io/cluster-issuer: letsencrypt - # Traefik annotations for HTTPS routing - traefik.ingress.kubernetes.io/router.entrypoints: websecure - traefik.ingress.kubernetes.io/router.tls: "true" - hosts: - core: harbor.dvirlabs.com - notary: notary.dvirlabs.com -``` - -### Why This Works - -1. **`certSource: secret`**: - - Harbor chart will NOT auto-generate a certificate - - Harbor expects the `harbor-ingress` secret to exist (created externally) - - Harbor chart won't interfere with cert-manager's secret management - -2. **`cert-manager.io/cluster-issuer: letsencrypt` annotation**: - - cert-manager detects this annotation on the Ingress - - Automatically creates a `Certificate` resource - - Issues certificate via Let's Encrypt using ClusterIssuer - - Stores certificate in the `harbor-ingress` secret - -3. **Clean secret ownership**: - - cert-manager is the sole owner/manager of the `harbor-ingress` secret - - No "IncorrectIssuer" conflict - - Certificate renewals handled automatically by cert-manager - -4. **Stable resource names**: - - Ingress: `harbor-ingress` - - Secret: `harbor-ingress` - - Certificate: `harbor-ingress` (auto-created by cert-manager) - - No duplicate/workaround names like `harbor-ingress-v2` - ---- - -## Why No IncorrectIssuer or Duplicate Orders - -### IncorrectIssuer Prevention -- The old `harbor-ingress` secret (created by Harbor's auto CA) is **deleted** before redeployment -- cert-manager creates a fresh secret from scratch -- No conflict with previous issuers - -### Duplicate Order Prevention -- Only ONE source of truth: **cert-manager via ingress annotation** -- Harbor chart does NOT create certificates (certSource: secret) -- No separate Certificate manifest needed (ingress annotation is cleaner for GitOps) -- cert-manager intelligently reuses valid certificates - -### Clean GitOps Management -- Single values file controls everything -- ArgoCD manages Harbor deployment -- cert-manager automatically handles certificate lifecycle -- No manual kubectl hacks needed - ---- - -## Deployment Steps - -### 1. Review Changes -```bash -cd ~/OneDrive/Desktop/gitea/dev-tools -git diff manifests/harbor/values.yaml -``` - -### 2. Run the Fix Script -```bash -bash fix-harbor-cert.sh -``` - -This script will: -- Delete the old Harbor-generated secret -- Clean up failed cert-manager resources -- Commit and push changes to git -- Trigger ArgoCD sync -- Monitor certificate issuance - -### 3. Manual Steps (if you prefer) -```bash -# Delete old resources -kubectl delete secret harbor-ingress -n dev-tools -kubectl delete certificate harbor-ingress -n dev-tools --ignore-not-found -kubectl delete certificaterequest -n dev-tools -l cert-manager.io/certificate-name=harbor-ingress - -# Commit changes -git add manifests/harbor/values.yaml -git commit -m "fix: Configure Harbor to use cert-manager for TLS" -git push - -# Sync ArgoCD -kubectl patch app harbor -n argocd --type merge -p '{"operation":{"initiatedBy":{"username":"manual"},"sync":{"revision":"HEAD"}}}' - -# Monitor -kubectl get certificate harbor-ingress -n dev-tools -w -``` - ---- - -## Verification - -### Check Certificate Status -```bash -kubectl get certificate,secret,ingress -n dev-tools | grep harbor-ingress -``` - -Expected output: -``` -certificate.cert-manager.io/harbor-ingress True harbor-ingress 1m -secret/harbor-ingress kubernetes.io/tls 3 1m -ingress.networking.k8s.io/harbor-ingress traefik harbor.dvirlabs.com 192.168.10.240 80, 443 1m -``` - -### Check Certificate Details -```bash -kubectl describe certificate harbor-ingress -n dev-tools -``` - -Should show: -- `Status: True` -- `Message: Certificate is up to date and has not expired` -- `Issuer: letsencrypt` (ClusterIssuer) - -### Test Access -```bash -curl -I https://harbor.dvirlabs.com -``` - -Should return `HTTP/2 200` (not 502) - -### Verify Certificate in Browser -Visit `https://harbor.dvirlabs.com` - should show: -- Valid Let's Encrypt certificate -- Issued to: harbor.dvirlabs.com -- Issued by: Let's Encrypt Authority - ---- - -## Alternative: Explicit Certificate Resource - -If you prefer **declarative Certificate management** instead of ingress annotations, you can: - -### Create `manifests/harbor/certificate.yaml`: -```yaml -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: harbor-ingress - namespace: dev-tools -spec: - secretName: harbor-ingress - issuerRef: - name: letsencrypt - kind: ClusterIssuer - dnsNames: - - harbor.dvirlabs.com -``` - -### Remove ingress annotation: -In `manifests/harbor/values.yaml`, remove: -```yaml -cert-manager.io/cluster-issuer: letsencrypt # Remove this line -``` - -### Why I recommend ingress annotations instead: -1. **Simpler**: No separate Certificate file to manage -2. **DRY principle**: Hostname defined once in values.yaml -3. **Less resources**: One less manifest file -4. **Harbor chart native**: Uses Harbor's standard annotation mechanism - ---- - -## Production Checklist - -- [x] Harbor chart uses `certSource: secret` (not "auto") -- [x] Secret name is stable: `harbor-ingress` -- [x] cert-manager annotation present on ingress -- [x] ClusterIssuer `letsencrypt` exists and is ready -- [x] Old self-signed secret deleted before redeployment -- [x] Traefik-specific annotations only (no nginx annotations) -- [x] HTTPS entrypoint configured: `websecure` -- [x] externalURL uses HTTPS: `https://harbor.dvirlabs.com` -- [x] GitOps workflow preserved (ArgoCD manages Harbor) -- [x] No duplicate resource names (no -v2, -copy suffixes) - ---- - -## Troubleshooting - -### Certificate stuck in "Issuing" state -```bash -# Check certificate details -kubectl describe certificate harbor-ingress -n dev-tools - -# Check ACME order -kubectl get order -n dev-tools - -# Check ACME challenge -kubectl get challenge -n dev-tools -``` - -### DNS-01 validation fails -```bash -# Check Cloudflare credentials secret -kubectl get secret cloudflare-api-token-secret -n cert-manager - -# Check ClusterIssuer status -kubectl describe clusterissuer letsencrypt -``` - -### Harbor pods healthy but 502 error persists -```bash -# Check if secret exists and has valid cert -kubectl get secret harbor-ingress -n dev-tools -o yaml - -# Restart Traefik to reload ingress config -kubectl rollout restart deployment traefik -n kube-system - -# Check Traefik logs -kubectl logs -n kube-system -l app.kubernetes.io/name=traefik --tail=50 -``` - ---- - -## Summary - -**Before:** -- Harbor generates self-signed cert → `harbor-ingress` secret -- cert-manager tries to manage same secret → **IncorrectIssuer conflict** -- ACME orders fail repeatedly - -**After:** -- cert-manager manages certificate lifecycle cleanly -- Harbor references the cert-manager-created secret -- Single source of truth for TLS management -- Clean GitOps workflow with stable resource names - -**Files Changed:** -- `manifests/harbor/values.yaml` - TLS configuration fixed - -**No Changes Needed:** -- Harbor chart templates (work as designed) -- Ingress class (traefik) -- ClusterIssuer (letsencrypt) -- Secret name (harbor-ingress) diff --git a/argocd-apps/harbor.yaml b/argocd-apps/harbor.yaml index 1394717..d9c7553 100644 --- a/argocd-apps/harbor.yaml +++ b/argocd-apps/harbor.yaml @@ -1,23 +1,23 @@ -# apiVersion: argoproj.io/v1alpha1 -# kind: Application -# metadata: -# name: harbor -# namespace: argocd -# labels: -# env: dev-tools -# spec: -# project: dev-tools -# source: -# repoURL: ssh://git@gitea-ssh.dev-tools.svc.cluster.local:2222/dvirlabs/dev-tools.git -# targetRevision: HEAD -# path: charts/harbor -# helm: -# valueFiles: -# - ../../manifests/harbor/values.yaml -# destination: -# server: https://kubernetes.default.svc -# namespace: dev-tools -# syncPolicy: -# automated: -# prune: true -# selfHeal: true +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: harbor + namespace: argocd + labels: + env: dev-tools +spec: + project: dev-tools + source: + repoURL: ssh://git@gitea-ssh.dev-tools.svc.cluster.local:2222/dvirlabs/dev-tools.git + targetRevision: HEAD + path: charts/harbor + helm: + valueFiles: + - ../../manifests/harbor/values.yaml + destination: + server: https://kubernetes.default.svc + namespace: dev-tools + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/fix-harbor-cert.sh b/fix-harbor-cert.sh deleted file mode 100644 index b17371a..0000000 --- a/fix-harbor-cert.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -# Harbor + cert-manager cleanup and fix script -# This removes the Harbor-generated certificate and lets cert-manager create a clean one - -set -e - -echo "=== Harbor cert-manager Fix ===" -echo "" - -# Step 1: Delete the old Harbor-generated TLS secret -echo "1. Deleting existing harbor-ingress secret (Harbor's self-signed cert)..." -kubectl delete secret harbor-ingress -n dev-tools --ignore-not-found=true -echo " ✓ Secret deleted" -echo "" - -# Step 2: Delete any failed cert-manager Certificate resources -echo "2. Cleaning up failed cert-manager resources..." -kubectl delete certificate harbor-ingress -n dev-tools --ignore-not-found=true -kubectl delete certificaterequest -n dev-tools -l cert-manager.io/certificate-name=harbor-ingress --ignore-not-found=true -echo " ✓ Old certificates cleaned" -echo "" - -# Step 3: Commit and push the fixed values.yaml -echo "3. Committing fixed Harbor values to git..." -cd "$(dirname "$0")" -git add manifests/harbor/values.yaml -git commit -m "fix: Configure Harbor to use cert-manager for TLS (secretName: harbor-ingress)" -git push -echo " ✓ Changes pushed to git" -echo "" - -# Step 4: Wait for ArgoCD to sync (or trigger manually) -echo "4. Waiting for ArgoCD to sync Harbor application..." -sleep 5 -kubectl patch app harbor -n argocd --type merge -p '{"operation":{"initiatedBy":{"username":"manual"},"sync":{"revision":"HEAD"}}}' -echo " ✓ ArgoCD sync triggered" -echo "" - -# Step 5: Monitor the certificate issuance -echo "5. Monitoring certificate creation..." -echo " (This may take 1-2 minutes for DNS-01 validation)" -echo "" - -for i in {1..24}; do - STATUS=$(kubectl get certificate harbor-ingress -n dev-tools -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "NotFound") - - if [ "$STATUS" == "True" ]; then - echo " ✓ Certificate issued successfully!" - break - elif [ "$STATUS" == "NotFound" ]; then - echo " ⏳ Waiting for certificate to be created... ($i/24)" - else - echo " ⏳ Certificate status: $STATUS ($i/24)" - fi - - sleep 5 -done - -echo "" -echo "=== Verification ===" -kubectl get certificate harbor-ingress -n dev-tools -echo "" -kubectl get secret harbor-ingress -n dev-tools -echo "" -echo "=== Complete! ===" -echo "Test Harbor at: https://harbor.dvirlabs.com" diff --git a/manifests/harbor/values.yaml b/manifests/harbor/values.yaml index 10a7aac..fd51218 100644 --- a/manifests/harbor/values.yaml +++ b/manifests/harbor/values.yaml @@ -55,13 +55,13 @@ trivy: enabled: true metrics: - enabled: true + enabled: false exporter: - enabled: true + enabled: false cache: - enabled: true + enabled: false nodeSelector: workload: general @@ -76,4 +76,6 @@ affinity: values: - general - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist + - key: gpu operator: DoesNotExist \ No newline at end of file