From eafef6d665167f7d7be79b7c23a8e4f498ddd4dd Mon Sep 17 00:00:00 2001 From: dvirlabs Date: Sun, 22 Jun 2025 22:27:41 +0300 Subject: [PATCH] Test order values by sections --- .woodpecker.yml | 2 + automation/cloudflared-sync.sh | 71 +++++++++++++++++++--------------- 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 769d005..c87578b 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,5 +1,7 @@ steps: sync-cloudflare: + when: + branch: [master] name: Scan Apps and Update Cloudflared image: alpine commands: diff --git a/automation/cloudflared-sync.sh b/automation/cloudflared-sync.sh index c1254c5..f47f0e9 100644 --- a/automation/cloudflared-sync.sh +++ b/automation/cloudflared-sync.sh @@ -1,9 +1,6 @@ #!/bin/bash set -e -# ๐Ÿ“ฆ Ensure apk installs: yq, jq, git, bash, curl -# Required for pipeline: alpine image + apk add --no-cache git bash curl yq jq - apk add --no-cache git bash curl yq jq echo "๐Ÿ” Scanning for apps with cname.yaml..." @@ -12,76 +9,92 @@ mkdir -p generated-values rm -rf .tmp-repos mkdir -p .tmp-repos -# === REPO CONFIG === SANDBOX_REPO_URL="https://git.dvirlabs.com/dvirlabs/sandbox.git" INFRA_REPO_URL="https://${GIT_TOKEN}@git.dvirlabs.com/dvirlabs/infra.git" SANDBOX_CLONE=".tmp-repos/sandbox" INFRA_CLONE=".tmp-repos/infra" -GENERATED_FILE="$(pwd)/generated-values/cloudflared-values.yaml" +GENERATED_FILE="generated-values/cloudflared-values.yaml" ORIGINAL_FILE="$INFRA_CLONE/manifests/cloudflared/values.yaml" MERGED_FILE="$INFRA_CLONE/manifests/cloudflared/values.yaml" -# === STEP 1: Clone Repos === echo "๐Ÿ“ฆ Cloning sandbox-apps..." git clone --depth=1 "$SANDBOX_REPO_URL" "$SANDBOX_CLONE" - echo "๐Ÿ“ฆ Cloning infra..." git clone --depth=1 "$INFRA_REPO_URL" "$INFRA_CLONE" -ls -l .tmp-repos/ - -# === STEP 2: Extract Generated CNAMEs from sandbox === echo "โš™๏ธ Generating sandbox ingress list..." cat < "$GENERATED_FILE" ingress: [] EOF find "$SANDBOX_CLONE/manifests" -name cname.yaml | while read -r cname_file; do - app_name=$(basename "$(dirname "$cname_file")") - enabled=$(yq '.enabled' "$cname_file") + app_dir=$(dirname "$cname_file") + app_name=$(basename "$app_dir") + namespace=$(basename "$(dirname "$app_dir")") + enabled=$(yq '.enabled' "$cname_file") if [[ "$enabled" == "true" ]]; then hostname=$(yq '.hostname' "$cname_file") - service="http://${app_name}.sandbox.svc.cluster.local:80" + service="http://${app_name}.${namespace}.svc.cluster.local:80" echo "โœ… Found $hostname โ†’ $service" - - yq eval ".ingress += [{\"hostname\": \"$hostname\", \"service\": \"$service\"}]" -i "$GENERATED_FILE" + yq eval ".ingress += [{\"hostname\": \"$hostname\", \"service\": \"$service\", \"namespace\": \"$namespace\"}]" -i "$GENERATED_FILE" fi done echo "๐Ÿ“„ Generated Ingress:" cat "$GENERATED_FILE" -# === STEP 3: Merge with existing cloudflared values === echo "๐Ÿ” Merging new entries into: $ORIGINAL_FILE" TEMP_FILE=$(mktemp) cp "$ORIGINAL_FILE" "$TEMP_FILE" -# Loop through new entries and append only if hostname not exists +# Strip namespace key from existing values if present +yq eval '.cloudflare.ingress' "$TEMP_FILE" | yq -o=json | jq -c '.[]' > /tmp/existing.json + yq eval '.ingress' "$GENERATED_FILE" | yq -o=json | jq -c '.[]' | while read -r new_entry; do hostname=$(echo "$new_entry" | jq -r '.hostname') service=$(echo "$new_entry" | jq -r '.service') + namespace=$(echo "$new_entry" | jq -r '.namespace') - exists=$(yq e ".cloudflare.ingress[] | select(.hostname == \"$hostname\")" "$TEMP_FILE") + exists=$(jq --arg hostname "$hostname" '.[] | select(.hostname == $hostname)' /tmp/existing.json) if [ -z "$exists" ]; then echo "โž• Adding $hostname โ†’ $service" - yq eval ".cloudflare.ingress += [{\"hostname\": \"$hostname\", \"service\": \"$service\"}]" -i "$TEMP_FILE" + yq eval ".cloudflare.ingress += [{\"hostname\": \"$hostname\", \"service\": \"$service\", \"namespace\": \"$namespace\"}]" -i "$TEMP_FILE" else echo "โš ๏ธ $hostname already exists, skipping" fi done -# === STEP 4: Overwrite only ingress list and preserve all other fields === -echo "๐Ÿ“ Writing final merged values.yaml" -cp "$TEMP_FILE" "$MERGED_FILE" +# === ๐Ÿงน Group and sort with comments === +echo "๐Ÿงผ Grouping ingress entries by namespace..." +GROUPED=$(mktemp) + +echo "cloudflare:" > "$GROUPED" +yq eval '.cloudflare | del(.ingress)' "$TEMP_FILE" | tail -n +2 >> "$GROUPED" +echo " ingress:" >> "$GROUPED" + +# Extract ingress entries with namespace +yq eval '.cloudflare.ingress' "$TEMP_FILE" | yq -o=json | jq -s 'group_by(.namespace)[]' | while read -r group; do + namespace=$(echo "$group" | jq -r '.[0].namespace') + echo " # ############ $namespace ############" >> "$GROUPED" + echo "$group" | jq -c '.[]' | while read -r item; do + hostname=$(echo "$item" | jq -r '.hostname') + service=$(echo "$item" | jq -r '.service') + echo " - hostname: $hostname" >> "$GROUPED" + echo " service: $service" >> "$GROUPED" + done +done + +# Copy to merged location +cp "$GROUPED" "$MERGED_FILE" echo "โœ… Final merged values.yaml:" cat "$MERGED_FILE" -# === STEP 5: Optional push to Git === +# === Commit if changed === cd "$INFRA_CLONE" git config user.name "woodpecker-bot" git config user.email "ci@dvirlabs.com" @@ -89,30 +102,24 @@ git remote set-url origin "https://${GIT_TOKEN}@git.dvirlabs.com/dvirlabs/infra. if ! git diff --quiet manifests/cloudflared/values.yaml; then git add manifests/cloudflared/values.yaml - git commit -m "chore(cloudflared): auto-merge CNAME entries from sandbox" + git commit -m "chore(cloudflared): grouped ingress by namespace" git push origin HEAD echo "โœ… Changes pushed successfully." else echo "โ„น๏ธ No changes to commit." fi -# === STEP 5: Create CNAME records in Cloudflare === -ls -l -pwd -ls -l "$GENERATED_FILE" +# === Create Cloudflare CNAMEs === echo "๐ŸŒ Creating CNAME records in Cloudflare..." CLOUDFLARE_API="https://api.cloudflare.com/client/v4" TARGET="b50bbf48-0a2f-47ce-b73e-336b6718318b.cfargotunnel.com" -# Load required secrets : "${CLOUDFLARE_API_TOKEN:?CLOUDFLARE_API_TOKEN not set}" : "${CLOUDFLARE_ZONE_ID:?CLOUDFLARE_ZONE_ID not set}" -# Check and create each CNAME -yq eval '.ingress' "$GENERATED_FILE" | yq eval -o=json '.' - | jq -c '.[]' | while read -r record; do +yq eval '.ingress' "$GENERATED_FILE" | yq eval -o=json '.' | jq -c '.[]' | while read -r record; do name=$(echo "$record" | jq -r '.hostname' | sed 's/\.dvirlabs\.com//') - # Check if already exists exists=$(curl -s -X GET "$CLOUDFLARE_API/zones/$CLOUDFLARE_ZONE_ID/dns_records?type=CNAME&name=$name.dvirlabs.com" \ -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" -H "Content-Type: application/json" | jq '.result | length')