#!/bin/bash # ============================================================================= # update-gitops-status.sh # # Purpose: # Runs drift-check playbook and generates a JSON status snapshot for # gitops-status-server. This replaces Pushgateway metric updates with # richer JSON status suitable for Grafana visualization via Infinity DS. # # Flow: # 1. Execute ansible/playbooks/drift-check.yml (check mode, read-only) # 2. Capture exit code to determine sync status # 3. Parse playbook output to extract changed files # 4. Build structured JSON with metadata # 5. POST JSON to gitops-status-server API # # Usage: # ./update-gitops-status.sh # # Environment Variables: # GITOPS_STATUS_SERVER_URL - URL of gitops-status-server API # (default: http://gitops-status-server.observability-stack.svc.cluster.local:80) # REPO_NAME - Repository name (default: rsyslog) # SERVER_NAME - Server name (default: rsyslog-lab) # # Generated JSON Structure: # { # "repo": "rsyslog", # "server": "rsyslog-lab", # "sync_status": "SYNCED" or "OUT_OF_SYNC", # "drift_count": , # "files": [{"name": "rsyslog.conf"}, {"name": "rsyslog.d/30-lab.conf"}], # "last_check": "2026-04-21T10:30:00Z" # } # # Exit Codes: # 0 - Success (JSON posted to gitops-status-server regardless of sync status) # 1 - Failure (playbook error, network error, JSON post failure, etc.) # # ============================================================================= set -euo pipefail # ───────────────────────────────────────────────────────────────────────────── # Configuration # ───────────────────────────────────────────────────────────────────────────── GITOPS_STATUS_SERVER_URL="${GITOPS_STATUS_SERVER_URL:-http://gitops-status-server.observability-stack.svc.cluster.local:5000}" REPO_NAME="${REPO_NAME:-rsyslog}" SERVER_NAME="${SERVER_NAME:-rsyslog-lab}" INVENTORY_FILE="ansible/inventory/hosts.yml" PLAYBOOK="ansible/playbooks/drift-check.yml" MODE="${MODE:-drift-check}" # drift-check or post-deploy PLAYBOOK_LOG="" # Initialize to avoid unbound variable error echo "═══════════════════════════════════════════════════════════════════════════════" echo " GitOps Status Update" echo " Repository: $REPO_NAME | Server: $SERVER_NAME" echo " Target: $GITOPS_STATUS_SERVER_URL" echo " Mode: $MODE" echo "═══════════════════════════════════════════════════════════════════════════════" echo "" CHANGED_FILES=() DRIFT_COUNT=0 SYNC_STATUS="SYNCED" # ───────────────────────────────────────────────────────────────────────────────── # MODE 1: post-deploy - Report what files were deployed from Git # ───────────────────────────────────────────────────────────────────────────────── if [ "$MODE" = "post-deploy" ]; then echo "Step 1/4: Analyzing Git changes (what was just deployed)..." # Check what files changed in the last commit in files/ directory if command -v git >/dev/null 2>&1 && [ -d .git ]; then # Try multiple methods to detect changed files (Woodpecker might do shallow clone) # Method 1: Use git diff to see actual file changes (most accurate) CHANGED_FILE_PATHS=$(git diff --name-only HEAD~1 HEAD -- files/ 2>/dev/null || echo "") # Method 2: If method 1 fails (no parent commit), use git show if [ -z "$CHANGED_FILE_PATHS" ]; then CHANGED_FILE_PATHS=$(git show --name-only --pretty="" HEAD -- files/ 2>/dev/null || echo "") fi # Method 3: If still nothing, check Woodpecker CI environment variables if [ -z "$CHANGED_FILE_PATHS" ] && [ -n "${CI_COMMIT_CHANGED_FILES:-}" ]; then # Filter CI_COMMIT_CHANGED_FILES to only files/ directory CHANGED_FILE_PATHS=$(echo "$CI_COMMIT_CHANGED_FILES" | tr ' ' '\n' | grep '^files/' || echo "") fi echo " DEBUG: Git detection method results: '$CHANGED_FILE_PATHS'" if [ -n "$CHANGED_FILE_PATHS" ]; then echo " Files changed in last commit:" while IFS= read -r filepath; do if [ -n "$filepath" ]; then # Strip "files/" prefix to get the config file name filename="${filepath#files/}" # Skip if it's just "files/" (directory change) if [ "$filename" != "" ] && [ "$filename" != "files/" ]; then CHANGED_FILES+=("$filename") echo " - $filename" fi fi done <<< "$CHANGED_FILE_PATHS" DRIFT_COUNT=${#CHANGED_FILES[@]} else echo " No files changed in files/ directory" fi else echo " Git not available, cannot determine deployed files" fi # Always SYNCED after successful deploy SYNC_STATUS="SYNCED" echo " ✓ Status: SYNCED - files were deployed successfully" echo " Total deployed files: $DRIFT_COUNT" echo "" # ───────────────────────────────────────────────────────────────────────────────── # MODE 2: drift-check - Check for manual changes on server (drift detection) # ───────────────────────────────────────────────────────────────────────────────── else echo "Step 1/4: Running drift-check playbook..." # Capture playbook output to a temp file for parsing PLAYBOOK_LOG=$(mktemp) KEEP_LOG="${KEEP_PLAYBOOK_LOG:-false}" if [ "$KEEP_LOG" = "true" ]; then PLAYBOOK_LOG="./drift-check-output.log" echo " Playbook output will be saved to: $PLAYBOOK_LOG" fi # Set up cleanup trap (will be updated later with RESPONSE_BODY) trap "rm -f $PLAYBOOK_LOG" EXIT # Run playbook (no -v flag to avoid file descriptor exhaustion in containers) # Exit code: 0 = synced, non-zero = drift detected (expected) # Limit forks to 1 to reduce file descriptor usage set +e ANSIBLE_FORCE_COLOR=false \ ANSIBLE_FORKS=1 \ ansible-playbook \ -i "$INVENTORY_FILE" \ "$PLAYBOOK" \ > "$PLAYBOOK_LOG" 2>&1 DRIFT_RC=$? set -e # Show playbook output for debugging (compact) echo "Playbook output (last 25 lines):" cat "$PLAYBOOK_LOG" | tail -25 echo "" echo "DEBUG: Full playbook output saved to: $PLAYBOOK_LOG" echo "" # ───────────────────────────────────────────────────────────────────────────────── # Step 2: Determine sync status and collect changed files # ───────────────────────────────────────────────────────────────────────────────── echo "Step 2/4: Analyzing drift detection results..." # Exit code 0 = synced (all tasks succeeded) # Exit code non-zero = drift detected (fail task was reached) if [ "$DRIFT_RC" -eq 0 ]; then SYNC_STATUS="SYNCED" echo " ✓ Status: SYNCED - server configuration matches Git" else SYNC_STATUS="OUT_OF_SYNC" echo " ✗ Status: OUT OF SYNC - configuration drift detected" fi # Extract structured drifted files from playbook output # The drift-check.yml playbook outputs: DRIFTED_FILES=file1,file2,file3 echo " DEBUG: Searching for DRIFTED_FILES in playbook output..." if grep -q "DRIFTED_FILES=" "$PLAYBOOK_LOG"; then echo " DEBUG: Found DRIFTED_FILES pattern" DRIFTED_FILES_LINE=$(grep "DRIFTED_FILES=" "$PLAYBOOK_LOG" | tail -1) echo " DEBUG: Raw line: $DRIFTED_FILES_LINE" # Extract value after DRIFTED_FILES= (handles both YAML and default callback formats) # Format: "msg: DRIFTED_FILES=file1,file2" or "DRIFTED_FILES=file1,file2" DRIFTED_FILES_STR=$(echo "$DRIFTED_FILES_LINE" | sed 's/.*DRIFTED_FILES=//' | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//' | tr -d '"') echo " DEBUG: Extracted value: '$DRIFTED_FILES_STR'" # Check if the value is an empty list ([] or empty string) if [ -n "$DRIFTED_FILES_STR" ] && [ "$DRIFTED_FILES_STR" != "[]" ] && [ "$DRIFTED_FILES_STR" != "" ]; then # Parse comma-separated list into array IFS=',' read -ra CHANGED_FILES <<<"$DRIFTED_FILES_STR" echo " DEBUG: Parsed ${#CHANGED_FILES[@]} files" # Clean up whitespace and normalize paths for i in "${!CHANGED_FILES[@]}"; do CHANGED_FILES[$i]=$(echo "${CHANGED_FILES[$i]}" | xargs) # Convert full paths to relative paths for cleaner output if [[ "${CHANGED_FILES[$i]}" == /etc/rsyslog.conf ]]; then CHANGED_FILES[$i]="rsyslog.conf" elif [[ "${CHANGED_FILES[$i]}" == /etc/rsyslog.d/* ]]; then CHANGED_FILES[$i]=$(echo "${CHANGED_FILES[$i]}" | sed 's|^/etc/||') fi if [ "$SYNC_STATUS" = "OUT_OF_SYNC" ]; then echo " - Drift detected in: ${CHANGED_FILES[$i]}" fi done DRIFT_COUNT=${#CHANGED_FILES[@]} else echo " DEBUG: DRIFTED_FILES is empty or []" fi else echo " DEBUG: DRIFTED_FILES not found in playbook output" fi # Additional validation: If OUT_OF_SYNC but no files found, show warning if [ "$SYNC_STATUS" = "OUT_OF_SYNC" ] && [ "$DRIFT_COUNT" -eq 0 ]; then echo " ⚠️ WARNING: Status is OUT_OF_SYNC but no drifted files were extracted" echo " ⚠️ This might indicate a parsing issue. Check the playbook output above." fi echo " Total drift count: $DRIFT_COUNT" echo "" fi # ───────────────────────────────────────────────────────────────────────────────── # Step 3: Build JSON payload # ───────────────────────────────────────────────────────────────────────────────── echo "Step 3/4: Building JSON payload..." # Convert files array to JSON FILES_JSON="[]" if [ ${#CHANGED_FILES[@]} -gt 0 ]; then FILES_JSON="[" for i in "${!CHANGED_FILES[@]}"; do if [ "$i" -gt 0 ]; then FILES_JSON+="," fi # Escape special characters in filenames for JSON escaped_name="${CHANGED_FILES[$i]//\\/\\\\}" escaped_name="${escaped_name//\"/\\\"}" FILES_JSON+="{\"name\":\"$escaped_name\"}" done FILES_JSON+="]" fi # Generate ISO 8601 timestamp TIMESTAMP=$(date -u +'%Y-%m-%dT%H:%M:%SZ') # Build complete JSON status STATUS_JSON=$(cat </dev/null || echo "$STATUS_JSON" echo "" # ───────────────────────────────────────────────────────────────────────────────── # Step 4: Send JSON to gitops-status-server # ───────────────────────────────────────────────────────────────────────────────── echo "Step 4/4: Sending status to gitops-status-server..." echo " URL: $GITOPS_STATUS_SERVER_URL/api/status" echo " Method: POST" echo " Request Payload:" echo "$STATUS_JSON" | jq '.' 2>/dev/null | sed 's/^/ /' || echo "$STATUS_JSON" | sed 's/^/ /' echo "" echo "Step 4/4: Sending status to gitops-status-server..." echo " URL: $GITOPS_STATUS_SERVER_URL/api/status" echo " Method: POST" echo " Payload size: $(echo "$STATUS_JSON" | wc -c) bytes" echo "" echo " ========== JSON PAYLOAD BEING SENT ==========" echo "$STATUS_JSON" | jq '.' 2>/dev/null || echo "$STATUS_JSON" echo " ==========================================" echo "" # Test connectivity first echo " Testing connectivity to gitops-status-server..." if ! curl -s -m 5 "$GITOPS_STATUS_SERVER_URL/health" > /dev/null 2>&1; then echo " ✗ WARNING: Cannot reach $GITOPS_STATUS_SERVER_URL/health" echo " Attempting DNS resolution..." nslookup gitops-status-server.observability-stack.svc.cluster.local || true echo "" else echo " ✓ Server is reachable" fi echo "" # Create temporary files for response RESPONSE_BODY=$(mktemp) trap "rm -f $RESPONSE_BODY $PLAYBOOK_LOG" EXIT echo " Sending POST request with curl..." echo " Command: curl -X POST -H 'Content-Type: application/json' -d '' $GITOPS_STATUS_SERVER_URL/api/status" echo "" # POST the JSON to the gitops-status-server API with full error reporting # Capture both response code and body for debugging set +e HTTP_RESPONSE=$(curl -s -w "\n%{http_code}" \ -X POST \ -H "Content-Type: application/json" \ -d "$STATUS_JSON" \ "$GITOPS_STATUS_SERVER_URL/api/status" \ 2>&1) CURL_EXIT=$? set -e if [ $CURL_EXIT -ne 0 ]; then echo " ✗ CURL FAILED with exit code $CURL_EXIT" echo " Error output: $HTTP_RESPONSE" exit 1 fi # Split response: body is everything except last line, code is last line HTTP_CODE=$(echo "$HTTP_RESPONSE" | tail -1) RESPONSE_CONTENT=$(echo "$HTTP_RESPONSE" | head -n -1) echo "$RESPONSE_CONTENT" > "$RESPONSE_BODY" # Validate HTTP code is numeric if ! [[ "$HTTP_CODE" =~ ^[0-9]+$ ]]; then echo " ✗ ERROR: Invalid HTTP response code: $HTTP_CODE" echo " Full response: $HTTP_RESPONSE" exit 1 fi echo " Response: HTTP $HTTP_CODE" # Show response body for debugging (especially on error) if [ -s "$RESPONSE_BODY" ]; then echo " Response Body:" sed 's/^/ /' "$RESPONSE_BODY" fi echo "" if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then echo "═══════════════════════════════════════════════════════════════════════════════" echo " ✓ Status update successful (HTTP $HTTP_CODE)" echo " JSON has been sent to gitops-status-server" echo "" # Verify the JSON was actually received and stored echo " Verifying JSON storage on gitops-status-server..." sleep 1 # Brief delay to ensure server processed the POST VERIFY_JSON=$(curl -s "$GITOPS_STATUS_SERVER_URL/api/status" 2>&1 || true) if echo "$VERIFY_JSON" | grep -q "\"repo\".*\"$REPO_NAME\""; then echo " ✓ Verified: Latest JSON stored correctly on server" echo "" echo " Grafana Infinity datasource will now read the updated JSON from:" echo " $GITOPS_STATUS_SERVER_URL/api/status" else echo " ⚠ Warning: Could not verify JSON storage" echo " Response from /status.json:" echo " $VERIFY_JSON" | sed 's/^/ /' fi echo "═══════════════════════════════════════════════════════════════════════════════" exit 0 else echo "═══════════════════════════════════════════════════════════════════════════════" echo " ✗ ERROR: Status update failed with HTTP $HTTP_CODE" echo " Debugging Information:" echo " - Server URL: $GITOPS_STATUS_SERVER_URL" echo " - Endpoint: /api/status" echo " - Check gitops-status-server connectivity (ping/nslookup)" echo " - Verify service port and internal port mapping" echo " - Ensure /api/status endpoint accepts POST requests" echo "═══════════════════════════════════════════════════════════════════════════════" exit 1 fi