#!/bin/bash # ============================================================================= # update-gitops-status.sh # # Purpose: # Runs drift-check playbook and generates a JSON status snapshot for # gitops-status-server. This replaces Pushgateway metric updates with # richer JSON status suitable for Grafana visualization via Infinity DS. # # Flow: # 1. Execute ansible/playbooks/drift-check.yml (check mode, read-only) # 2. Capture exit code to determine sync status # 3. Parse playbook output to extract changed files # 4. Build structured JSON with metadata # 5. POST JSON to gitops-status-server API # # Usage: # ./update-gitops-status.sh # # Environment Variables: # GITOPS_STATUS_SERVER_URL - URL of gitops-status-server API # (default: http://gitops-status-server.observability-stack.svc.cluster.local:80) # REPO_NAME - Repository name (default: rsyslog) # SERVER_NAME - Server name (default: rsyslog-lab) # # Generated JSON Structure: # { # "repo": "rsyslog", # "server": "rsyslog-lab", # "sync_status": "SYNCED" or "OUT_OF_SYNC", # "drift_count": , # "files": [{"name": "rsyslog.conf"}, {"name": "rsyslog.d/30-lab.conf"}], # "last_check": "2026-04-21T10:30:00Z" # } # # Exit Codes: # 0 - Success (JSON posted to gitops-status-server regardless of sync status) # 1 - Failure (playbook error, network error, JSON post failure, etc.) # # ============================================================================= set -euo pipefail # ───────────────────────────────────────────────────────────────────────────── # Configuration # ───────────────────────────────────────────────────────────────────────────── GITOPS_STATUS_SERVER_URL="${GITOPS_STATUS_SERVER_URL:-http://gitops-status-server.observability-stack.svc.cluster.local:80}" REPO_NAME="${REPO_NAME:-rsyslog}" SERVER_NAME="${SERVER_NAME:-rsyslog-lab}" INVENTORY_FILE="ansible/inventory/hosts.yml" PLAYBOOK="ansible/playbooks/drift-check.yml" echo "═══════════════════════════════════════════════════════════════════════════════" echo " GitOps Status Update" echo " Repository: $REPO_NAME | Server: $SERVER_NAME" echo " Target: $GITOPS_STATUS_SERVER_URL" echo "═══════════════════════════════════════════════════════════════════════════════" echo "" # ───────────────────────────────────────────────────────────────────────────────── # Step 1: Run drift-check playbook # ───────────────────────────────────────────────────────────────────────────────── echo "Step 1/4: Running drift-check playbook..." # Capture playbook output to a temp file for parsing PLAYBOOK_LOG=$(mktemp) KEEP_LOG="${KEEP_PLAYBOOK_LOG:-false}" if [ "$KEEP_LOG" = "true" ]; then PLAYBOOK_LOG="./drift-check-output.log" echo " Playbook output will be saved to: $PLAYBOOK_LOG" else trap "rm -f $PLAYBOOK_LOG" EXIT fi # Run playbook (no -v flag to avoid file descriptor exhaustion in containers) # Exit code: 0 = synced, non-zero = drift detected (expected) # Disable callbacks and reduce file watchers to prevent inotify exhaustion set +e ANSIBLE_CALLBACKS_ENABLED="" \ ANSIBLE_STDOUT_CALLBACK=minimal \ ansible-playbook \ -i "$INVENTORY_FILE" \ "$PLAYBOOK" \ > "$PLAYBOOK_LOG" 2>&1 DRIFT_RC=$? set -e # Show playbook output for debugging (compact) echo "Playbook output (last 25 lines):" cat "$PLAYBOOK_LOG" | tail -25 echo "" echo "DEBUG: Full playbook output saved to: $PLAYBOOK_LOG" echo "" # ───────────────────────────────────────────────────────────────────────────────── # Step 2: Determine sync status and collect changed files # ───────────────────────────────────────────────────────────────────────────────── echo "Step 2/4: Analyzing drift detection results..." CHANGED_FILES=() DRIFT_COUNT=0 # Exit code 0 = synced (all tasks succeeded) # Exit code non-zero = drift detected (fail task was reached) if [ "$DRIFT_RC" -eq 0 ]; then SYNC_STATUS="SYNCED" echo " ✓ Status: SYNCED - server configuration matches Git" else SYNC_STATUS="OUT_OF_SYNC" echo " ✗ Status: OUT OF SYNC - configuration drift detected" fi # Extract structured drifted files from playbook output # The drift-check.yml playbook outputs: DRIFTED_FILES=file1,file2,file3 # Search for the pattern in the output echo " DEBUG: Searching for DRIFTED_FILES in playbook output..." if grep -q "DRIFTED_FILES=" "$PLAYBOOK_LOG"; then echo " DEBUG: Found DRIFTED_FILES pattern" DRIFTED_FILES_STR=$(grep "DRIFTED_FILES=" "$PLAYBOOK_LOG" | tail -1) echo " DEBUG: Raw line: $DRIFTED_FILES_STR" # Remove ANSI color codes and extract the value # Handle both formats: "DRIFTED_FILES=..." and "msg": "DRIFTED_FILES=..." DRIFTED_FILES_STR=$(echo "$DRIFTED_FILES_STR" | sed 's/.*DRIFTED_FILES=//' | sed 's/\x1b\[[0-9;]*m//g' | sed 's/".*$//' | xargs) echo " DEBUG: Extracted value: '$DRIFTED_FILES_STR'" # Check if the value is an empty list ([] or empty string) if [ -n "$DRIFTED_FILES_STR" ] && [ "$DRIFTED_FILES_STR" != "[]" ] && [ "$DRIFTED_FILES_STR" != "" ]; then # Parse comma-separated list into array IFS=',' read -ra CHANGED_FILES <<<"$DRIFTED_FILES_STR" echo " DEBUG: Parsed ${#CHANGED_FILES[@]} files" # Clean up whitespace and normalize paths for i in "${!CHANGED_FILES[@]}"; do CHANGED_FILES[$i]=$(echo "${CHANGED_FILES[$i]}" | xargs) # Convert full paths to relative paths for cleaner output if [[ "${CHANGED_FILES[$i]}" == /etc/rsyslog.conf ]]; then CHANGED_FILES[$i]="rsyslog.conf" elif [[ "${CHANGED_FILES[$i]}" == /etc/rsyslog.d/* ]]; then CHANGED_FILES[$i]=$(echo "${CHANGED_FILES[$i]}" | sed 's|^/etc/||') fi if [ "$SYNC_STATUS" = "OUT_OF_SYNC" ]; then echo " - Drift detected in: ${CHANGED_FILES[$i]}" fi done DRIFT_COUNT=${#CHANGED_FILES[@]} else echo " DEBUG: DRIFTED_FILES is empty or []" fi else echo " DEBUG: DRIFTED_FILES not found in playbook output" echo " DEBUG: Attempting to parse from changed task output..." # Fallback: Look for "changed:" indicators in the playbook output if grep -q "changed: \[" "$PLAYBOOK_LOG"; then echo " DEBUG: Found changed tasks, but no structured DRIFTED_FILES output" echo " DEBUG: This might indicate a playbook output format issue" fi fi # Additional validation: If OUT_OF_SYNC but no files found, show warning if [ "$SYNC_STATUS" = "OUT_OF_SYNC" ] && [ "$DRIFT_COUNT" -eq 0 ]; then echo " ⚠️ WARNING: Status is OUT_OF_SYNC but no drifted files were extracted" echo " ⚠️ This might indicate a parsing issue. Check the playbook output above." fi echo " Total drift count: $DRIFT_COUNT" echo "" # ───────────────────────────────────────────────────────────────────────────────── # Step 3: Build JSON payload # ───────────────────────────────────────────────────────────────────────────────── echo "Step 3/4: Building JSON payload..." # Convert files array to JSON FILES_JSON="[]" if [ ${#CHANGED_FILES[@]} -gt 0 ]; then FILES_JSON="[" for i in "${!CHANGED_FILES[@]}"; do if [ "$i" -gt 0 ]; then FILES_JSON+="," fi # Escape special characters in filenames for JSON escaped_name="${CHANGED_FILES[$i]//\\/\\\\}" escaped_name="${escaped_name//\"/\\\"}" FILES_JSON+="{\"name\":\"$escaped_name\"}" done FILES_JSON+="]" fi # Generate ISO 8601 timestamp TIMESTAMP=$(date -u +'%Y-%m-%dT%H:%M:%SZ') # Build complete JSON status STATUS_JSON=$(cat </dev/null || echo "$STATUS_JSON" echo "" # ───────────────────────────────────────────────────────────────────────────────── # Step 4: Send JSON to gitops-status-server # ───────────────────────────────────────────────────────────────────────────────── echo "Step 4/4: Sending status to gitops-status-server..." echo " URL: $GITOPS_STATUS_SERVER_URL/api/status" echo " Method: POST" echo "" # Create temporary files for response RESPONSE_BODY=$(mktemp) trap "rm -f $RESPONSE_BODY" EXIT # POST the JSON to the gitops-status-server API with full error reporting # Capture both response code and body for debugging HTTP_CODE=$(curl -s -w "%{http_code}" \ -X POST \ -H "Content-Type: application/json" \ -d "$STATUS_JSON" \ "$GITOPS_STATUS_SERVER_URL/api/status" \ -o "$RESPONSE_BODY" 2>&1 || true) # Extract HTTP code (last 3 digits) HTTP_CODE="${HTTP_CODE: -3}" echo " Response: HTTP $HTTP_CODE" # Show response body for debugging (especially on error) if [ -s "$RESPONSE_BODY" ]; then echo " Response Body:" sed 's/^/ /' "$RESPONSE_BODY" fi echo "" if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then echo "═══════════════════════════════════════════════════════════════════════════════" echo " ✓ Status update successful (HTTP $HTTP_CODE)" echo " JSON has been sent to gitops-status-server" echo "" # Verify the JSON was actually received and stored echo " Verifying JSON storage on gitops-status-server..." sleep 1 # Brief delay to ensure server processed the POST VERIFY_JSON=$(curl -s "$GITOPS_STATUS_SERVER_URL/status.json" 2>&1 || true) if echo "$VERIFY_JSON" | grep -q "\"repo\".*\"$REPO_NAME\""; then echo " ✓ Verified: Latest JSON stored correctly on server" echo "" echo " Grafana Infinity datasource will now read the updated JSON from:" echo " $GITOPS_STATUS_SERVER_URL/status.json" else echo " ⚠ Warning: Could not verify JSON storage" echo " Response from /status.json:" echo " $VERIFY_JSON" | sed 's/^/ /' fi echo "═══════════════════════════════════════════════════════════════════════════════" exit 0 else echo "═══════════════════════════════════════════════════════════════════════════════" echo " ✗ ERROR: Status update failed with HTTP $HTTP_CODE" echo " Debugging Information:" echo " - Server URL: $GITOPS_STATUS_SERVER_URL" echo " - Endpoint: /api/status" echo " - Check gitops-status-server connectivity (ping/nslookup)" echo " - Verify service port and internal port mapping" echo " - Ensure /api/status endpoint accepts POST requests" echo "═══════════════════════════════════════════════════════════════════════════════" exit 1 fi