1cbe3d4de7
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
CRITICAL FIX: Problem: drift-check.yml was using 'copy' module in check_mode, which compares: - File content ✓ - Permissions (owner, group, mode) ✗ - Ownership ✗ After deploy, files have root:root 0644 permissions. Even though content matches, the copy module marked files as 'changed' because permissions were being compared. This caused false OUT_OF_SYNC reports even when configuration was actually synced. Solution: Use MD5 checksum-based comparison instead: - Compare only file CONTENT using stat checksums - Ignore permissions/ownership differences - This is what matters for config management Also fixed URLs: - Changed back from port 80 to port 5000 (API only) - Updated service name to gitops-status-api Now drift detection only triggers on actual config changes, not permission differences. After successful deploy, should correctly report SYNCED status.
332 lines
14 KiB
Bash
332 lines
14 KiB
Bash
#!/bin/bash
|
|
# =============================================================================
|
|
# update-gitops-status.sh
|
|
#
|
|
# Purpose:
|
|
# Runs drift-check playbook and generates a JSON status snapshot for
|
|
# gitops-status-server. This replaces Pushgateway metric updates with
|
|
# richer JSON status suitable for Grafana visualization via Infinity DS.
|
|
#
|
|
# Flow:
|
|
# 1. Execute ansible/playbooks/drift-check.yml (check mode, read-only)
|
|
# 2. Capture exit code to determine sync status
|
|
# 3. Parse playbook output to extract changed files
|
|
# 4. Build structured JSON with metadata
|
|
# 5. POST JSON to gitops-status-server API
|
|
#
|
|
# Usage:
|
|
# ./update-gitops-status.sh
|
|
#
|
|
# Environment Variables:
|
|
# GITOPS_STATUS_SERVER_URL - URL of gitops-status-server API
|
|
# (default: http://gitops-status-server.observability-stack.svc.cluster.local:80)
|
|
# REPO_NAME - Repository name (default: rsyslog)
|
|
# SERVER_NAME - Server name (default: rsyslog-lab)
|
|
#
|
|
# Generated JSON Structure:
|
|
# {
|
|
# "repo": "rsyslog",
|
|
# "server": "rsyslog-lab",
|
|
# "sync_status": "SYNCED" or "OUT_OF_SYNC",
|
|
# "drift_count": <number>,
|
|
# "files": [{"name": "rsyslog.conf"}, {"name": "rsyslog.d/30-lab.conf"}],
|
|
# "last_check": "2026-04-21T10:30:00Z"
|
|
# }
|
|
#
|
|
# Exit Codes:
|
|
# 0 - Success (JSON posted to gitops-status-server regardless of sync status)
|
|
# 1 - Failure (playbook error, network error, JSON post failure, etc.)
|
|
#
|
|
# =============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Configuration
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
GITOPS_STATUS_SERVER_URL="${GITOPS_STATUS_SERVER_URL:-http://gitops-status-api.observability-stack.svc.cluster.local:5000}"
|
|
REPO_NAME="${REPO_NAME:-rsyslog}"
|
|
SERVER_NAME="${SERVER_NAME:-rsyslog-lab}"
|
|
INVENTORY_FILE="ansible/inventory/hosts.yml"
|
|
PLAYBOOK="ansible/playbooks/drift-check.yml"
|
|
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
echo " GitOps Status Update"
|
|
echo " Repository: $REPO_NAME | Server: $SERVER_NAME"
|
|
echo " Target: $GITOPS_STATUS_SERVER_URL"
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
echo ""
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
# Step 1: Run drift-check playbook
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
echo "Step 1/4: Running drift-check playbook..."
|
|
|
|
# Capture playbook output to a temp file for parsing
|
|
PLAYBOOK_LOG=$(mktemp)
|
|
KEEP_LOG="${KEEP_PLAYBOOK_LOG:-false}"
|
|
if [ "$KEEP_LOG" = "true" ]; then
|
|
PLAYBOOK_LOG="./drift-check-output.log"
|
|
echo " Playbook output will be saved to: $PLAYBOOK_LOG"
|
|
fi
|
|
|
|
# Set up cleanup trap (will be updated later with RESPONSE_BODY)
|
|
trap "rm -f $PLAYBOOK_LOG" EXIT
|
|
|
|
# Run playbook (no -v flag to avoid file descriptor exhaustion in containers)
|
|
# Exit code: 0 = synced, non-zero = drift detected (expected)
|
|
# Use YAML callback for consistent debug output format
|
|
# Limit forks to 1 to reduce file descriptor usage
|
|
set +e
|
|
ANSIBLE_FORCE_COLOR=false \
|
|
ANSIBLE_STDOUT_CALLBACK=yaml \
|
|
ANSIBLE_FORKS=1 \
|
|
ansible-playbook \
|
|
-i "$INVENTORY_FILE" \
|
|
"$PLAYBOOK" \
|
|
> "$PLAYBOOK_LOG" 2>&1
|
|
DRIFT_RC=$?
|
|
set -e
|
|
|
|
# Show playbook output for debugging (compact)
|
|
echo "Playbook output (last 25 lines):"
|
|
cat "$PLAYBOOK_LOG" | tail -25
|
|
echo ""
|
|
echo "DEBUG: Full playbook output saved to: $PLAYBOOK_LOG"
|
|
echo ""
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
# Step 2: Determine sync status and collect changed files
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
echo "Step 2/4: Analyzing drift detection results..."
|
|
|
|
CHANGED_FILES=()
|
|
DRIFT_COUNT=0
|
|
|
|
# Exit code 0 = synced (all tasks succeeded)
|
|
# Exit code non-zero = drift detected (fail task was reached)
|
|
if [ "$DRIFT_RC" -eq 0 ]; then
|
|
SYNC_STATUS="SYNCED"
|
|
echo " ✓ Status: SYNCED - server configuration matches Git"
|
|
else
|
|
SYNC_STATUS="OUT_OF_SYNC"
|
|
echo " ✗ Status: OUT OF SYNC - configuration drift detected"
|
|
fi
|
|
|
|
# Extract structured drifted files from playbook output
|
|
# The drift-check.yml playbook outputs: DRIFTED_FILES=file1,file2,file3
|
|
# With YAML callback, the output format is: msg: DRIFTED_FILES=...
|
|
echo " DEBUG: Searching for DRIFTED_FILES in playbook output..."
|
|
if grep -q "DRIFTED_FILES=" "$PLAYBOOK_LOG"; then
|
|
echo " DEBUG: Found DRIFTED_FILES pattern"
|
|
DRIFTED_FILES_LINE=$(grep "DRIFTED_FILES=" "$PLAYBOOK_LOG" | tail -1)
|
|
echo " DEBUG: Raw line: $DRIFTED_FILES_LINE"
|
|
|
|
# Extract value after DRIFTED_FILES= (handles both YAML and default callback formats)
|
|
# Format: "msg: DRIFTED_FILES=file1,file2" or "DRIFTED_FILES=file1,file2"
|
|
DRIFTED_FILES_STR=$(echo "$DRIFTED_FILES_LINE" | sed 's/.*DRIFTED_FILES=//' | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//' | tr -d '"')
|
|
echo " DEBUG: Extracted value: '$DRIFTED_FILES_STR'"
|
|
|
|
# Check if the value is an empty list ([] or empty string)
|
|
if [ -n "$DRIFTED_FILES_STR" ] && [ "$DRIFTED_FILES_STR" != "[]" ] && [ "$DRIFTED_FILES_STR" != "" ]; then
|
|
# Parse comma-separated list into array
|
|
IFS=',' read -ra CHANGED_FILES <<<"$DRIFTED_FILES_STR"
|
|
|
|
echo " DEBUG: Parsed ${#CHANGED_FILES[@]} files"
|
|
|
|
# Clean up whitespace and normalize paths
|
|
for i in "${!CHANGED_FILES[@]}"; do
|
|
CHANGED_FILES[$i]=$(echo "${CHANGED_FILES[$i]}" | xargs)
|
|
|
|
# Convert full paths to relative paths for cleaner output
|
|
if [[ "${CHANGED_FILES[$i]}" == /etc/rsyslog.conf ]]; then
|
|
CHANGED_FILES[$i]="rsyslog.conf"
|
|
elif [[ "${CHANGED_FILES[$i]}" == /etc/rsyslog.d/* ]]; then
|
|
CHANGED_FILES[$i]=$(echo "${CHANGED_FILES[$i]}" | sed 's|^/etc/||')
|
|
fi
|
|
|
|
if [ "$SYNC_STATUS" = "OUT_OF_SYNC" ]; then
|
|
echo " - Drift detected in: ${CHANGED_FILES[$i]}"
|
|
fi
|
|
done
|
|
|
|
DRIFT_COUNT=${#CHANGED_FILES[@]}
|
|
else
|
|
echo " DEBUG: DRIFTED_FILES is empty or []"
|
|
fi
|
|
else
|
|
echo " DEBUG: DRIFTED_FILES not found in playbook output"
|
|
echo " DEBUG: Attempting to parse from changed task output..."
|
|
|
|
# Fallback: Look for "changed:" indicators in the playbook output
|
|
if grep -q "changed: \[" "$PLAYBOOK_LOG"; then
|
|
echo " DEBUG: Found changed tasks, but no structured DRIFTED_FILES output"
|
|
echo " DEBUG: This might indicate a playbook output format issue"
|
|
fi
|
|
fi
|
|
|
|
# Additional validation: If OUT_OF_SYNC but no files found, show warning
|
|
if [ "$SYNC_STATUS" = "OUT_OF_SYNC" ] && [ "$DRIFT_COUNT" -eq 0 ]; then
|
|
echo " ⚠️ WARNING: Status is OUT_OF_SYNC but no drifted files were extracted"
|
|
echo " ⚠️ This might indicate a parsing issue. Check the playbook output above."
|
|
fi
|
|
|
|
echo " Total drift count: $DRIFT_COUNT"
|
|
echo ""
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
# Step 3: Build JSON payload
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
echo "Step 3/4: Building JSON payload..."
|
|
|
|
# Convert files array to JSON
|
|
FILES_JSON="[]"
|
|
if [ ${#CHANGED_FILES[@]} -gt 0 ]; then
|
|
FILES_JSON="["
|
|
for i in "${!CHANGED_FILES[@]}"; do
|
|
if [ "$i" -gt 0 ]; then
|
|
FILES_JSON+=","
|
|
fi
|
|
# Escape special characters in filenames for JSON
|
|
escaped_name="${CHANGED_FILES[$i]//\\/\\\\}"
|
|
escaped_name="${escaped_name//\"/\\\"}"
|
|
FILES_JSON+="{\"name\":\"$escaped_name\"}"
|
|
done
|
|
FILES_JSON+="]"
|
|
fi
|
|
|
|
# Generate ISO 8601 timestamp
|
|
TIMESTAMP=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
|
|
|
|
# Build complete JSON status
|
|
STATUS_JSON=$(cat <<EOF
|
|
{
|
|
"repo": "$REPO_NAME",
|
|
"server": "$SERVER_NAME",
|
|
"sync_status": "$SYNC_STATUS",
|
|
"drift_count": $DRIFT_COUNT,
|
|
"files": $FILES_JSON,
|
|
"last_check": "$TIMESTAMP"
|
|
}
|
|
EOF
|
|
)
|
|
|
|
echo " Generated JSON:"
|
|
echo "$STATUS_JSON" | jq '.' 2>/dev/null || echo "$STATUS_JSON"
|
|
echo ""
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
# Step 4: Send JSON to gitops-status-server
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
echo "Step 4/4: Sending status to gitops-status-server..."
|
|
echo " URL: $GITOPS_STATUS_SERVER_URL/api/status"
|
|
echo " Method: POST"
|
|
echo " Request Payload:"
|
|
echo "$STATUS_JSON" | jq '.' 2>/dev/null | sed 's/^/ /' || echo "$STATUS_JSON" | sed 's/^/ /'
|
|
echo ""
|
|
echo "Step 4/4: Sending status to gitops-status-server..."
|
|
echo " URL: $GITOPS_STATUS_SERVER_URL/api/status"
|
|
echo " Method: POST"
|
|
echo " Payload size: $(echo "$STATUS_JSON" | wc -c) bytes"
|
|
echo ""
|
|
echo " ========== JSON PAYLOAD BEING SENT =========="
|
|
echo "$STATUS_JSON" | jq '.' 2>/dev/null || echo "$STATUS_JSON"
|
|
echo " =========================================="
|
|
echo ""
|
|
|
|
# Test connectivity first
|
|
echo " Testing connectivity to gitops-status-server..."
|
|
if ! curl -s -m 5 "$GITOPS_STATUS_SERVER_URL/health" > /dev/null 2>&1; then
|
|
echo " ✗ WARNING: Cannot reach $GITOPS_STATUS_SERVER_URL/health"
|
|
echo " Attempting DNS resolution..."
|
|
nslookup gitops-status-server.observability-stack.svc.cluster.local || true
|
|
echo ""
|
|
else
|
|
echo " ✓ Server is reachable"
|
|
fi
|
|
echo ""
|
|
|
|
# Create temporary files for response
|
|
RESPONSE_BODY=$(mktemp)
|
|
trap "rm -f $RESPONSE_BODY $PLAYBOOK_LOG" EXIT
|
|
|
|
echo " Sending POST request with curl..."
|
|
echo " Command: curl -X POST -H 'Content-Type: application/json' -d '<JSON>' $GITOPS_STATUS_SERVER_URL/api/status"
|
|
echo ""
|
|
|
|
# POST the JSON to the gitops-status-server API with full error reporting
|
|
# Capture both response code and body for debugging
|
|
set +e
|
|
HTTP_RESPONSE=$(curl -s -w "\n%{http_code}" \
|
|
-X POST \
|
|
-H "Content-Type: application/json" \
|
|
-d "$STATUS_JSON" \
|
|
"$GITOPS_STATUS_SERVER_URL/api/status" \
|
|
2>&1)
|
|
CURL_EXIT=$?
|
|
set -e
|
|
|
|
if [ $CURL_EXIT -ne 0 ]; then
|
|
echo " ✗ CURL FAILED with exit code $CURL_EXIT"
|
|
echo " Error output: $HTTP_RESPONSE"
|
|
exit 1
|
|
fi
|
|
|
|
# Split response: body is everything except last line, code is last line
|
|
HTTP_CODE=$(echo "$HTTP_RESPONSE" | tail -1)
|
|
RESPONSE_CONTENT=$(echo "$HTTP_RESPONSE" | head -n -1)
|
|
echo "$RESPONSE_CONTENT" > "$RESPONSE_BODY"
|
|
|
|
# Validate HTTP code is numeric
|
|
if ! [[ "$HTTP_CODE" =~ ^[0-9]+$ ]]; then
|
|
echo " ✗ ERROR: Invalid HTTP response code: $HTTP_CODE"
|
|
echo " Full response: $HTTP_RESPONSE"
|
|
exit 1
|
|
fi
|
|
|
|
echo " Response: HTTP $HTTP_CODE"
|
|
|
|
# Show response body for debugging (especially on error)
|
|
if [ -s "$RESPONSE_BODY" ]; then
|
|
echo " Response Body:"
|
|
sed 's/^/ /' "$RESPONSE_BODY"
|
|
fi
|
|
echo ""
|
|
|
|
if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
echo " ✓ Status update successful (HTTP $HTTP_CODE)"
|
|
echo " JSON has been sent to gitops-status-server"
|
|
echo ""
|
|
|
|
# Verify the JSON was actually received and stored
|
|
echo " Verifying JSON storage on gitops-status-server..."
|
|
sleep 1 # Brief delay to ensure server processed the POST
|
|
|
|
VERIFY_JSON=$(curl -s "$GITOPS_STATUS_SERVER_URL/api/status" 2>&1 || true)
|
|
if echo "$VERIFY_JSON" | grep -q "\"repo\".*\"$REPO_NAME\""; then
|
|
echo " ✓ Verified: Latest JSON stored correctly on server"
|
|
echo ""
|
|
echo " Grafana Infinity datasource will now read the updated JSON from:"
|
|
echo " $GITOPS_STATUS_SERVER_URL/api/status"
|
|
else
|
|
echo " ⚠ Warning: Could not verify JSON storage"
|
|
echo " Response from /status.json:"
|
|
echo " $VERIFY_JSON" | sed 's/^/ /'
|
|
fi
|
|
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
exit 0
|
|
else
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
echo " ✗ ERROR: Status update failed with HTTP $HTTP_CODE"
|
|
echo " Debugging Information:"
|
|
echo " - Server URL: $GITOPS_STATUS_SERVER_URL"
|
|
echo " - Endpoint: /api/status"
|
|
echo " - Check gitops-status-server connectivity (ping/nslookup)"
|
|
echo " - Verify service port and internal port mapping"
|
|
echo " - Ensure /api/status endpoint accepts POST requests"
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
exit 1
|
|
fi
|