415 lines
18 KiB
Bash
415 lines
18 KiB
Bash
#!/bin/bash
|
|
# =============================================================================
|
|
# update-gitops-status.sh
|
|
#
|
|
# Purpose:
|
|
# Runs drift-check playbook and generates a JSON status snapshot for
|
|
# gitops-status-server. This replaces Pushgateway metric updates with
|
|
# richer JSON status suitable for Grafana visualization via Infinity DS.
|
|
#
|
|
# Flow:
|
|
# 1. Execute ansible/playbooks/drift-check.yml (check mode, read-only)
|
|
# 2. Capture exit code to determine sync status
|
|
# 3. Parse playbook output to extract changed files
|
|
# 4. Build structured JSON with metadata
|
|
# 5. POST JSON to gitops-status-server API
|
|
#
|
|
# Usage:
|
|
# ./update-gitops-status.sh
|
|
#
|
|
# Environment Variables:
|
|
# GITOPS_STATUS_SERVER_URL - URL of gitops-status-server API
|
|
# (default: http://gitops-status-server.observability-stack.svc.cluster.local:80)
|
|
# REPO_NAME - Repository name (default: rsyslog)
|
|
# SERVER_NAME - Server name (default: rsyslog-lab)
|
|
#
|
|
# Generated JSON Structure:
|
|
# {
|
|
# "repo": "rsyslog",
|
|
# "server": "rsyslog-lab",
|
|
# "sync_status": "SYNCED" or "OUT_OF_SYNC",
|
|
# "drift_count": <number>,
|
|
# "files": [{"name": "rsyslog.conf"}, {"name": "rsyslog.d/30-lab.conf"}],
|
|
# "last_check": "2026-04-21T10:30:00Z"
|
|
# }
|
|
#
|
|
# Exit Codes:
|
|
# 0 - Success (JSON posted to gitops-status-server regardless of sync status)
|
|
# 1 - Failure (playbook error, network error, JSON post failure, etc.)
|
|
#
|
|
# =============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Configuration
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
GITOPS_STATUS_SERVER_URL="${GITOPS_STATUS_SERVER_URL:-http://gitops-status-server.observability-stack.svc.cluster.local:5000}"
|
|
REPO_NAME="${REPO_NAME:-rsyslog}"
|
|
SERVER_NAME="${SERVER_NAME:-rsyslog-lab}"
|
|
INVENTORY_FILE="ansible/inventory/hosts.yml"
|
|
PLAYBOOK="ansible/playbooks/drift-check.yml"
|
|
MODE="${MODE:-drift-check}" # drift-check or post-deploy
|
|
PLAYBOOK_LOG="" # Initialize to avoid unbound variable error
|
|
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
echo " GitOps Status Update"
|
|
echo " Repository: $REPO_NAME | Server: $SERVER_NAME"
|
|
echo " Target: $GITOPS_STATUS_SERVER_URL"
|
|
echo " Mode: $MODE"
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
echo ""
|
|
|
|
CHANGED_FILES=()
|
|
DRIFT_COUNT=0
|
|
SYNC_STATUS="SYNCED"
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
# MODE 1: post-deploy - Report what files were deployed from Git
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
if [ "$MODE" = "post-deploy" ]; then
|
|
echo "Step 1/4: Analyzing Git changes (what was just deployed)..."
|
|
|
|
# Check what files changed in the last commit in files/ directory
|
|
if command -v git >/dev/null 2>&1 && [ -d .git ]; then
|
|
# Try multiple methods to detect changed files (Woodpecker might do shallow clone)
|
|
|
|
# Method 1: Use Woodpecker CI environment variables (most reliable in CI)
|
|
if [ -n "${CI_PREV_COMMIT_SHA:-}" ] && [ -n "${CI_COMMIT_SHA:-}" ]; then
|
|
echo " DEBUG: Using Woodpecker commit range: ${CI_PREV_COMMIT_SHA}..${CI_COMMIT_SHA}"
|
|
CHANGED_FILE_PATHS=$(git diff --name-only "${CI_PREV_COMMIT_SHA}" "${CI_COMMIT_SHA}" -- files/ 2>/dev/null || echo "")
|
|
# Method 2: Use git diff with HEAD~1
|
|
elif git rev-parse HEAD~1 >/dev/null 2>&1; then
|
|
echo " DEBUG: Using git diff HEAD~1..HEAD"
|
|
CHANGED_FILE_PATHS=$(git diff --name-only HEAD~1 HEAD -- files/ 2>/dev/null || echo "")
|
|
# Method 3: Use git show (less accurate - shows all files in commit)
|
|
else
|
|
echo " DEBUG: Using git show HEAD (may show all files in commit)"
|
|
CHANGED_FILE_PATHS=$(git show --name-only --pretty="" HEAD -- files/ 2>/dev/null || echo "")
|
|
fi
|
|
|
|
echo " DEBUG: Git detection method results: '$CHANGED_FILE_PATHS'"
|
|
|
|
if [ -n "$CHANGED_FILE_PATHS" ]; then
|
|
echo " Files changed in last commit:"
|
|
while IFS= read -r filepath; do
|
|
if [ -n "$filepath" ]; then
|
|
# Strip "files/" prefix to get the config file name
|
|
filename="${filepath#files/}"
|
|
|
|
# Skip if it's just "files/" (directory change)
|
|
if [ "$filename" != "" ] && [ "$filename" != "files/" ]; then
|
|
CHANGED_FILES+=("$filename")
|
|
echo " - $filename"
|
|
fi
|
|
fi
|
|
done <<< "$CHANGED_FILE_PATHS"
|
|
|
|
DRIFT_COUNT=${#CHANGED_FILES[@]}
|
|
else
|
|
echo " No files changed in files/ directory"
|
|
fi
|
|
else
|
|
echo " Git not available, cannot determine deployed files"
|
|
fi
|
|
|
|
# Always SYNCED after successful deploy
|
|
SYNC_STATUS="SYNCED"
|
|
echo " ✓ Status: SYNCED - files were deployed successfully"
|
|
echo " Total deployed files: $DRIFT_COUNT"
|
|
echo ""
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
# MODE 2: drift-check - Check for manual changes on server (drift detection)
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
else
|
|
echo "Step 1/4: Running drift-check playbook..."
|
|
|
|
# Capture playbook output to a temp file for parsing
|
|
PLAYBOOK_LOG=$(mktemp)
|
|
KEEP_LOG="${KEEP_PLAYBOOK_LOG:-false}"
|
|
if [ "$KEEP_LOG" = "true" ]; then
|
|
PLAYBOOK_LOG="./drift-check-output.log"
|
|
echo " Playbook output will be saved to: $PLAYBOOK_LOG"
|
|
fi
|
|
|
|
# Set up cleanup trap (will be updated later with RESPONSE_BODY)
|
|
trap "rm -f $PLAYBOOK_LOG" EXIT
|
|
|
|
# Run playbook (no -v flag to avoid file descriptor exhaustion in containers)
|
|
# Exit code: 0 = synced, non-zero = drift detected (expected)
|
|
# Limit forks to 1 to reduce file descriptor usage
|
|
set +e
|
|
ANSIBLE_FORCE_COLOR=false \
|
|
ANSIBLE_FORKS=1 \
|
|
ansible-playbook \
|
|
-i "$INVENTORY_FILE" \
|
|
"$PLAYBOOK" \
|
|
> "$PLAYBOOK_LOG" 2>&1
|
|
DRIFT_RC=$?
|
|
set -e
|
|
|
|
# Show playbook output for debugging (compact)
|
|
echo "Playbook output (last 25 lines):"
|
|
cat "$PLAYBOOK_LOG" | tail -25
|
|
echo ""
|
|
echo "DEBUG: Full playbook output saved to: $PLAYBOOK_LOG"
|
|
echo ""
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
# Step 2: Determine sync status and collect changed files
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
echo "Step 2/4: Analyzing drift detection results..."
|
|
|
|
# Exit code 0 = synced (all tasks succeeded)
|
|
# Exit code non-zero = drift detected (fail task was reached)
|
|
if [ "$DRIFT_RC" -eq 0 ]; then
|
|
SYNC_STATUS="SYNCED"
|
|
echo " ✓ Status: SYNCED - server configuration matches Git"
|
|
else
|
|
SYNC_STATUS="OUT_OF_SYNC"
|
|
echo " ✗ Status: OUT OF SYNC - configuration drift detected"
|
|
fi
|
|
|
|
# Extract structured drifted files from playbook output
|
|
# The drift-check.yml playbook outputs: DRIFTED_FILES=file1,file2,file3
|
|
echo " DEBUG: Searching for DRIFTED_FILES in playbook output..."
|
|
if grep -q "DRIFTED_FILES=" "$PLAYBOOK_LOG"; then
|
|
echo " DEBUG: Found DRIFTED_FILES pattern"
|
|
DRIFTED_FILES_LINE=$(grep "DRIFTED_FILES=" "$PLAYBOOK_LOG" | tail -1)
|
|
echo " DEBUG: Raw line: $DRIFTED_FILES_LINE"
|
|
|
|
# Extract value after DRIFTED_FILES= (handles both YAML and default callback formats)
|
|
# Format: "msg: DRIFTED_FILES=file1,file2" or "DRIFTED_FILES=file1,file2"
|
|
DRIFTED_FILES_STR=$(echo "$DRIFTED_FILES_LINE" | sed 's/.*DRIFTED_FILES=//' | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//' | tr -d '"')
|
|
echo " DEBUG: Extracted value: '$DRIFTED_FILES_STR'"
|
|
|
|
# Check if the value is an empty list ([] or empty string)
|
|
if [ -n "$DRIFTED_FILES_STR" ] && [ "$DRIFTED_FILES_STR" != "[]" ] && [ "$DRIFTED_FILES_STR" != "" ]; then
|
|
# Parse comma-separated list into array
|
|
IFS=',' read -ra CHANGED_FILES <<<"$DRIFTED_FILES_STR"
|
|
|
|
echo " DEBUG: Parsed ${#CHANGED_FILES[@]} files"
|
|
|
|
# Clean up whitespace and normalize paths
|
|
for i in "${!CHANGED_FILES[@]}"; do
|
|
CHANGED_FILES[$i]=$(echo "${CHANGED_FILES[$i]}" | xargs)
|
|
|
|
# Convert full paths to relative paths for cleaner output
|
|
if [[ "${CHANGED_FILES[$i]}" == /etc/rsyslog.conf ]]; then
|
|
CHANGED_FILES[$i]="rsyslog.conf"
|
|
elif [[ "${CHANGED_FILES[$i]}" == /etc/rsyslog.d/* ]]; then
|
|
CHANGED_FILES[$i]=$(echo "${CHANGED_FILES[$i]}" | sed 's|^/etc/||')
|
|
fi
|
|
|
|
if [ "$SYNC_STATUS" = "OUT_OF_SYNC" ]; then
|
|
echo " - Drift detected in: ${CHANGED_FILES[$i]}"
|
|
fi
|
|
done
|
|
|
|
DRIFT_COUNT=${#CHANGED_FILES[@]}
|
|
else
|
|
echo " DEBUG: DRIFTED_FILES is empty or []"
|
|
fi
|
|
else
|
|
echo " DEBUG: DRIFTED_FILES not found in playbook output"
|
|
fi
|
|
|
|
# Additional validation: If OUT_OF_SYNC but no files found, show warning
|
|
if [ "$SYNC_STATUS" = "OUT_OF_SYNC" ] && [ "$DRIFT_COUNT" -eq 0 ]; then
|
|
echo " ⚠️ WARNING: Status is OUT_OF_SYNC but no drifted files were extracted"
|
|
echo " ⚠️ This might indicate a parsing issue. Check the playbook output above."
|
|
fi
|
|
|
|
echo " Total drift count: $DRIFT_COUNT"
|
|
echo ""
|
|
fi
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
# Step 3: Build JSON payload
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
echo "Step 3/4: Building JSON payload..."
|
|
|
|
# Determine which field to populate based on mode
|
|
if [ "$MODE" = "post-deploy" ]; then
|
|
# Post-deploy: populate deployed_files, keep drifted_files empty
|
|
DEPLOYED_FILES_JSON="["
|
|
DRIFTED_FILES_JSON="[]"
|
|
|
|
if [ ${#CHANGED_FILES[@]} -gt 0 ]; then
|
|
for i in "${!CHANGED_FILES[@]}"; do
|
|
if [ "$i" -gt 0 ]; then
|
|
DEPLOYED_FILES_JSON+=","
|
|
fi
|
|
escaped_name="${CHANGED_FILES[$i]//\\/\\\\}"
|
|
escaped_name="${escaped_name//\"/\\\"}"
|
|
DEPLOYED_FILES_JSON+="{\"name\":\"$escaped_name\"}"
|
|
done
|
|
fi
|
|
DEPLOYED_FILES_JSON+="]"
|
|
else
|
|
# Drift-check mode: Keep existing deployed_files, only update drifted_files
|
|
# First fetch current status to preserve deployed_files
|
|
echo " Fetching current deployed_files from API..."
|
|
CURRENT_JSON=$(curl -s "$GITOPS_STATUS_SERVER_URL/api/status" 2>/dev/null || echo "{}")
|
|
|
|
# Extract deployed_files from current JSON (use jq if available)
|
|
if command -v jq >/dev/null 2>&1; then
|
|
DEPLOYED_FILES_JSON=$(echo "$CURRENT_JSON" | jq -c '.deployed_files // []' 2>/dev/null || echo "[]")
|
|
echo " Preserving deployed_files: $DEPLOYED_FILES_JSON"
|
|
else
|
|
DEPLOYED_FILES_JSON="[]"
|
|
echo " WARNING: jq not available, cannot preserve deployed_files"
|
|
fi
|
|
|
|
# Build drifted_files from detected drift
|
|
DRIFTED_FILES_JSON="["
|
|
if [ ${#CHANGED_FILES[@]} -gt 0 ]; then
|
|
for i in "${!CHANGED_FILES[@]}"; do
|
|
if [ "$i" -gt 0 ]; then
|
|
DRIFTED_FILES_JSON+=","
|
|
fi
|
|
escaped_name="${CHANGED_FILES[$i]//\\/\\\\}"
|
|
escaped_name="${escaped_name//\"/\\\"}"
|
|
DRIFTED_FILES_JSON+="{\"name\":\"$escaped_name\"}"
|
|
done
|
|
fi
|
|
DRIFTED_FILES_JSON+="]"
|
|
fi
|
|
|
|
# Generate ISO 8601 timestamp
|
|
TIMESTAMP=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
|
|
|
|
# Build complete JSON status with both fields
|
|
STATUS_JSON=$(cat <<EOF
|
|
{
|
|
"repo": "$REPO_NAME",
|
|
"server": "$SERVER_NAME",
|
|
"sync_status": "$SYNC_STATUS",
|
|
"drift_count": $DRIFT_COUNT,
|
|
"deployed_files": $DEPLOYED_FILES_JSON,
|
|
"drifted_files": $DRIFTED_FILES_JSON,
|
|
"last_check": "$TIMESTAMP"
|
|
}
|
|
EOF
|
|
)
|
|
|
|
echo " Generated JSON:"
|
|
echo "$STATUS_JSON" | jq '.' 2>/dev/null || echo "$STATUS_JSON"
|
|
echo ""
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
# Step 4: Send JSON to gitops-status-server
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
echo "Step 4/4: Sending status to gitops-status-server..."
|
|
echo " URL: $GITOPS_STATUS_SERVER_URL/api/status"
|
|
echo " Method: POST"
|
|
echo " Request Payload:"
|
|
echo "$STATUS_JSON" | jq '.' 2>/dev/null | sed 's/^/ /' || echo "$STATUS_JSON" | sed 's/^/ /'
|
|
echo ""
|
|
echo "Step 4/4: Sending status to gitops-status-server..."
|
|
echo " URL: $GITOPS_STATUS_SERVER_URL/api/status"
|
|
echo " Method: POST"
|
|
echo " Payload size: $(echo "$STATUS_JSON" | wc -c) bytes"
|
|
echo ""
|
|
echo " ========== JSON PAYLOAD BEING SENT =========="
|
|
echo "$STATUS_JSON" | jq '.' 2>/dev/null || echo "$STATUS_JSON"
|
|
echo " =========================================="
|
|
echo ""
|
|
|
|
# Test connectivity first
|
|
echo " Testing connectivity to gitops-status-server..."
|
|
if ! curl -s -m 5 "$GITOPS_STATUS_SERVER_URL/health" > /dev/null 2>&1; then
|
|
echo " ✗ WARNING: Cannot reach $GITOPS_STATUS_SERVER_URL/health"
|
|
echo " Attempting DNS resolution..."
|
|
nslookup gitops-status-server.observability-stack.svc.cluster.local || true
|
|
echo ""
|
|
else
|
|
echo " ✓ Server is reachable"
|
|
fi
|
|
echo ""
|
|
|
|
# Create temporary files for response
|
|
RESPONSE_BODY=$(mktemp)
|
|
trap "rm -f $RESPONSE_BODY $PLAYBOOK_LOG" EXIT
|
|
|
|
echo " Sending POST request with curl..."
|
|
echo " Command: curl -X POST -H 'Content-Type: application/json' -d '<JSON>' $GITOPS_STATUS_SERVER_URL/api/status"
|
|
echo ""
|
|
|
|
# POST the JSON to the gitops-status-server API with full error reporting
|
|
# Capture both response code and body for debugging
|
|
set +e
|
|
HTTP_RESPONSE=$(curl -s -w "\n%{http_code}" \
|
|
-X POST \
|
|
-H "Content-Type: application/json" \
|
|
-d "$STATUS_JSON" \
|
|
"$GITOPS_STATUS_SERVER_URL/api/status" \
|
|
2>&1)
|
|
CURL_EXIT=$?
|
|
set -e
|
|
|
|
if [ $CURL_EXIT -ne 0 ]; then
|
|
echo " ✗ CURL FAILED with exit code $CURL_EXIT"
|
|
echo " Error output: $HTTP_RESPONSE"
|
|
exit 1
|
|
fi
|
|
|
|
# Split response: body is everything except last line, code is last line
|
|
HTTP_CODE=$(echo "$HTTP_RESPONSE" | tail -1)
|
|
RESPONSE_CONTENT=$(echo "$HTTP_RESPONSE" | head -n -1)
|
|
echo "$RESPONSE_CONTENT" > "$RESPONSE_BODY"
|
|
|
|
# Validate HTTP code is numeric
|
|
if ! [[ "$HTTP_CODE" =~ ^[0-9]+$ ]]; then
|
|
echo " ✗ ERROR: Invalid HTTP response code: $HTTP_CODE"
|
|
echo " Full response: $HTTP_RESPONSE"
|
|
exit 1
|
|
fi
|
|
|
|
echo " Response: HTTP $HTTP_CODE"
|
|
|
|
# Show response body for debugging (especially on error)
|
|
if [ -s "$RESPONSE_BODY" ]; then
|
|
echo " Response Body:"
|
|
sed 's/^/ /' "$RESPONSE_BODY"
|
|
fi
|
|
echo ""
|
|
|
|
if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
echo " ✓ Status update successful (HTTP $HTTP_CODE)"
|
|
echo " JSON has been sent to gitops-status-server"
|
|
echo ""
|
|
|
|
# Verify the JSON was actually received and stored
|
|
echo " Verifying JSON storage on gitops-status-server..."
|
|
sleep 1 # Brief delay to ensure server processed the POST
|
|
|
|
VERIFY_JSON=$(curl -s "$GITOPS_STATUS_SERVER_URL/api/status" 2>&1 || true)
|
|
if echo "$VERIFY_JSON" | grep -q "\"repo\".*\"$REPO_NAME\""; then
|
|
echo " ✓ Verified: Latest JSON stored correctly on server"
|
|
echo ""
|
|
echo " Grafana Infinity datasource will now read the updated JSON from:"
|
|
echo " $GITOPS_STATUS_SERVER_URL/api/status"
|
|
else
|
|
echo " ⚠ Warning: Could not verify JSON storage"
|
|
echo " Response from /status.json:"
|
|
echo " $VERIFY_JSON" | sed 's/^/ /'
|
|
fi
|
|
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
exit 0
|
|
else
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
echo " ✗ ERROR: Status update failed with HTTP $HTTP_CODE"
|
|
echo " Debugging Information:"
|
|
echo " - Server URL: $GITOPS_STATUS_SERVER_URL"
|
|
echo " - Endpoint: /api/status"
|
|
echo " - Check gitops-status-server connectivity (ping/nslookup)"
|
|
echo " - Verify service port and internal port mapping"
|
|
echo " - Ensure /api/status endpoint accepts POST requests"
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
exit 1
|
|
fi
|