fc8cb0c40e
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
- Remove -v verbose flag from ansible-playbook (was causing file descriptor exhaustion) - Add recurse: false to find tasks (prevents recursive directory traversal) - Set ulimit -n 4096 in Woodpecker container to increase FD limit - Add ANSIBLE_* environment variables for container optimization: - ANSIBLE_HOST_KEY_CHECKING=False (skip SSH key verification) - ANSIBLE_CALLBACK_WHITELIST=minimal (reduce output verbosity) - ANSIBLE_FORCE_COLOR=False (no ANSI color codes) - ANSIBLE_RETRY_FILES_ENABLED=False (don't create retry files) This resolves fsnotify watcher errors in container environments with low FD limits.
247 lines
11 KiB
Bash
247 lines
11 KiB
Bash
#!/bin/bash
|
|
# =============================================================================
|
|
# update-gitops-status.sh
|
|
#
|
|
# Purpose:
|
|
# Runs drift-check playbook and generates a JSON status snapshot for
|
|
# gitops-status-server. This replaces Pushgateway metric updates with
|
|
# richer JSON status suitable for Grafana visualization via Infinity DS.
|
|
#
|
|
# Flow:
|
|
# 1. Execute ansible/playbooks/drift-check.yml (check mode, read-only)
|
|
# 2. Capture exit code to determine sync status
|
|
# 3. Parse playbook output to extract changed files
|
|
# 4. Build structured JSON with metadata
|
|
# 5. POST JSON to gitops-status-server API
|
|
#
|
|
# Usage:
|
|
# ./update-gitops-status.sh
|
|
#
|
|
# Environment Variables:
|
|
# GITOPS_STATUS_SERVER_URL - URL of gitops-status-server API
|
|
# (default: http://gitops-status-server.observability-stack.svc.cluster.local:80)
|
|
# REPO_NAME - Repository name (default: rsyslog)
|
|
# SERVER_NAME - Server name (default: rsyslog-lab)
|
|
#
|
|
# Generated JSON Structure:
|
|
# {
|
|
# "repo": "rsyslog",
|
|
# "server": "rsyslog-lab",
|
|
# "sync_status": "SYNCED" or "OUT_OF_SYNC",
|
|
# "drift_count": <number>,
|
|
# "files": [{"name": "rsyslog.conf"}, {"name": "rsyslog.d/30-lab.conf"}],
|
|
# "last_check": "2026-04-21T10:30:00Z"
|
|
# }
|
|
#
|
|
# Exit Codes:
|
|
# 0 - Success (JSON posted to gitops-status-server regardless of sync status)
|
|
# 1 - Failure (playbook error, network error, JSON post failure, etc.)
|
|
#
|
|
# =============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Configuration
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
GITOPS_STATUS_SERVER_URL="${GITOPS_STATUS_SERVER_URL:-http://gitops-status-server.observability-stack.svc.cluster.local:80}"
|
|
REPO_NAME="${REPO_NAME:-rsyslog}"
|
|
SERVER_NAME="${SERVER_NAME:-rsyslog-lab}"
|
|
INVENTORY_FILE="ansible/inventory/hosts.yml"
|
|
PLAYBOOK="ansible/playbooks/drift-check.yml"
|
|
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
echo " GitOps Status Update"
|
|
echo " Repository: $REPO_NAME | Server: $SERVER_NAME"
|
|
echo " Target: $GITOPS_STATUS_SERVER_URL"
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
echo ""
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
# Step 1: Run drift-check playbook
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
echo "Step 1/4: Running drift-check playbook..."
|
|
|
|
# Capture playbook output to a temp file for parsing
|
|
PLAYBOOK_LOG=$(mktemp)
|
|
trap "rm -f $PLAYBOOK_LOG" EXIT
|
|
|
|
# Run playbook (no -v flag to avoid file descriptor exhaustion in containers)
|
|
# Exit code: 0 = synced, non-zero = drift detected (expected)
|
|
set +e
|
|
ansible-playbook \
|
|
-i "$INVENTORY_FILE" \
|
|
"$PLAYBOOK" \
|
|
> "$PLAYBOOK_LOG" 2>&1
|
|
DRIFT_RC=$?
|
|
set -e
|
|
|
|
# Show playbook output for debugging (compact)
|
|
echo "Playbook output:"
|
|
cat "$PLAYBOOK_LOG" | tail -20
|
|
echo ""
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
# Step 2: Determine sync status and collect changed files
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
echo "Step 2/4: Analyzing drift detection results..."
|
|
|
|
CHANGED_FILES=()
|
|
DRIFT_COUNT=0
|
|
|
|
# Exit code 0 = synced (all tasks succeeded)
|
|
# Exit code non-zero = drift detected (fail task was reached)
|
|
if [ "$DRIFT_RC" -eq 0 ]; then
|
|
SYNC_STATUS="SYNCED"
|
|
echo " ✓ Status: SYNCED - server configuration matches Git"
|
|
else
|
|
SYNC_STATUS="OUT_OF_SYNC"
|
|
echo " ✗ Status: OUT OF SYNC - configuration drift detected"
|
|
fi
|
|
|
|
# Extract structured drifted files from playbook output
|
|
# The drift-check.yml playbook outputs: DRIFTED_FILES=file1,file2,file3
|
|
# Search for the pattern in the output
|
|
if grep -q "DRIFTED_FILES=" "$PLAYBOOK_LOG"; then
|
|
DRIFTED_FILES_STR=$(grep "DRIFTED_FILES=" "$PLAYBOOK_LOG" | tail -1)
|
|
# Remove ANSI color codes and extract the value
|
|
DRIFTED_FILES_STR=$(echo "$DRIFTED_FILES_STR" | sed 's/.*DRIFTED_FILES=//' | sed 's/\x1b\[[0-9;]*m//g' | xargs)
|
|
|
|
if [ -n "$DRIFTED_FILES_STR" ]; then
|
|
# Parse comma-separated list into array
|
|
IFS=',' read -ra CHANGED_FILES <<<"$DRIFTED_FILES_STR"
|
|
|
|
# Clean up whitespace
|
|
for i in "${!CHANGED_FILES[@]}"; do
|
|
CHANGED_FILES[$i]=$(echo "${CHANGED_FILES[$i]}" | xargs)
|
|
|
|
# Convert full paths to relative paths for cleaner output
|
|
if [[ "${CHANGED_FILES[$i]}" == /etc/rsyslog.conf ]]; then
|
|
CHANGED_FILES[$i]="rsyslog.conf"
|
|
elif [[ "${CHANGED_FILES[$i]}" == /etc/rsyslog.d/* ]]; then
|
|
CHANGED_FILES[$i]=$(echo "${CHANGED_FILES[$i]}" | sed 's|^/etc/||')
|
|
fi
|
|
|
|
if [ "$SYNC_STATUS" = "OUT_OF_SYNC" ]; then
|
|
echo " - Drift detected in: ${CHANGED_FILES[$i]}"
|
|
fi
|
|
done
|
|
|
|
DRIFT_COUNT=${#CHANGED_FILES[@]}
|
|
fi
|
|
fi
|
|
|
|
echo " Total drift count: $DRIFT_COUNT"
|
|
echo ""
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
# Step 3: Build JSON payload
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
echo "Step 3/4: Building JSON payload..."
|
|
|
|
# Convert files array to JSON
|
|
FILES_JSON="[]"
|
|
if [ ${#CHANGED_FILES[@]} -gt 0 ]; then
|
|
FILES_JSON="["
|
|
for i in "${!CHANGED_FILES[@]}"; do
|
|
if [ "$i" -gt 0 ]; then
|
|
FILES_JSON+=","
|
|
fi
|
|
# Escape special characters in filenames for JSON
|
|
escaped_name="${CHANGED_FILES[$i]//\\/\\\\}"
|
|
escaped_name="${escaped_name//\"/\\\"}"
|
|
FILES_JSON+="{\"name\":\"$escaped_name\"}"
|
|
done
|
|
FILES_JSON+="]"
|
|
fi
|
|
|
|
# Generate ISO 8601 timestamp
|
|
TIMESTAMP=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
|
|
|
|
# Build complete JSON status
|
|
STATUS_JSON=$(cat <<EOF
|
|
{
|
|
"repo": "$REPO_NAME",
|
|
"server": "$SERVER_NAME",
|
|
"sync_status": "$SYNC_STATUS",
|
|
"drift_count": $DRIFT_COUNT,
|
|
"files": $FILES_JSON,
|
|
"last_check": "$TIMESTAMP"
|
|
}
|
|
EOF
|
|
)
|
|
|
|
echo " Generated JSON:"
|
|
echo "$STATUS_JSON" | jq '.' 2>/dev/null || echo "$STATUS_JSON"
|
|
echo ""
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
# Step 4: Send JSON to gitops-status-server
|
|
# ─────────────────────────────────────────────────────────────────────────────────
|
|
echo "Step 4/4: Sending status to gitops-status-server..."
|
|
echo " URL: $GITOPS_STATUS_SERVER_URL/api/status"
|
|
echo " Method: POST"
|
|
echo ""
|
|
|
|
# Create temporary files for response
|
|
RESPONSE_BODY=$(mktemp)
|
|
trap "rm -f $RESPONSE_BODY" EXIT
|
|
|
|
# POST the JSON to the gitops-status-server API with full error reporting
|
|
# Capture both response code and body for debugging
|
|
HTTP_CODE=$(curl -s -w "%{http_code}" \
|
|
-X POST \
|
|
-H "Content-Type: application/json" \
|
|
-d "$STATUS_JSON" \
|
|
"$GITOPS_STATUS_SERVER_URL/api/status" \
|
|
-o "$RESPONSE_BODY" 2>&1 || true)
|
|
|
|
# Extract HTTP code (last 3 digits)
|
|
HTTP_CODE="${HTTP_CODE: -3}"
|
|
|
|
echo " Response: HTTP $HTTP_CODE"
|
|
|
|
# Show response body for debugging (especially on error)
|
|
if [ -s "$RESPONSE_BODY" ]; then
|
|
echo " Response Body:"
|
|
sed 's/^/ /' "$RESPONSE_BODY"
|
|
fi
|
|
echo ""
|
|
|
|
if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
echo " ✓ Status update successful (HTTP $HTTP_CODE)"
|
|
echo " JSON has been sent to gitops-status-server"
|
|
echo ""
|
|
|
|
# Verify the JSON was actually received and stored
|
|
echo " Verifying JSON storage on gitops-status-server..."
|
|
sleep 1 # Brief delay to ensure server processed the POST
|
|
|
|
VERIFY_JSON=$(curl -s "$GITOPS_STATUS_SERVER_URL/status.json" 2>&1 || true)
|
|
if echo "$VERIFY_JSON" | grep -q "\"repo\".*\"$REPO_NAME\""; then
|
|
echo " ✓ Verified: Latest JSON stored correctly on server"
|
|
echo ""
|
|
echo " Grafana Infinity datasource will now read the updated JSON from:"
|
|
echo " $GITOPS_STATUS_SERVER_URL/status.json"
|
|
else
|
|
echo " ⚠ Warning: Could not verify JSON storage"
|
|
echo " Response from /status.json:"
|
|
echo " $VERIFY_JSON" | sed 's/^/ /'
|
|
fi
|
|
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
exit 0
|
|
else
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
echo " ✗ ERROR: Status update failed with HTTP $HTTP_CODE"
|
|
echo " Debugging Information:"
|
|
echo " - Server URL: $GITOPS_STATUS_SERVER_URL"
|
|
echo " - Endpoint: /api/status"
|
|
echo " - Check gitops-status-server connectivity (ping/nslookup)"
|
|
echo " - Verify service port and internal port mapping"
|
|
echo " - Ensure /api/status endpoint accepts POST requests"
|
|
echo "═══════════════════════════════════════════════════════════════════════════════"
|
|
exit 1
|
|
fi
|