380eaf175a
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
CRITICAL FIXES: 1. Fix API URL port: 5000 → 80 (.woodpecker.yml) - update-gitops-status step was POSTing to wrong port - gitops-status-server Service exposes port 80, not 5000 - This caused silent POST failures that weren't detected 2. Initialize missing_on_server variable (drift-check.yml) - Variable was only set inside block scope - Could remain undefined if block failed or didn't execute - Now initialized to empty list before block runs - Prevents undefined variable errors in container environment 3. Fix drift detection logic (drift-check.yml) - Changed from: drift_detected uses extra_files_on_server flag - Changed to: drift_detected directly checks missing_on_server length - Adds safety with | default([]) filter - Prevents false positives when extra_files_on_server wasn't set properly ROOT CAUSE: The combination of port 5000, uninitialized variables, and flag-based logic caused the playbook to report OUT_OF_SYNC without listing changed files (drift_count=0, files=[]). After deployment, server config matches Git, so drift_detected should be false and playbook should exit 0 with SYNCED status. Now correctly reports SYNCED after successful deploy.
205 lines
7.9 KiB
YAML
205 lines
7.9 KiB
YAML
# =============================================================================
|
||
# Single pipeline – all event types handled via step-level when conditions:
|
||
#
|
||
# pull_request → syntax-check, validate
|
||
# ONLY validates config correctness.
|
||
# Does NOT compare to the live server – a PR branch is
|
||
# expected to differ from the server (not yet deployed),
|
||
# so a drift-check here would always be OUT_OF_SYNC by
|
||
# design and is meaningless as a failure signal.
|
||
#
|
||
# push (master) → syntax-check, validate, deploy, update-gitops-status
|
||
# Deploys to the server, then verifies sync and sends
|
||
# JSON status snapshot to gitops-status-server for Grafana.
|
||
#
|
||
# cron → gitops_sync_check (read-only drift check, no deploy)
|
||
# Continuously verifies that the live server still matches
|
||
# Git even when no push has happened. Detects manual edits
|
||
# made directly on the server. Sends JSON status with
|
||
# detailed file-level drift information to gitops-status-server.
|
||
#
|
||
# NOTE: Woodpecker does not support multiple YAML documents (---) in one file.
|
||
# All pipelines must live in a single document with step-level filtering.
|
||
# =============================================================================
|
||
when:
|
||
- event: pull_request
|
||
- event: push
|
||
branch: master
|
||
- event: cron
|
||
|
||
steps:
|
||
# ---------------------------------------------------------------------------
|
||
# syntax-check: Lint all playbooks – runs on pull_request and push
|
||
# ---------------------------------------------------------------------------
|
||
syntax-check:
|
||
image: alpine/ansible:latest
|
||
environment:
|
||
ANSIBLE_CONFIG: ansible.cfg
|
||
commands:
|
||
- ansible-playbook -i ansible/inventory/hosts.yml --syntax-check ansible/playbooks/*.yml
|
||
when:
|
||
- event: pull_request
|
||
- event: push
|
||
branch: master
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# validate: Validate server state – runs on pull_request and push
|
||
# ---------------------------------------------------------------------------
|
||
validate:
|
||
image: alpine/ansible:latest
|
||
depends_on: [syntax-check]
|
||
environment:
|
||
ANSIBLE_CONFIG: ansible.cfg
|
||
SSH_PRIVATE_KEY:
|
||
from_secret: SSH_PRIVATE_KEY
|
||
commands:
|
||
- mkdir -p ~/.ssh
|
||
- printf '%s\n' "$${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
||
- chmod 600 ~/.ssh/id_rsa
|
||
- ansible-playbook -i ansible/inventory/hosts.yml ansible/playbooks/validate.yml
|
||
when:
|
||
- event: pull_request
|
||
- event: push
|
||
branch: master
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# deploy: Apply Git configuration to server – runs on push to master only
|
||
# ---------------------------------------------------------------------------
|
||
deploy:
|
||
image: alpine/ansible:latest
|
||
depends_on: [syntax-check, validate]
|
||
environment:
|
||
ANSIBLE_CONFIG: ansible.cfg
|
||
SSH_PRIVATE_KEY:
|
||
from_secret: SSH_PRIVATE_KEY
|
||
commands:
|
||
- mkdir -p ~/.ssh
|
||
- printf '%s\n' "$${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
||
- chmod 600 ~/.ssh/id_rsa
|
||
- ansible-playbook -i ansible/inventory/hosts.yml ansible/playbooks/apply.yml
|
||
when:
|
||
branch: master
|
||
event: push
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# update-gitops-status: Post-deploy sync check + JSON status update
|
||
# Runs on push to master only, after deploy succeeds.
|
||
# Generates structured JSON with sync status, drift count, and changed files.
|
||
# Sends JSON to gitops-status-server for Grafana visualization.
|
||
# ---------------------------------------------------------------------------
|
||
update-gitops-status:
|
||
image: alpine/ansible:latest
|
||
depends_on: [deploy]
|
||
environment:
|
||
ANSIBLE_CONFIG: ansible.cfg
|
||
SSH_PRIVATE_KEY:
|
||
from_secret: SSH_PRIVATE_KEY
|
||
GITOPS_STATUS_SERVER_URL: http://gitops-status-server.observability-stack.svc.cluster.local:80
|
||
REPO_NAME: rsyslog
|
||
SERVER_NAME: rsyslog-lab
|
||
# Optimize Ansible for container environment
|
||
ANSIBLE_HOST_KEY_CHECKING: "False"
|
||
ANSIBLE_FORCE_COLOR: "False"
|
||
ANSIBLE_RETRY_FILES_ENABLED: "False"
|
||
ANSIBLE_UNSAFE_WRITES: "True"
|
||
ANSIBLE_FORKS: "1"
|
||
commands:
|
||
- |
|
||
# Increase file descriptor limit for Ansible (max safe value)
|
||
ulimit -n 65536
|
||
|
||
# Install dependencies: curl for HTTP requests, jq for JSON formatting
|
||
apk add --no-cache curl jq > /dev/null 2>&1
|
||
|
||
# Setup SSH key for Ansible
|
||
mkdir -p ~/.ssh
|
||
printf '%s\n' "$${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
||
chmod 600 ~/.ssh/id_rsa
|
||
|
||
echo "==> Running post-deploy GitOps status check..."
|
||
|
||
# Make script executable and run it
|
||
chmod +x update-gitops-status.sh
|
||
./update-gitops-status.sh
|
||
|
||
echo "==> JSON status update complete. Pipeline always succeeds."
|
||
when:
|
||
branch: master
|
||
event: push
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# gitops_sync_check: ArgoCD-style cron drift check – read-only, no deploy
|
||
# Detects manual changes made directly on the server between pushes.
|
||
# Generates structured JSON with sync status, drift count, and changed files.
|
||
# Sends JSON to gitops-status-server for continuous GitOps monitoring.
|
||
# Pipeline marked FAILED when drift found so it is visible in the UI.
|
||
#
|
||
# ─── Woodpecker Cron UI settings ──────────────────────────────────────────
|
||
# Name: gitops_sync_check
|
||
# Branch: master
|
||
# Schedule: */2 * * * *
|
||
# ---------------------------------------------------------------------------
|
||
gitops_sync_check:
|
||
image: alpine/ansible:latest
|
||
environment:
|
||
ANSIBLE_CONFIG: ansible.cfg
|
||
SSH_PRIVATE_KEY:
|
||
from_secret: SSH_PRIVATE_KEY
|
||
GITOPS_STATUS_SERVER_URL: http://gitops-status-server.observability-stack.svc.cluster.local:80
|
||
REPO_NAME: rsyslog
|
||
SERVER_NAME: rsyslog-lab
|
||
# Optimize Ansible for container environment
|
||
ANSIBLE_HOST_KEY_CHECKING: "False"
|
||
ANSIBLE_FORCE_COLOR: "False"
|
||
ANSIBLE_RETRY_FILES_ENABLED: "False"
|
||
ANSIBLE_UNSAFE_WRITES: "True"
|
||
ANSIBLE_FORKS: "1"
|
||
commands:
|
||
- |
|
||
# Increase file descriptor limit for Ansible (max safe value)
|
||
ulimit -n 65536
|
||
|
||
# Install dependencies: curl for HTTP requests, jq for JSON formatting
|
||
apk add --no-cache curl jq bash > /dev/null 2>&1
|
||
|
||
# Setup SSH key for Ansible
|
||
mkdir -p ~/.ssh
|
||
printf '%s\n' "$${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
||
chmod 600 ~/.ssh/id_rsa
|
||
|
||
echo "==> [cron] Running continuous GitOps drift check..."
|
||
|
||
# Make script executable and run it
|
||
chmod +x update-gitops-status.sh
|
||
|
||
# Capture exit code to determine if drift was detected
|
||
set +e
|
||
./update-gitops-status.sh
|
||
SCRIPT_RC=$?
|
||
set -e
|
||
|
||
if [ "$SCRIPT_RC" -ne 0 ]; then
|
||
echo "==> ERROR: Status update failed"
|
||
exit 1
|
||
fi
|
||
|
||
# Check sync status to determine pipeline result
|
||
# Read the generated JSON or re-run drift check
|
||
echo "==> Verifying drift status for pipeline result..."
|
||
set +e
|
||
ANSIBLE_FORCE_COLOR=false \
|
||
ansible-playbook -i ansible/inventory/hosts.yml ansible/playbooks/drift-check.yml > /dev/null 2>&1
|
||
DRIFT_RC=$?
|
||
set -e
|
||
|
||
if [ "$DRIFT_RC" -eq 0 ]; then
|
||
echo "==> Pipeline SUCCESS: Server is SYNCED with Git"
|
||
exit 0
|
||
else
|
||
echo "==> Pipeline FAILED: OUT OF SYNC - manual drift detected on server"
|
||
echo " Check gitops-status-server for detailed file-level drift information"
|
||
exit 1
|
||
fi
|
||
when:
|
||
event: cron
|