From c83725a027a6d4dea95966a601b751ea176113c9 Mon Sep 17 00:00:00 2001 From: dvirlabs <114520947+dvirlabs@users.noreply.github.com> Date: Wed, 20 May 2026 16:07:38 +0300 Subject: [PATCH] Make this repo generic --- CHANGES_OVERVIEW.md | 377 ------------- DEBUGGING_GITOPS_STATUS.md | 187 ------- DELIVERABLES.md | 438 --------------- DEPLOYMENT_CHECKLIST.md | 310 ---------- GITOPS_STATUS_API_REFERENCE.md | 326 ----------- GITOPS_STATUS_FIX.md | 260 --------- GITOPS_STATUS_INTEGRATION.md | 117 ---- GITOPS_STATUS_SERVER_INTEGRATION.md | 493 ---------------- MIGRATION_SUMMARY.md | 168 ------ QUICK_REFERENCE.md | 227 -------- README.md | 528 +++++++++++------- README_GITOPS_STATUS.md | 462 --------------- REFACTOR_SUMMARY.md | 382 ------------- ansible/inventory/group_vars/all.yml | 12 +- ansible/inventory/hosts.yml | 10 +- ansible/playbooks/apply.yml | 162 +----- ansible/playbooks/drift-check.yml | 177 ++---- ansible/playbooks/validate.yml | 73 ++- apply.sh | 38 -- drift-check.sh | 55 -- files/dvir.txt | 3 + files/rsyslog.conf | 60 -- files/rsyslog.d/30-lab.conf | 2 - gitops-status-server/.helmignore | 23 - gitops-status-server/Chart.yaml | 14 - gitops-status-server/README.md | 478 ---------------- gitops-status-server/templates/_helpers.tpl | 63 --- gitops-status-server/templates/api-app.yaml | 142 ----- gitops-status-server/templates/configmap.yaml | 22 - .../templates/deployment.yaml | 190 ------- .../templates/nginx-config.yaml | 96 ---- gitops-status-server/templates/service.yaml | 24 - .../templates/serviceaccount.yaml | 15 - gitops-status-server/values.yaml | 103 ---- update-gitops-status.sh | 414 -------------- 35 files changed, 436 insertions(+), 6015 deletions(-) delete mode 100644 CHANGES_OVERVIEW.md delete mode 100644 DEBUGGING_GITOPS_STATUS.md delete mode 100644 DELIVERABLES.md delete mode 100644 DEPLOYMENT_CHECKLIST.md delete mode 100644 GITOPS_STATUS_API_REFERENCE.md delete mode 100644 GITOPS_STATUS_FIX.md delete mode 100644 GITOPS_STATUS_INTEGRATION.md delete mode 100644 GITOPS_STATUS_SERVER_INTEGRATION.md delete mode 100644 MIGRATION_SUMMARY.md delete mode 100644 QUICK_REFERENCE.md delete mode 100644 README_GITOPS_STATUS.md delete mode 100644 REFACTOR_SUMMARY.md delete mode 100755 apply.sh delete mode 100755 drift-check.sh create mode 100644 files/dvir.txt delete mode 100644 files/rsyslog.conf delete mode 100644 files/rsyslog.d/30-lab.conf delete mode 100644 gitops-status-server/.helmignore delete mode 100644 gitops-status-server/Chart.yaml delete mode 100644 gitops-status-server/README.md delete mode 100644 gitops-status-server/templates/_helpers.tpl delete mode 100644 gitops-status-server/templates/api-app.yaml delete mode 100644 gitops-status-server/templates/configmap.yaml delete mode 100644 gitops-status-server/templates/deployment.yaml delete mode 100644 gitops-status-server/templates/nginx-config.yaml delete mode 100644 gitops-status-server/templates/service.yaml delete mode 100644 gitops-status-server/templates/serviceaccount.yaml delete mode 100644 gitops-status-server/values.yaml delete mode 100644 update-gitops-status.sh diff --git a/CHANGES_OVERVIEW.md b/CHANGES_OVERVIEW.md deleted file mode 100644 index 7dcc414..0000000 --- a/CHANGES_OVERVIEW.md +++ /dev/null @@ -1,377 +0,0 @@ -# Change Summary: Pushgateway → gitops-status-server - -## Files Modified: 2 | Files Created: 2 - -### Files Changed -``` -MODIFIED .woodpecker.yml - ├─ Replaced Pushgateway URL with gitops-status-server URL - ├─ Removed metric push commands - ├─ Changed to call update-gitops-status.sh script - ├─ Both update-gitops-status and gitops_sync_check steps updated - └─ Enhanced comments explaining new flow - -MODIFIED ansible/playbooks/drift-check.yml - ├─ Added file collection logic (drifted_files fact) - ├─ Added debug output markers (DRIFTED_FILES=...) - ├─ Added status markers (SYNC_STATUS=...) - ├─ Original drift detection logic unchanged - └─ Fully backward compatible -``` - -### Files Created -``` -NEW update-gitops-status.sh - ├─ Step 1: Run drift-check.yml - ├─ Step 2: Parse output for changed files - ├─ Step 3: Build JSON payload - ├─ Step 4: POST to gitops-status-server/api/status - ├─ 4-step flow with clear logging - └─ Environment-configurable (GITOPS_STATUS_SERVER_URL, REPO_NAME, SERVER_NAME) - -NEW Documentation files (4 total) - ├─ GITOPS_STATUS_SERVER_INTEGRATION.md (comprehensive guide) - ├─ QUICK_REFERENCE.md (quick start & troubleshooting) - ├─ REFACTOR_SUMMARY.md (before/after details) - └─ README_GITOPS_STATUS.md (this overview) -``` - ---- - -## Architecture Change - -### BEFORE: Pushgateway-based -``` -Drift Check → Exit Code (0/1) → Pushgateway → Prometheus → Grafana - ↓ ↓ ↓ ↓ -Binary Lost file info Scraping Only 0/1 shown -``` - -### AFTER: gitops-status-server -``` -Drift Check → JSON Generation → gitops-status-server → Grafana - ↓ ↓ ↓ ↓ -Ansible Structured REST API Rich metadata -Output Parsing (POST) File names - ↓ + timestamps - DRIFTED_FILES=... + counts -``` - ---- - -## Step-by-Step Flow - -``` -┌─── CRON or POST-DEPLOY TRIGGER ────────────────────────────────────┐ -│ │ -│ Woodpecker Step Executes: │ -│ 1. chmod +x update-gitops-status.sh │ -│ 2. ./update-gitops-status.sh │ -│ │ -│ ↓ │ -│ -│ STEP 1: Run Drift Detection │ -│ ───────────────────────────────── │ -│ ansible-playbook drift-check.yml │ -│ │ -│ Tasks: │ -│ ├─ Copy rsyslog.conf (check mode) │ -│ ├─ Copy rsyslog.d/* (check mode) │ -│ ├─ Find missing files │ -│ └─ Output: DRIFTED_FILES=file1,file2,file3 │ -│ SYNC_STATUS=OUT_OF_SYNC │ -│ │ -│ Exit Code: 0 (SYNCED) or non-zero (OUT_OF_SYNC) │ -│ │ -│ ↓ │ -│ -│ STEP 2: Parse Output │ -│ ────────────────────── │ -│ Extract from Ansible debug output: │ -│ ├─ DRIFTED_FILES=/etc/rsyslog.conf,/etc/rsyslog.d/30-lab.conf │ -│ ├─ Split on commas: ['/etc/rsyslog.conf', '/etc/rsyslog.d/...'] │ -│ ├─ Convert paths: [rsyslog.conf, rsyslog.d/30-lab.conf] │ -│ └─ Count: 2 changed files │ -│ │ -│ ↓ │ -│ -│ STEP 3: Generate JSON │ -│ ────────────────────── │ -│ { │ -│ "repo": "rsyslog", │ -│ "server": "rsyslog-lab", │ -│ "sync_status": "OUT_OF_SYNC", │ -│ "drift_count": 2, │ -│ "files": [ │ -│ { "name": "rsyslog.conf" }, │ -│ { "name": "rsyslog.d/30-lab.conf" } │ -│ ], │ -│ "last_check": "2026-04-21T10:30:00Z" │ -│ } │ -│ │ -│ ↓ │ -│ -│ STEP 4: POST to gitops-status-server │ -│ ───────────────────────────────────── │ -│ curl -X POST │ -│ -H "Content-Type: application/json" │ -│ -d "$JSON" │ -│ http://gitops-status-server.observability-stack.svc...:80/api/status -│ │ -│ Response: HTTP 200 OK ✓ │ -│ │ -└─────────────────────────────────────────────────────────────────────┘ - -┌─── GITOPS-STATUS-SERVER ───────────────────────────────────────────┐ -│ │ -│ Receives POST /api/status │ -│ Updates internal state │ -│ Serves /status.json │ -│ (Available to Grafana) │ -│ │ -└────────────────────────────────────────────────────────────────────┘ - -┌─── GRAFANA ────────────────────────────────────────────────────────┐ -│ │ -│ Infinity Datasource │ -│ ├─ Polls /status.json (configurable interval) │ -│ ├─ Parses JSON response │ -│ └─ Updates dashboard panels: │ -│ ├─ Sync Status: 🔴 OUT_OF_SYNC │ -│ ├─ Drift Count: 2 │ -│ ├─ Files: │ -│ │ - rsyslog.conf │ -│ │ - rsyslog.d/30-lab.conf │ -│ └─ Last Check: 2026-04-21 10:30 UTC │ -│ │ -└────────────────────────────────────────────────────────────────────┘ -``` - ---- - -## Integration Points - -### Pull Request Event -``` -push: branches: PR - ├─ syntax-check - ├─ validate - └─ (NO gitops-status update, correct by design) -``` - -### Master Push Event -``` -push: branch: master - ├─ syntax-check - ├─ validate - ├─ deploy - └─ update-gitops-status ← NEW: calls update-gitops-status.sh -``` - -### Scheduled Cron Event -``` -cron: */2 * * * * - └─ gitops_sync_check ← calls update-gitops-status.sh - (every 2 minutes) -``` - ---- - -## JSON Payload Examples - -### Case 1: Everything Synced -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "SYNCED", - "drift_count": 0, - "files": [], - "last_check": "2026-04-21T10:30:00Z" -} -``` - -### Case 2: Manual Edit Detected -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 1, - "files": [ - { "name": "rsyslog.conf" } - ], - "last_check": "2026-04-21T10:32:00Z" -} -``` - -### Case 3: Multiple Files Changed -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 3, - "files": [ - { "name": "rsyslog.conf" }, - { "name": "rsyslog.d/30-lab.conf" }, - { "name": "rsyslog.d/31-remote.conf" } - ], - "last_check": "2026-04-21T10:34:00Z" -} -``` - ---- - -## Deployment Checklist - -- [ ] Review changes: `git diff origin/master` -- [ ] Commit: `git add . && git commit -m "refactor: gitops-status-server integration"` -- [ ] Push: `git push` -- [ ] Verify pipeline runs (check `update-gitops-status` step) -- [ ] Create Woodpecker cron job: `gitops_sync_check` at `*/2 * * * *` -- [ ] Test cron execution (wait 2 minutes) -- [ ] Verify gitops-status-server receives JSON -- [ ] Check Grafana dashboard displays status -- [ ] Test manual file edit (detect within 2 minutes) -- [ ] Monitor for 24 hours - ---- - -## Key Improvements - -| Aspect | Before (Pushgateway) | After (gitops-status-server) | -|--------|---------------------|-------------------------------| -| **Data Format** | Single metric (0/1) | Rich JSON with metadata | -| **File Details** | None | List of changed files | -| **Infrastructure** | Prometheus + Pushgateway | Single service call | -| **Query Language** | PromQL | Simple JSON API | -| **Grafana Plugin** | Prometheus DS | Infinity DS (native) | -| **Audit Trail** | Basic | Structured snapshots | -| **Setup Complexity** | High | Low | -| **Multi-repo Support** | Difficult | Built-in | - ---- - -## Testing Scenarios - -### Scenario 1: Server is synced -``` -Cron triggers → drift-check runs → No changes found -→ SYNC_STATUS=SYNCED → JSON sent → Grafana shows ✓ SYNCED (0 drift) -``` - -### Scenario 2: Manual file edit -``` -Someone edits /etc/rsyslog.conf directly on server -→ Cron triggers (≤2 min) → drift-check detects change -→ DRIFTED_FILES=/etc/rsyslog.conf → JSON sent -→ Grafana shows ✗ OUT_OF_SYNC (1 drift: rsyslog.conf) -``` - -### Scenario 3: Deploy succeeds -``` -Push to master → Pipeline runs → deploy completes -→ update-gitops-status runs → drift-check shows no drift -→ JSON sent: SYNC_STATUS=SYNCED → Grafana updated -``` - -### Scenario 4: Manual drift recovery -``` -Admin runs ansible apply to fix drift -→ Next cron (≤2 min) → drift-check shows SYNCED -→ JSON sent: SYNC_STATUS=SYNCED -→ Grafana updates to show ✓ SYNCED (0 drift) -``` - ---- - -## Code Quality - -- ✅ Comments explain flow -- ✅ Error handling included -- ✅ Exit codes meaningful -- ✅ No hardcoded paths (environment vars) -- ✅ Ansible playbook backward compatible -- ✅ JSON properly escaped -- ✅ ISO 8601 timestamps - ---- - -## Security - -- ✅ SSH credentials in Woodpecker secrets -- ✅ JSON exposes only metadata (no config contents) -- ✅ No sensitive data in logs -- ✅ gitops-status-server internal only (ClusterIP) -- ✅ Network-isolated (Kubernetes) - ---- - -## Performance - -- **Frequency:** Every 2 minutes (cron) + immediate post-deploy -- **Duration:** ~30 seconds per run (drift-check + JSON POST) -- **Data size:** ~500 bytes per JSON update -- **Network:** Internal only (ClusterIP service) -- **Load:** Minimal (one HTTP POST per cycle) - ---- - -## Success Criteria - -✅ Cron job runs every 2 minutes -✅ JSON posted to gitops-status-server with HTTP 200 -✅ gitops-status-server receives and stores JSON -✅ Grafana dashboard displays sync status -✅ Changed files listed in Grafana -✅ Manual edits detected within 2 minutes -✅ Post-deploy status updates immediately -✅ No errors in pipeline logs - ---- - -## Rollback (if needed) - -```bash -# Revert to previous version -git revert HEAD --no-edit -git push - -# Remove cron job from Woodpecker UI -``` - ---- - -## Documentation Generated - -1. **GITOPS_STATUS_SERVER_INTEGRATION.md** – 400+ lines, comprehensive guide -2. **QUICK_REFERENCE.md** – Quick start, testing, troubleshooting -3. **REFACTOR_SUMMARY.md** – Before/after code comparison -4. **README_GITOPS_STATUS.md** – Overview and quick start -5. **This file** – Visual summary of changes - ---- - -## Next Actions - -1. **Review** all changes -2. **Test** locally with `./update-gitops-status.sh` -3. **Commit** and **push** to Git -4. **Monitor** Woodpecker pipeline run -5. **Configure** Woodpecker cron job -6. **Verify** gitops-status-server receives JSON -7. **Check** Grafana dashboard works -8. **Monitor** for 24 hours - ---- - -## Summary - -**Changed:** Pushgateway-based metrics → gitops-status-server JSON API -**Result:** Richer metadata, simpler architecture, better audit trail -**Impact:** No breaking changes, backward compatible -**Time to deploy:** ~30 minutes (including testing) -**Monitoring:** Every 2 minutes + post-deploy - -✅ **Ready for production deployment** diff --git a/DEBUGGING_GITOPS_STATUS.md b/DEBUGGING_GITOPS_STATUS.md deleted file mode 100644 index 3492327..0000000 --- a/DEBUGGING_GITOPS_STATUS.md +++ /dev/null @@ -1,187 +0,0 @@ -# Debugging GitOps Status Issues - -This guide helps troubleshoot issues with the GitOps status reporting system. - -## Common Issue: Status shows OUT_OF_SYNC after deployment - -### Symptoms -- You pushed changes to the repo -- The deploy step succeeded -- But `update-gitops-status` shows OUT_OF_SYNC -- Changed files are not displayed - -### Root Causes - -#### 1. **Deployment didn't actually succeed** -The deploy step might have failed silently or the configuration wasn't applied correctly. - -**How to check:** -- Look at the deploy step logs in Woodpecker -- SSH to the server and verify files match Git: - ```bash - diff /etc/rsyslog.conf files/rsyslog.conf - diff -r /etc/rsyslog.d/ files/rsyslog.d/ - ``` - -#### 2. **Parsing issue with DRIFTED_FILES output** -The script might not be correctly extracting the file list from Ansible's output. - -**How to debug:** -Run the status update script locally with debug mode: -```bash -export KEEP_PLAYBOOK_LOG=true -./update-gitops-status.sh -``` - -This will save the playbook output to `drift-check-output.log`. Check: -- Does the log contain `DRIFTED_FILES=` line? -- What does the line look like exactly? -- Are there ANSI color codes interfering? - -Look for these debug lines in the output: -``` -DEBUG: Searching for DRIFTED_FILES in playbook output... -DEBUG: Found DRIFTED_FILES pattern -DEBUG: Raw line: ... -DEBUG: Extracted value: '...' -``` - -#### 3. **Too many open files error** -If you see "failed to create fsnotify watcher: too many open files": - -**Fixed in latest version:** -- `.woodpecker.yml` now sets `ANSIBLE_CALLBACKS_ENABLED=""` and `ANSIBLE_GATHERING=explicit` -- `update-gitops-status.sh` uses `ANSIBLE_CALLBACKS_ENABLED=""` when running playbooks -- These settings prevent Ansible from exhausting inotify watches - -**If issue persists:** -- The container might need privileged mode to adjust kernel parameters -- Or reduce Ansible parallelism in inventory settings - -#### 4. **Ansible output format changed** -If Ansible version changed, the debug output format might be different. - -**How to fix:** -Check `drift-check-output.log` and adjust the parsing in `update-gitops-status.sh`: -```bash -# Current parsing (line ~110 in update-gitops-status.sh): -DRIFTED_FILES_STR=$(echo "$DRIFTED_FILES_STR" | sed 's/.*DRIFTED_FILES=//' | sed 's/\x1b\[[0-9;]*m//g' | sed 's/".*$//' | xargs) -``` - -You might need to adjust the `sed` commands based on the actual format. - -## Testing the Fix - -### 1. Test locally -```bash -# Set up SSH key -export SSH_PRIVATE_KEY="$(cat ~/.ssh/id_rsa)" - -# Run the script with debug output -export KEEP_PLAYBOOK_LOG=true -./update-gitops-status.sh - -# Check the log -cat drift-check-output.log | grep -A 2 "DRIFTED_FILES=" -``` - -### 2. Test in Woodpecker -Push a small change and monitor the `update-gitops-status` step: -```bash -# Make a small comment change -echo "# Test change $(date)" >> files/rsyslog.conf - -# Commit and push -git add files/rsyslog.conf -git commit -m "test: verify gitops status detection" -git push - -# Watch the pipeline in Woodpecker UI -# The update-gitops-status step should: -# 1. Run deploy (apply.yml) -# 2. Run drift-check immediately after -# 3. Show SYNCED (because deploy just ran) -# 4. Show no drifted files -``` - -### 3. Test drift detection (manual change on server) -```bash -# SSH to the server -ssh rsyslog-lab - -# Make a manual change -echo "# Manual change" >> /etc/rsyslog.conf - -# Wait for the cron job to run (every 2 minutes) -# Or manually trigger it in Woodpecker - -# The status should now show: -# - Status: OUT_OF_SYNC -# - Files: rsyslog.conf -``` - -## Expected Behavior - -### After successful deployment (push to master) -``` -Step 2/4: Analyzing drift detection results... - ✓ Status: SYNCED - server configuration matches Git - Total drift count: 0 - -Step 3/4: Building JSON payload... - Generated JSON: - { - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "SYNCED", - "drift_count": 0, - "files": [], - "last_check": "2026-04-22T14:30:00Z" - } -``` - -### When drift is detected (cron job or manual server change) -``` -Step 2/4: Analyzing drift detection results... - ✗ Status: OUT OF SYNC - configuration drift detected - - Drift detected in: rsyslog.conf - Total drift count: 1 - -Step 3/4: Building JSON payload... - Generated JSON: - { - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 1, - "files": [ - {"name": "rsyslog.conf"} - ], - "last_check": "2026-04-22T14:32:00Z" - } -``` - -## Quick Reference - -### Enable debug mode -```bash -export KEEP_PLAYBOOK_LOG=true -./update-gitops-status.sh -cat drift-check-output.log -``` - -### Manually run drift-check -```bash -ansible-playbook -i ansible/inventory/hosts.yml ansible/playbooks/drift-check.yml -``` - -### Manually run deployment -```bash -ansible-playbook -i ansible/inventory/hosts.yml ansible/playbooks/apply.yml -``` - -### Check server state -```bash -ssh rsyslog-lab "md5sum /etc/rsyslog.conf /etc/rsyslog.d/*.conf" -md5sum files/rsyslog.conf files/rsyslog.d/*.conf -``` diff --git a/DELIVERABLES.md b/DELIVERABLES.md deleted file mode 100644 index b1c5e53..0000000 --- a/DELIVERABLES.md +++ /dev/null @@ -1,438 +0,0 @@ -# Deliverables: GitOps Status Server Integration - -## ✅ Implementation Complete - ---- - -## What You Requested - -1. ✅ Stop using Pushgateway for GitOps sync-status monitoring -2. ✅ Integrate with gitops-status-server (Kubernetes internal service) -3. ✅ Generate JSON status payload with changed files -4. ✅ Post JSON to gitops-status-server/api/status -5. ✅ Keep existing deploy/apply logic intact -6. ✅ Keep PR validation unchanged -7. ✅ Maintain cron-based drift detection -8. ✅ Add clear comments explaining the flow -9. ✅ Production-ready implementation - -## What You Received - -### Core Implementation Files - -**1. Modified: `.woodpecker.yml`** -- Location: Repository root -- Changes: Both `update-gitops-status` (post-deploy) and `gitops_sync_check` (cron) steps updated -- Configuration: - - `GITOPS_STATUS_SERVER_URL` environment variable - - `REPO_NAME` and `SERVER_NAME` parameters -- Behavior: Calls `update-gitops-status.sh` script instead of Pushgateway -- Status: Ready to use - -**2. Modified: `ansible/playbooks/drift-check.yml`** -- Location: ansible/playbooks/ -- Changes: Added structured file output -- Output markers: - - `DRIFTED_FILES=file1,file2,file3` - - `SYNC_STATUS=SYNCED|OUT_OF_SYNC` -- Compatibility: Fully backward compatible (drift detection unchanged) -- Status: Ready to use - -**3. Created: `update-gitops-status.sh`** -- Location: Repository root -- Purpose: Orchestrate the entire status update flow -- 4-step process: - 1. Run drift-check.yml - 2. Parse output for changed files - 3. Generate JSON payload - 4. POST to gitops-status-server -- Configuration: Environment variables (URL, repo name, server name) -- Exit codes: 0 (success) or 1 (failure) -- Status: Fully functional, ready to use - -### Documentation Files - -**4. Created: `GITOPS_STATUS_SERVER_INTEGRATION.md`** -- 600+ lines comprehensive documentation -- Includes: - - Full architecture diagram - - Component descriptions - - Data flow examples - - Configuration details - - Troubleshooting guide - - Security considerations -- Status: Complete reference guide - -**5. Created: `QUICK_REFERENCE.md`** -- Quick start guide (5 steps) -- Testing procedures -- Troubleshooting checklist -- Environment variables -- Rollback instructions -- Status: Quick implementation guide - -**6. Created: `REFACTOR_SUMMARY.md`** -- Before/after architecture comparison -- Code changes with explanations -- Integration points -- Migration steps -- Testing checklist -- Status: Change documentation - -**7. Created: `README_GITOPS_STATUS.md`** -- Overview document -- How it works section -- Files changed explained -- Testing guide -- Examples -- Status: Main entry point - -**8. Created: `CHANGES_OVERVIEW.md`** -- Visual summary of all changes -- Flow diagrams -- JSON examples -- Deployment checklist -- Success criteria -- Status: Visual reference - ---- - -## Implementation Details - -### Workflow Stages - -**Stage 1: Cron Detection (Every 2 minutes)** -``` -Cron timer triggers -→ update-gitops-status.sh runs -→ drift-check.yml compares files -→ Changed files extracted -→ JSON generated -→ POST to gitops-status-server -→ Grafana reads /status.json -→ Dashboard updated -``` - -**Stage 2: Post-Deployment (Immediate after push)** -``` -Push to master -→ Pipeline: syntax-check → validate → deploy → update-gitops-status -→ Deployment completes -→ Drift check verifies sync -→ JSON sent to gitops-status-server -→ Grafana updates immediately -``` - -### JSON Payload Structure - -**Template (Synced):** -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "SYNCED", - "drift_count": 0, - "files": [], - "last_check": "2026-04-21T10:30:00Z" -} -``` - -**Template (Out of Sync):** -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 2, - "files": [ - { "name": "rsyslog.conf" }, - { "name": "rsyslog.d/30-lab.conf" } - ], - "last_check": "2026-04-21T10:30:00Z" -} -``` - -### Environment Variables - -All set in `.woodpecker.yml`: -```yaml -GITOPS_STATUS_SERVER_URL: http://gitops-status-server.observability-stack.svc.cluster.local:80 -REPO_NAME: rsyslog -SERVER_NAME: rsyslog-lab -SSH_PRIVATE_KEY: from_secret: SSH_PRIVATE_KEY -ANSIBLE_CONFIG: ansible.cfg -``` - -### Exit Codes - -- **0:** Success (JSON posted) -- **1:** Failure (playbook error, network error, etc.) - ---- - -## Ready to Deploy - -### Prerequisites Met -- ✅ gitops-status-server exists in Kubernetes -- ✅ Exposed via ClusterIP at observability-stack namespace -- ✅ API endpoint: POST /api/status -- ✅ Grafana Infinity datasource configured -- ✅ Woodpecker CI/CD running - -### Files Ready -- ✅ `.woodpecker.yml` – Updated with new flow -- ✅ `ansible/playbooks/drift-check.yml` – Enhanced with output -- ✅ `update-gitops-status.sh` – Created and ready -- ✅ Documentation – 5 comprehensive guides - -### Next Steps -1. **Review** the changes (run `git diff`) -2. **Test** locally if possible (run script) -3. **Commit** changes to Git -4. **Push** to trigger pipeline -5. **Create** Woodpecker cron job -6. **Monitor** first execution -7. **Verify** gitops-status-server receives JSON -8. **Check** Grafana dashboard - ---- - -## File Locations - -``` -rsyslog/ -├── .woodpecker.yml ← MODIFIED -├── ansible/ -│ └── playbooks/ -│ └── drift-check.yml ← MODIFIED -├── update-gitops-status.sh ← NEW -├── GITOPS_STATUS_SERVER_INTEGRATION.md ← NEW (comprehensive) -├── QUICK_REFERENCE.md ← NEW (quick start) -├── REFACTOR_SUMMARY.md ← NEW (changes) -├── README_GITOPS_STATUS.md ← NEW (overview) -├── CHANGES_OVERVIEW.md ← NEW (visual) -└── (this file) -``` - ---- - -## Testing Checklist - -- [ ] Script is executable: `chmod +x update-gitops-status.sh` -- [ ] Test locally: `./update-gitops-status.sh` -- [ ] Woodpecker pipeline runs successfully -- [ ] `update-gitops-status` step completes (post-deploy) -- [ ] Cron job created: `gitops_sync_check` at `*/2 * * * *` -- [ ] Cron job executes on schedule -- [ ] gitops-status-server receives POST requests -- [ ] HTTP 200 responses in logs -- [ ] Grafana dashboard displays sync status -- [ ] Changed files shown in Grafana panel -- [ ] Manual server edit detected within 2 minutes -- [ ] Post-deployment status updated immediately - ---- - -## Key Features Delivered - -1. **Structured JSON Output** - - Sync status (SYNCED / OUT_OF_SYNC) - - Drift count (numeric) - - Changed files (list with names) - - Last check timestamp (ISO 8601) - -2. **Two Integration Points** - - Post-deployment (immediate verification) - - Cron-based (continuous monitoring) - -3. **No Breaking Changes** - - Existing deploy logic unchanged - - Drift detection logic unchanged - - PR validation unchanged - - Only status reporting replaced - -4. **Robust Implementation** - - Error handling included - - Clear logging at each step - - Exit codes meaningful - - Environment variables configurable - - Fully documented - -5. **Production-Ready** - - Tested patterns used - - Security considered - - Comments explain flow - - Easy to troubleshoot - - Scalable architecture - ---- - -## Advantages Over Previous Implementation - -| Factor | Previous | New | -|--------|----------|-----| -| **Infrastructure** | Pushgateway + Prometheus | gitops-status-server (single service) | -| **Data richness** | 0/1 metric only | Full JSON with file names | -| **File-level details** | None | Complete list of changed files | -| **Grafana integration** | Prometheus datasource | Infinity datasource (native) | -| **Audit trail** | Basic metrics | Detailed snapshots with timestamps | -| **Setup complexity** | High | Low | -| **Query language** | PromQL (complex) | JSON API (simple) | - ---- - -## Documentation Quick Links - -1. **For Implementation Overview:** - → `README_GITOPS_STATUS.md` - -2. **For Quick Start (5 steps):** - → `QUICK_REFERENCE.md` - -3. **For Detailed Architecture:** - → `GITOPS_STATUS_SERVER_INTEGRATION.md` - -4. **For Understanding Changes:** - → `CHANGES_OVERVIEW.md` or `REFACTOR_SUMMARY.md` - -5. **For Code Details:** - → Comments in `.woodpecker.yml` and `update-gitops-status.sh` - ---- - -## Support Resources - -### When Cron Doesn't Run -- Check `QUICK_REFERENCE.md` → "Troubleshooting" section -- Check Woodpecker cron job configuration -- Verify schedule is `*/2 * * * *` - -### When JSON Isn't Posted -- Check gitops-status-server is running and healthy -- Verify URL in `.woodpecker.yml` is correct -- Check network connectivity from Woodpecker to gitops-status-server -- Review gitops-status-server logs - -### When Grafana Shows No Data -- Check Infinity datasource configuration -- Verify gitops-status-server is serving `/status.json` -- Check dashboard panel query -- Test datasource with "Test" button - -### For Any Other Issues -- Review relevant documentation file -- Check Woodpecker pipeline logs -- Check gitops-status-server application logs -- Manual test: `./update-gitops-status.sh` - ---- - -## Performance Characteristics - -- **Cron frequency:** Every 2 minutes (configurable) -- **Execution time:** ~30 seconds per run -- **JSON payload size:** ~500 bytes -- **Network impact:** Minimal (internal only) -- **CPU impact:** Negligible -- **Storage impact:** Single JSON file (~500 bytes) - ---- - -## Maintenance - -### Regular Tasks -- Monitor cron execution (verify runs every 2 minutes) -- Check gitops-status-server health -- Review Grafana dashboard (verify updates) -- Monitor logs for errors - -### When to Troubleshoot -- Cron stops running -- gitops-status-server unreachable -- Grafana shows stale data -- HTTP errors in logs - -### Updates -- To change cron frequency: Edit `.woodpecker.yml` -- To change server name: Edit `.woodpecker.yml` -- To change gitops-status-server URL: Edit `.woodpecker.yml` - ---- - -## Success Criteria (How to Know It's Working) - -✅ **Cron runs on schedule** -- Woodpecker shows execution every 2 minutes - -✅ **JSON posted successfully** -- Logs show "✓ Status update successful (HTTP 200)" - -✅ **gitops-status-server receives data** -- Application logs show POST requests -- `/status.json` contains latest snapshot - -✅ **Grafana dashboard works** -- Shows sync status (green/red) -- Shows drift count -- Lists changed files -- Displays last check time - -✅ **Drift detection works** -- Manual server edit detected within 2 minutes -- Status changes from SYNCED to OUT_OF_SYNC -- Changed files listed correctly - -✅ **No errors** -- Pipeline logs are clean -- No ERROR or FAIL messages -- All steps complete successfully - ---- - -## Estimated Time to Deploy - -1. Review changes: **5 min** -2. Test locally (optional): **5 min** -3. Commit and push: **2 min** -4. Monitor first run: **5 min** -5. Create cron job: **5 min** -6. Verify cron execution: **5 min** -7. Test dashboard: **5 min** -8. Full validation: **20 min** - -**Total:** 30-45 minutes (including testing) - ---- - -## Conclusion - -You now have a **production-ready implementation** that: - -✅ Removes Pushgateway dependency for this use case -✅ Provides rich metadata (file-level details) -✅ Integrates seamlessly with gitops-status-server -✅ Works with Grafana Infinity datasource -✅ Detects drift automatically every 2 minutes -✅ Verifies deployments immediately after push -✅ Is fully documented and commented -✅ Includes comprehensive troubleshooting guides - -### Ready to Deploy -All files are in place, tested, and documented. Simply push to Git and follow the quick start guide. - ---- - -## Questions? - -Refer to the appropriate documentation file: -- Overview? → `README_GITOPS_STATUS.md` -- Quick start? → `QUICK_REFERENCE.md` -- Architecture? → `GITOPS_STATUS_SERVER_INTEGRATION.md` -- Changes? → `CHANGES_OVERVIEW.md` -- Troubleshooting? → `QUICK_REFERENCE.md` (Troubleshooting section) - ---- - -**Status:** ✅ **COMPLETE AND READY FOR DEPLOYMENT** - -All deliverables have been implemented, documented, and tested. You can proceed with confidence. diff --git a/DEPLOYMENT_CHECKLIST.md b/DEPLOYMENT_CHECKLIST.md deleted file mode 100644 index 0a54d89..0000000 --- a/DEPLOYMENT_CHECKLIST.md +++ /dev/null @@ -1,310 +0,0 @@ -# Deployment Checklist - -This checklist guides you through deploying the updated rsyslog repository with gitops-status-server integration. - -## Prerequisites - -Before deploying, ensure: - -- [ ] gitops-status-server is deployed and accessible at: - `http://gitops-status-server.observability-stack.svc.cluster.local:80` -- [ ] gitops-status-server has `/api/status` endpoint implemented (see [GITOPS_STATUS_API_REFERENCE.md](GITOPS_STATUS_API_REFERENCE.md)) -- [ ] Woodpecker CI is configured for this repository -- [ ] SSH access to rsyslog-lab server is configured -- [ ] SSH_PRIVATE_KEY secret is set in Woodpecker repository settings - -## Files Changed/Added - -### New Files -- ✅ `update-gitops-status.sh` - Main script for JSON status generation -- ✅ `GITOPS_STATUS_INTEGRATION.md` - Integration documentation -- ✅ `GITOPS_STATUS_API_REFERENCE.md` - API reference with examples -- ✅ `MIGRATION_SUMMARY.md` - Summary of all changes made - -### Modified Files -- ✅ `.woodpecker.yml` - Updated pipeline to use JSON status -- ✅ `README.md` - Updated documentation with new flow - -### Unchanged Files (no action needed) -- ✅ All Ansible playbooks -- ✅ All Ansible inventory files -- ✅ All rsyslog config files -- ✅ Local scripts (apply.sh, drift-check.sh) - -## Deployment Steps - -### 1. Review Changes - -- [ ] Read [MIGRATION_SUMMARY.md](MIGRATION_SUMMARY.md) to understand all changes -- [ ] Review [.woodpecker.yml](.woodpecker.yml) pipeline changes -- [ ] Review [update-gitops-status.sh](update-gitops-status.sh) script logic - -### 2. Verify gitops-status-server - -Test the gitops-status-server API endpoint: - -```bash -# Test POST endpoint (should return 200 or 404 if not implemented yet) -curl -X POST http://gitops-status-server.observability-stack.svc.cluster.local:80/api/status \ - -H "Content-Type: application/json" \ - -d '{ - "repo": "test", - "server": "test", - "sync_status": "SYNCED", - "drift_count": 0, - "files": [], - "last_check": "2026-04-21T10:00:00Z" - }' - -# Test GET endpoint -curl http://gitops-status-server.observability-stack.svc.cluster.local:80/status.json -``` - -If the API is not implemented yet: -- [ ] Implement gitops-status-server API (use [GITOPS_STATUS_API_REFERENCE.md](GITOPS_STATUS_API_REFERENCE.md)) -- [ ] Deploy to Kubernetes cluster -- [ ] Verify endpoints are accessible - -### 3. Test Locally (Optional) - -Before pushing to Git, you can test the script locally: - -```bash -# Set environment variables -export GITOPS_STATUS_SERVER_URL="http://gitops-status-server.observability-stack.svc.cluster.local:80" -export REPO_NAME="rsyslog" -export SERVER_NAME="rsyslog-lab" - -# Make script executable -chmod +x update-gitops-status.sh - -# Run the script (requires Ansible, jq, curl) -./update-gitops-status.sh -``` - -Expected output: -``` -==> Running drift check playbook... - Inventory: ansible/inventory/hosts.yml - Playbook: ansible/playbooks/drift-check.yml -==> Status: SYNCED - server configuration matches Git -==> Drift count: 0 -==> Generated JSON status: -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "SYNCED", - "drift_count": 0, - "files": [], - "last_check": "2026-04-21T10:30:00Z" -} -==> Sending status to gitops-status-server... - URL: http://gitops-status-server.observability-stack.svc.cluster.local:80/api/status -==> Status update successful (HTTP 200) -``` - -### 4. Commit and Push Changes - -```bash -# Stage all changes -git add . - -# Commit -git commit -m "Migrate from Pushgateway to gitops-status-server JSON status - -- Add update-gitops-status.sh script for JSON status generation -- Update .woodpecker.yml to use gitops-status-server -- Remove Pushgateway metric push logic -- Add comprehensive documentation -- Keep all Ansible playbooks unchanged" - -# Push to master (will trigger deploy pipeline) -git push origin master -``` - -### 5. Monitor First Deployment - -After pushing to master: - -- [ ] Watch Woodpecker pipeline execution -- [ ] Verify `syntax-check` step passes -- [ ] Verify `validate` step passes -- [ ] Verify `deploy` step completes -- [ ] Verify `update-gitops-status` step runs successfully -- [ ] Check that JSON is sent to gitops-status-server - -Example successful `update-gitops-status` step output: -``` -==> Running post-deploy GitOps status check... -==> Running drift check playbook... -==> Status: SYNCED - server configuration matches Git -==> Drift count: 0 -==> Generated JSON status: {...} -==> Sending status to gitops-status-server... -==> Status update successful (HTTP 200) -==> JSON status update complete. Pipeline always succeeds. -``` - -### 6. Verify Cron Pipeline - -The cron pipeline runs every 2 minutes: - -- [ ] Wait for next cron execution -- [ ] Check Woodpecker for `gitops_sync_check` pipeline run -- [ ] Verify JSON status is sent -- [ ] Verify pipeline succeeds (if synced) or fails (if drift detected) - -### 7. Test Drift Detection - -Manually create drift to test detection: - -```bash -# SSH to the server -ssh rsyslog-lab - -# Edit a config file -echo "# manual edit" >> /etc/rsyslog.conf - -# Wait up to 2 minutes for next cron run -``` - -Expected behavior: -- [ ] Cron pipeline runs -- [ ] Drift detected in Ansible playbook -- [ ] JSON sent with `sync_status: "OUT_OF_SYNC"` -- [ ] JSON includes `files: [{"name": "rsyslog.conf"}]` -- [ ] Pipeline marked as FAILED (for visibility) - -Verify in gitops-status-server: -```bash -curl http://gitops-status-server.observability-stack.svc.cluster.local:80/status.json -``` - -Should show: -```json -[ - { - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 1, - "files": [ - {"name": "rsyslog.conf"} - ], - "last_check": "...", - "updated_at": "..." - } -] -``` - -### 8. Configure Grafana Dashboard - -- [ ] Add Infinity datasource pointing to gitops-status-server -- [ ] Create dashboard to display GitOps status -- [ ] Add panels for: - - Sync status overview (SYNCED vs OUT_OF_SYNC) - - Drift count per repo - - Detailed file list for drifted repos - - Last check timestamp - - Historical trend (if gitops-status-server stores history) - -Example dashboard queries provided in [GITOPS_STATUS_API_REFERENCE.md](GITOPS_STATUS_API_REFERENCE.md). - -### 9. Cleanup (Optional) - -If everything works correctly: - -- [ ] Remove old Pushgateway metrics for rsyslog (if no longer needed) -- [ ] Update any alerts/dashboards that used old `gitops_sync_status` metric -- [ ] Document the new JSON status format in team wiki/docs - -## Troubleshooting - -### Script fails with "command not found: jq" - -**Problem:** `jq` is not installed in the Woodpecker container - -**Solution:** The `.woodpecker.yml` already includes `apk add --no-cache jq`. Verify the step runs before the script. - -### Script fails with "HTTP 404" or "HTTP 500" - -**Problem:** gitops-status-server endpoint not implemented or not accessible - -**Solution:** -1. Verify gitops-status-server is running: `curl http://gitops-status-server.observability-stack.svc.cluster.local:80/health` -2. Check Kubernetes service: `kubectl get svc -n observability-stack` -3. Implement `/api/status` endpoint using [GITOPS_STATUS_API_REFERENCE.md](GITOPS_STATUS_API_REFERENCE.md) - -### Script fails with "No such file or directory: update-gitops-status.sh" - -**Problem:** Script not found in workspace - -**Solution:** The script is created in the repository root. Verify it's committed to Git and available in the CI workspace. - -### Drift not detected when expected - -**Problem:** Manual changes not showing up in drift check - -**Solution:** -1. Verify changes are to files managed by Git (rsyslog.conf or rsyslog.d/*.conf) -2. Check Ansible playbook output for diff details -3. Verify SSH access to server from CI container - -### JSON has empty files array even when drift detected - -**Problem:** Script parsing logic not extracting filenames correctly - -**Solution:** -1. Check Ansible output format - script expects specific JSON structure -2. Run with `ANSIBLE_STDOUT_CALLBACK=json` to see raw output -3. Update regex patterns in `update-gitops-status.sh` if needed - -## Rollback Procedure - -If you need to rollback to Pushgateway: - -```bash -# Revert to previous commit -git revert HEAD - -# Or restore specific files -git checkout HEAD~1 .woodpecker.yml -git checkout HEAD~1 README.md - -# Remove new files -git rm update-gitops-status.sh GITOPS_STATUS_*.md MIGRATION_SUMMARY.md - -# Commit and push -git commit -m "Rollback to Pushgateway metrics" -git push origin master -``` - -## Success Criteria - -✅ All checks passed when: - -- [ ] Woodpecker pipeline completes successfully on push to master -- [ ] JSON status is sent to gitops-status-server after deploy -- [ ] Cron pipeline runs every 2 minutes and sends JSON status -- [ ] Drift is correctly detected and reported in JSON -- [ ] gitops-status-server `/status.json` endpoint returns correct data -- [ ] Grafana dashboard displays rsyslog sync status -- [ ] No errors in Woodpecker logs -- [ ] File-level drift details are visible in Grafana - -## Additional Resources - -- [README.md](README.md) - Repository overview and workflow -- [MIGRATION_SUMMARY.md](MIGRATION_SUMMARY.md) - Detailed migration changes -- [GITOPS_STATUS_INTEGRATION.md](GITOPS_STATUS_INTEGRATION.md) - Integration architecture -- [GITOPS_STATUS_API_REFERENCE.md](GITOPS_STATUS_API_REFERENCE.md) - API implementation guide - -## Support - -If you encounter issues: - -1. Check Woodpecker pipeline logs -2. Verify gitops-status-server logs -3. Test API endpoints manually with curl -4. Review Ansible playbook output -5. Check this repository's documentation files diff --git a/GITOPS_STATUS_API_REFERENCE.md b/GITOPS_STATUS_API_REFERENCE.md deleted file mode 100644 index d22823b..0000000 --- a/GITOPS_STATUS_API_REFERENCE.md +++ /dev/null @@ -1,326 +0,0 @@ -# gitops-status-server API Reference - -This document provides a reference implementation example for the gitops-status-server API endpoint that receives status updates from the rsyslog repository. - -## API Endpoint Specification - -### POST /api/status - -Receives GitOps status updates from repositories. - -**Request:** -``` -POST /api/status HTTP/1.1 -Host: gitops-status-server.observability-stack.svc.cluster.local:80 -Content-Type: application/json - -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 2, - "files": [ - { "name": "rsyslog.conf" }, - { "name": "rsyslog.d/30-lab.conf" } - ], - "last_check": "2026-04-21T10:32:15Z" -} -``` - -**Response (Success):** -``` -HTTP/1.1 200 OK -Content-Type: application/json - -{ - "status": "ok", - "message": "Status updated successfully" -} -``` - -**Response (Error):** -``` -HTTP/1.1 400 Bad Request -Content-Type: application/json - -{ - "status": "error", - "message": "Invalid JSON payload" -} -``` - -## Example Implementation (Python/Flask) - -```python -from flask import Flask, request, jsonify -from datetime import datetime -import json -import os - -app = Flask(__name__) - -# In-memory storage (replace with database in production) -status_data = {} - -@app.route('/api/status', methods=['POST']) -def update_status(): - """Receive and store GitOps status updates""" - try: - data = request.get_json() - - # Validate required fields - required_fields = ['repo', 'server', 'sync_status', 'drift_count', 'files', 'last_check'] - for field in required_fields: - if field not in data: - return jsonify({ - 'status': 'error', - 'message': f'Missing required field: {field}' - }), 400 - - # Validate sync_status value - if data['sync_status'] not in ['SYNCED', 'OUT_OF_SYNC']: - return jsonify({ - 'status': 'error', - 'message': 'sync_status must be SYNCED or OUT_OF_SYNC' - }), 400 - - # Create unique key for this repo/server combination - key = f"{data['repo']}:{data['server']}" - - # Store the status - status_data[key] = { - 'repo': data['repo'], - 'server': data['server'], - 'sync_status': data['sync_status'], - 'drift_count': data['drift_count'], - 'files': data['files'], - 'last_check': data['last_check'], - 'updated_at': datetime.utcnow().isoformat() + 'Z' - } - - # Log the update - print(f"Status update: {key} -> {data['sync_status']} (drift_count: {data['drift_count']})") - - return jsonify({ - 'status': 'ok', - 'message': 'Status updated successfully' - }), 200 - - except Exception as e: - print(f"Error processing status update: {e}") - return jsonify({ - 'status': 'error', - 'message': str(e) - }), 500 - - -@app.route('/status.json', methods=['GET']) -def get_status(): - """Serve aggregated status for Grafana Infinity datasource""" - # Convert dict to list for JSON array output - statuses = list(status_data.values()) - - return jsonify(statuses), 200 - - -@app.route('/health', methods=['GET']) -def health(): - """Health check endpoint""" - return jsonify({ - 'status': 'healthy', - 'timestamp': datetime.utcnow().isoformat() + 'Z', - 'tracked_repos': len(status_data) - }), 200 - - -if __name__ == '__main__': - app.run(host='0.0.0.0', port=8080) -``` - -## Example Implementation (Go) - -```go -package main - -import ( - "encoding/json" - "fmt" - "log" - "net/http" - "sync" - "time" -) - -type StatusUpdate struct { - Repo string `json:"repo"` - Server string `json:"server"` - SyncStatus string `json:"sync_status"` - DriftCount int `json:"drift_count"` - Files []File `json:"files"` - LastCheck string `json:"last_check"` -} - -type File struct { - Name string `json:"name"` -} - -type StoredStatus struct { - StatusUpdate - UpdatedAt string `json:"updated_at"` -} - -var ( - statusStore = make(map[string]StoredStatus) - storeMutex sync.RWMutex -) - -func updateStatusHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - var status StatusUpdate - if err := json.NewDecoder(r.Body).Decode(&status); err != nil { - http.Error(w, fmt.Sprintf("Invalid JSON: %v", err), http.StatusBadRequest) - return - } - - // Validate sync_status - if status.SyncStatus != "SYNCED" && status.SyncStatus != "OUT_OF_SYNC" { - http.Error(w, "sync_status must be SYNCED or OUT_OF_SYNC", http.StatusBadRequest) - return - } - - // Store the status - key := fmt.Sprintf("%s:%s", status.Repo, status.Server) - stored := StoredStatus{ - StatusUpdate: status, - UpdatedAt: time.Now().UTC().Format(time.RFC3339), - } - - storeMutex.Lock() - statusStore[key] = stored - storeMutex.Unlock() - - log.Printf("Status update: %s -> %s (drift_count: %d)", key, status.SyncStatus, status.DriftCount) - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(map[string]string{ - "status": "ok", - "message": "Status updated successfully", - }) -} - -func getStatusHandler(w http.ResponseWriter, r *http.Request) { - storeMutex.RLock() - statuses := make([]StoredStatus, 0, len(statusStore)) - for _, status := range statusStore { - statuses = append(statuses, status) - } - storeMutex.RUnlock() - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(statuses) -} - -func healthHandler(w http.ResponseWriter, r *http.Request) { - storeMutex.RLock() - count := len(statusStore) - storeMutex.RUnlock() - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(map[string]interface{}{ - "status": "healthy", - "timestamp": time.Now().UTC().Format(time.RFC3339), - "tracked_repos": count, - }) -} - -func main() { - http.HandleFunc("/api/status", updateStatusHandler) - http.HandleFunc("/status.json", getStatusHandler) - http.HandleFunc("/health", healthHandler) - - log.Println("Starting gitops-status-server on :8080") - log.Fatal(http.ListenAndServe(":8080", nil)) -} -``` - -## Testing the API - -### Using curl - -**Send a status update:** -```bash -curl -X POST http://localhost:8080/api/status \ - -H "Content-Type: application/json" \ - -d '{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 2, - "files": [ - {"name": "rsyslog.conf"}, - {"name": "rsyslog.d/30-lab.conf"} - ], - "last_check": "2026-04-21T10:32:15Z" - }' -``` - -**Get all statuses:** -```bash -curl http://localhost:8080/status.json -``` - -**Health check:** -```bash -curl http://localhost:8080/health -``` - -## Grafana Infinity Datasource Configuration - -1. Install Grafana Infinity datasource plugin -2. Add new datasource: - - Type: Infinity - - URL: `http://gitops-status-server.observability-stack.svc.cluster.local:80` -3. Create a panel with query: - - URL: `/status.json` - - Parser: Backend - - Format: Table - -Example query to show all repos: -``` -Source: URL -URL: /status.json -Parser: Backend -Format: Table -Columns: - - repo (string) - - server (string) - - sync_status (string) - - drift_count (number) - - last_check (time) -``` - -Example query to show drift details: -``` -Source: URL -URL: /status.json -Parser: Backend -Format: Table -Root/Rows: $[?(@.drift_count > 0)] -Columns: - - repo (string) - - server (string) - - drift_count (number) - - files (string, JSONata: $join(files.name, ', ')) -``` - -## Notes - -- The example implementations use in-memory storage; production should use a database -- Consider adding authentication/authorization for the POST endpoint -- Add monitoring/metrics for the status server itself -- Consider adding TTL/expiration for stale status entries -- The `/status.json` endpoint should support filtering (e.g., by repo or server) diff --git a/GITOPS_STATUS_FIX.md b/GITOPS_STATUS_FIX.md deleted file mode 100644 index 5227305..0000000 --- a/GITOPS_STATUS_FIX.md +++ /dev/null @@ -1,260 +0,0 @@ -# GitOps Status Fix - Root Cause Analysis and Solutions - -## Problem Statement - -After deploying configuration changes via the Woodpecker CI pipeline: -1. The status remained **OUT_OF_SYNC** even though deployment succeeded -2. The **files array** in the status JSON was empty/incorrect - -## Architecture Overview - -### Three Repository Structure: - -1. **rsyslog** (this repo) - - Contains Ansible playbooks and .woodpecker.yml - - Runs drift-check.yml to detect configuration drift - - Sends status JSON to gitops-status-server API - -2. **gitops-status-api** - - Flask API for storing/retrieving status - - Endpoints: - - POST /api/status - Update status - - GET /api/status - Retrieve status - - GET /status.json - Retrieve status (for Grafana Infinity datasource) - -3. **observability-stack** - - ArgoCD Application that deploys gitops-status-server - - Helm chart: `charts/gitops-status-server/` - - Deployment: Single Pod with Flask API container - - Service: ClusterIP on port 80 -> container port 5000 - -## Root Cause Analysis - -### Issue 1: Ansible Callback Breaking Output Parsing - -**Problem:** -- `.woodpecker.yml` set `ANSIBLE_STDOUT_CALLBACK=minimal` -- `update-gitops-status.sh` also forced `ANSIBLE_CALLBACKS_ENABLED=""` -- With minimal callback, debug task output format changes: - ``` - # Expected format (default callback): - ok: [host] => { - "msg": "DRIFTED_FILES=/etc/rsyslog.conf,/etc/rsyslog.d/30-lab.conf" - } - - # Actual format (minimal callback): - host | SUCCESS => { - "msg": "DRIFTED_FILES=/etc/rsyslog.conf,/etc/rsyslog.d/30-lab.conf" - } - ``` -- The `grep` and `sed` parsing in update-gitops-status.sh failed to extract DRIFTED_FILES correctly - -**Impact:** -- Even when drift was detected, the files array stayed empty -- `drift_count` was 0 even though `sync_status` was OUT_OF_SYNC -- Grafana showed incomplete information - -**Root Cause:** -Inconsistent Ansible callback configuration caused unpredictable debug output formatting. - -### Issue 2: Status Shows OUT_OF_SYNC After Successful Deploy - -**This is actually CORRECT behavior if drift exists!** - -The pipeline flow is: -1. `deploy` step runs `apply.yml` - deploys config to server -2. `update-gitops-status` step runs `drift-check.yml` - checks if server matches Git - -If drift-check shows OUT_OF_SYNC after deploy, it means: -- The deployment didn't fully succeed, OR -- There are other differences (permissions, extra files on server, etc.) - -**However**, the real issue was: -- We couldn't see WHICH files were drifted (files array was empty) -- This made it impossible to diagnose the root cause - -## Solutions Implemented - -### Fix 1: Use YAML Callback for Consistent Output - -**Changed in:** -- `update-gitops-status.sh` -- `.woodpecker.yml` (update-gitops-status step) -- `.woodpecker.yml` (gitops_sync_check cron step) - -**What changed:** -```bash -# BEFORE: -ANSIBLE_CALLBACKS_ENABLED="" \ -ANSIBLE_STDOUT_CALLBACK=minimal \ -ansible-playbook ... - -# AFTER: -ANSIBLE_FORCE_COLOR=false \ -ANSIBLE_STDOUT_CALLBACK=yaml \ -ansible-playbook ... -``` - -**Why YAML callback:** -- Consistent, structured output format -- Better for parsing than minimal callback -- Still compact and readable -- Widely supported across Ansible versions - -### Fix 2: Improved DRIFTED_FILES Parsing - -**Changed in:** `update-gitops-status.sh` - -**Old parsing:** -```bash -DRIFTED_FILES_STR=$(echo "$DRIFTED_FILES_STR" | sed 's/.*DRIFTED_FILES=//' | sed 's/\x1b\[[0-9;]*m//g' | sed 's/".*$//' | xargs) -``` - -Problems: -- Assumed specific ANSI color codes -- Used `xargs` which could break on certain characters -- The `sed 's/".*$//'` would strip everything after first quote - -**New parsing:** -```bash -DRIFTED_FILES_STR=$(echo "$DRIFTED_FILES_LINE" | sed 's/.*DRIFTED_FILES=//' | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//' | tr -d '"') -``` - -Improvements: -- Removes leading/trailing whitespace properly -- Strips quotes without breaking the content -- Works with both YAML and default callback formats -- More robust character handling - -### Fix 3: Removed Problematic Environment Variables - -**Removed from `.woodpecker.yml`:** -- `ANSIBLE_CALLBACK_WHITELIST: "minimal"` (conflicted with script settings) -- `ANSIBLE_LIBRARY_CACHING: "True"` (not needed, could cause issues) -- `ANSIBLE_CALLBACKS_ENABLED=""` export in commands (broke debug output) -- `ANSIBLE_GATHERING=explicit` export (not related to the issue) - -**Kept:** -- `ANSIBLE_HOST_KEY_CHECKING: "False"` (required for CI) -- `ANSIBLE_FORCE_COLOR: "False"` (helps with parsing) -- `ANSIBLE_RETRY_FILES_ENABLED: "False"` (cleaner CI runs) -- `ANSIBLE_UNSAFE_WRITES: "True"` (helps with temp files) - -## Testing the Fix - -### Expected Behavior After Fix - -#### Scenario 1: After Successful Deployment (push to master) -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "SYNCED", - "drift_count": 0, - "files": [], - "last_check": "2026-04-22T19:00:00Z" -} -``` - -#### Scenario 2: When Drift is Detected (cron job or manual server change) -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 2, - "files": [ - {"name": "rsyslog.conf"}, - {"name": "rsyslog.d/30-lab.conf"} - ], - "last_check": "2026-04-22T19:02:00Z" -} -``` - -### How to Test - -1. **Test normal deployment:** - ```bash - # Make a change - echo "# Test $(date)" >> files/rsyslog.conf - - # Commit and push - git add files/rsyslog.conf - git commit -m "test: verify status tracking" - git push - - # Watch pipeline in Woodpecker - # After deploy + update-gitops-status completes: - # - Check Grafana: sync_status should be SYNCED - # - drift_count should be 0 - # - files should be [] - ``` - -2. **Test drift detection:** - ```bash - # SSH to server - ssh rsyslog-lab - - # Make a manual change - echo "# Manual drift $(date)" >> /etc/rsyslog.conf - - # Wait for cron job (runs every 2 minutes) - # OR manually trigger in Woodpecker - - # Check Grafana: - # - sync_status should be OUT_OF_SYNC - # - drift_count should be 1 or more - # - files array should list "rsyslog.conf" - ``` - -3. **Debug mode (if issues persist):** - ```bash - # Run locally with debug logging - export KEEP_PLAYBOOK_LOG=true - ./update-gitops-status.sh - - # Check the output - cat drift-check-output.log | grep -A 5 "DRIFTED_FILES" - ``` - -## Verification Steps - -After deploying this fix: - -1. ✅ Check that DRIFTED_FILES appears in playbook output -2. ✅ Check that files array is populated when drift exists -3. ✅ Check that sync_status is SYNCED after successful deployment -4. ✅ Check that drift_count matches the number of files -5. ✅ Check that Grafana shows the correct data -6. ✅ Check that cron drift detection works correctly - -## Related Files Changed - -### rsyslog repo: -- `.woodpecker.yml` - Fixed Ansible callback configuration -- `update-gitops-status.sh` - Improved DRIFTED_FILES parsing -- `GITOPS_STATUS_FIX.md` - This document - -### No changes needed in: -- `gitops-status-api` repo (API code is correct) -- `observability-stack` repo (deployment is correct) -- `ansible/playbooks/drift-check.yml` (playbook logic is correct) - -## Summary - -**What was wrong:** -1. Inconsistent Ansible callback configuration broke debug output parsing -2. DRIFTED_FILES extraction failed silently -3. files array stayed empty even when drift was detected - -**What was fixed:** -1. Standardized on YAML callback for consistent output -2. Improved parsing to handle YAML format reliably -3. Removed conflicting environment variables -4. Added better debug logging - -**Result:** -- Files array now populates correctly when drift exists -- Sync status accurately reflects server state -- Grafana dashboards show complete information -- Drift detection works end-to-end diff --git a/GITOPS_STATUS_INTEGRATION.md b/GITOPS_STATUS_INTEGRATION.md deleted file mode 100644 index a0e760c..0000000 --- a/GITOPS_STATUS_INTEGRATION.md +++ /dev/null @@ -1,117 +0,0 @@ -# GitOps Status Server Integration - -This document explains how the rsyslog repository integrates with gitops-status-server for GitOps monitoring. - -## Overview - -Instead of pushing simple numeric metrics to Prometheus Pushgateway, the rsyslog repo now sends structured JSON status snapshots to gitops-status-server. This enables richer visualization in Grafana with file-level drift details. - -## Architecture - -``` -┌─────────────────┐ ┌──────────────────────┐ ┌─────────────────────┐ -│ Woodpecker CI │ │ gitops-status-server │ │ Grafana │ -│ (rsyslog) │────────►│ (Kubernetes) │────────►│ Infinity Plugin │ -│ │ POST │ │ GET │ │ -│ drift-check │ JSON │ serves /status.json │ │ Dashboard shows │ -│ every 2 min │ │ │ │ drift details │ -└─────────────────┘ └──────────────────────┘ └─────────────────────┘ -``` - -## API Endpoint - -The rsyslog repo sends JSON status updates to: - -``` -POST http://gitops-status-server.observability-stack.svc.cluster.local:80/api/status -Content-Type: application/json -``` - -## JSON Payload Format - -### When synced (no drift) - -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "SYNCED", - "drift_count": 0, - "files": [], - "last_check": "2026-04-21T10:30:00Z" -} -``` - -### When drift detected - -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 2, - "files": [ - { "name": "rsyslog.conf" }, - { "name": "rsyslog.d/30-lab.conf" } - ], - "last_check": "2026-04-21T10:32:15Z" -} -``` - -## Field Definitions - -| Field | Type | Description | -|----------------|----------|-------------------------------------------------------| -| `repo` | string | Repository name (e.g., "rsyslog") | -| `server` | string | Target server name (e.g., "rsyslog-lab") | -| `sync_status` | string | Either "SYNCED" or "OUT_OF_SYNC" | -| `drift_count` | integer | Number of files that have drifted from Git | -| `files` | array | List of files with drift (empty if synced) | -| `files[].name` | string | Relative path of drifted file | -| `last_check` | string | ISO 8601 timestamp of when drift check was performed | - -## When Updates Are Sent - -1. **After deployment** (push to master): - - Post-deploy verification runs - - JSON status sent to gitops-status-server - - Pipeline step: `update-gitops-status` - -2. **Scheduled cron check** (every 2 minutes): - - Continuous drift monitoring - - JSON status sent to gitops-status-server - - Pipeline step: `gitops_sync_check` - -## Error Handling - -If the HTTP POST to gitops-status-server fails: -- The pipeline step will fail -- Error message will be logged -- The drift check itself is still performed -- No retry logic (next cron run will retry) - -## Script Implementation - -The `update-gitops-status.sh` script handles: -1. Running the Ansible drift-check playbook -2. Parsing the output to extract changed file names -3. Building the JSON payload -4. Sending it to gitops-status-server via HTTP POST - -## Expected HTTP Response - -gitops-status-server should respond with: -- `200 OK` or `201 Created` on success -- `4xx` or `5xx` on error - -The rsyslog pipeline treats any 2xx response as success. - -## Grafana Visualization - -Grafana uses the Infinity datasource plugin to fetch `/status.json` from gitops-status-server and display: -- Current sync status (SYNCED vs OUT_OF_SYNC) -- Number of drifted files -- List of specific files that have drifted -- Last check timestamp - -This provides much richer information than a simple numeric metric. diff --git a/GITOPS_STATUS_SERVER_INTEGRATION.md b/GITOPS_STATUS_SERVER_INTEGRATION.md deleted file mode 100644 index a137a1e..0000000 --- a/GITOPS_STATUS_SERVER_INTEGRATION.md +++ /dev/null @@ -1,493 +0,0 @@ -# GitOps Status Server Integration - rsyslog Workflow - -## Overview - -This document describes the rsyslog repository's integration with **gitops-status-server**, a centralized Kubernetes-based service that manages GitOps status information for multiple repositories. - -**Architecture:** Rsyslog repo → generates JSON → POST to gitops-status-server → Grafana reads via Infinity datasource - ---- - -## Architecture - -``` -┌─ Scheduled Cron (every 2 minutes) ─────────────────────────────┐ -│ │ -│ Woodpecker Trigger: event: cron │ -│ ↓ │ -│ Step: gitops_sync_check │ -│ └─ Run: update-gitops-status.sh │ -│ ├─ Execute: ansible/playbooks/drift-check.yml │ -│ │ └─ Check rsyslog.conf and rsyslog.d/* files │ -│ │ └─ Detect drift vs Git repo │ -│ │ └─ Output: DRIFTED_FILES=file1,file2,file3 │ -│ ├─ Parse: Extract sync status and changed files │ -│ ├─ Generate: JSON payload with metadata │ -│ └─ POST: JSON to gitops-status-server/api/status │ -│ ↓ │ -│ gitops-status-server (Kubernetes service) │ -│ └─ Receives JSON │ -│ └─ Updates internal state │ -│ └─ Serves at: /status.json │ -│ ↓ │ -│ Grafana │ -│ └─ Infinity datasource polls /status.json │ -│ └─ Displays: sync status, drift count, changed files │ -│ │ -└──────────────────────────────────────────────────────────────────┘ - -┌─ Post-Deployment Verification (push to master) ─────────────────┐ -│ │ -│ Woodpecker Pipeline Event: push │ -│ Triggered on: push to master branch │ -│ ↓ │ -│ Steps: syntax-check → validate → deploy → update-gitops-status│ -│ ↓ │ -│ Run: update-gitops-status.sh (same as above) │ -│ └─ Verify deployment succeeded │ -│ └─ Generate JSON status │ -│ └─ POST to gitops-status-server │ -│ ↓ │ -│ gitops-status-server updates /status.json │ -│ ↓ │ -│ Grafana reflects latest deployment status │ -│ │ -└──────────────────────────────────────────────────────────────────┘ -``` - ---- - -## Components - -### 1. Ansible Playbook: `ansible/playbooks/drift-check.yml` - -**Purpose:** Detect configuration drift between Git repository and live server - -**Behavior:** -- Runs in check mode (read-only, no actual changes) -- Uses Ansible `copy` module with `check_mode: true` -- Compares controller files (from Git) with server files -- Detects: - - Changes in `/etc/rsyslog.conf` - - Changes in `/etc/rsyslog.d/*.conf` files - - Missing files on server - -**Output (debug messages):** -- `DRIFTED_FILES=file1,file2,file3` – Comma-separated list of changed files -- `SYNC_STATUS=SYNCED` or `SYNC_STATUS=OUT_OF_SYNC` – Simple status indicator - -**Exit Code:** -- `0` – SYNCED (all tasks pass) -- `non-zero` – OUT_OF_SYNC (fail task reached) - -**Example output:** -``` -DRIFTED_FILES=/etc/rsyslog.conf,/etc/rsyslog.d/30-lab.conf -SYNC_STATUS=OUT_OF_SYNC -``` - ---- - -### 2. Shell Script: `update-gitops-status.sh` - -**Purpose:** Orchestrates the complete status update workflow - -**Flow:** -1. **Step 1:** Run `drift-check.yml` playbook - - Captures stdout/stderr to temp file - - Stores exit code - -2. **Step 2:** Parse playbook output - - Extract `DRIFTED_FILES=...` line - - Parse comma-separated files - - Convert full paths to relative paths - - Determine sync status from exit code - -3. **Step 3:** Build JSON payload - - Repo name and server name - - Sync status (SYNCED / OUT_OF_SYNC) - - Drift count (number of changed files) - - Array of changed files with names - - ISO 8601 timestamp - -4. **Step 4:** POST JSON to gitops-status-server - - Endpoint: `$GITOPS_STATUS_SERVER_URL/api/status` - - Method: POST - - Content-Type: application/json - - Check HTTP response code (200-299 = success) - -**Environment Variables:** -```bash -GITOPS_STATUS_SERVER_URL=http://gitops-status-server.observability-stack.svc.cluster.local:80 -REPO_NAME=rsyslog -SERVER_NAME=rsyslog-lab -``` - -**JSON Payload Generated:** -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "SYNCED", - "drift_count": 0, - "files": [], - "last_check": "2026-04-21T10:30:00Z" -} -``` - -**Exit Codes:** -- `0` – Success (JSON posted to gitops-status-server) -- `1` – Failure (playbook error, network error, etc.) - ---- - -### 3. CI/CD Pipeline: `.woodpecker.yml` - -**Two integration points:** - -#### a) Post-Deployment (`update-gitops-status` step) -Runs after successful deployment on push to master: - -```yaml -update-gitops-status: - depends_on: [deploy] - when: - branch: master - event: push - commands: - - chmod +x update-gitops-status.sh - - ./update-gitops-status.sh -``` - -**Purpose:** Verify deployment and update status immediately - -#### b) Scheduled Drift Check (`gitops_sync_check` cron) -Runs on schedule (every 2 minutes by default): - -```yaml -gitops_sync_check: - when: - event: cron - commands: - - chmod +x update-gitops-status.sh - - ./update-gitops-status.sh - # Exit with appropriate code so cron shows as success/failed - - ansible-playbook ... # re-run to determine exit code -``` - -**Purpose:** Continuous monitoring of server state - -**Woodpecker UI Setup:** -1. Go to repository settings -2. Add cron job: - - Name: `gitops_sync_check` - - Branch: `master` - - Schedule: `*/2 * * * *` (every 2 minutes) - ---- - -## Data Flow - -### Input (from Ansible) -Drift-check playbook outputs structured markers: -``` -... -DRIFTED_FILES=/etc/rsyslog.conf,/etc/rsyslog.d/30-lab.conf -SYNC_STATUS=OUT_OF_SYNC -... -``` - -### Processing -Shell script parses output: -- Extract drifted files -- Convert paths: `/etc/rsyslog.conf` → `rsyslog.conf` -- Count changed files -- Get current timestamp - -### Output (to gitops-status-server) -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 2, - "files": [ - { "name": "rsyslog.conf" }, - { "name": "rsyslog.d/30-lab.conf" } - ], - "last_check": "2026-04-21T10:30:00Z" -} -``` - -### Final Display (in Grafana) -Grafana's Infinity datasource reads `/status.json` from gitops-status-server and displays: -- **Sync Status:** Visual indicator (green ✓ SYNCED / red ✗ OUT_OF_SYNC) -- **Drift Count:** Number of changed files -- **Files:** List of which files are different -- **Last Check:** Timestamp of last drift-check run - ---- - -## Workflow: Manual Server Changes Detection - -**Scenario:** Someone manually edits `/etc/rsyslog.conf` directly on the server - -**Detection flow:** -1. File is edited on server, Git repo remains unchanged -2. Cron timer triggers `gitops_sync_check` (every 2 minutes) -3. Woodpecker runs `update-gitops-status.sh` -4. Script executes `drift-check.yml` playbook -5. Ansible copy task in check mode detects difference -6. Playbook outputs: `DRIFTED_FILES=/etc/rsyslog.conf` -7. Script parses output and generates JSON -8. Script POSTs JSON to gitops-status-server -9. gitops-status-server updates its `/status.json` -10. Grafana Infinity datasource refreshes -11. Dashboard shows: - - Status: OUT_OF_SYNC - - Drift count: 1 - - Files: [rsyslog.conf] - - Last check: - -**Time to detection:** ≤ 2 minutes (next cron run) - ---- - -## Workflow: Deployment & Verification - -**Scenario:** Code is pushed to master branch - -**Deployment flow:** -1. Push to master triggers Woodpecker pipeline -2. Steps: syntax-check → validate → deploy → update-gitops-status -3. Deploy step runs `ansible/playbooks/apply.yml` - - Copies Git files to server - - Makes rsyslog.conf and rsyslog.d/* files match Git -4. Update-gitops-status step runs: - - Executes `drift-check.yml` again - - Should report no drift (files match Git) - - Generates JSON with `SYNC_STATUS=SYNCED` - - POSTs JSON to gitops-status-server -5. gitops-status-server updates `/status.json` to SYNCED -6. Grafana dashboard immediately shows SYNCED status - -**Expected result:** After deployment, status should be SYNCED within minutes - ---- - -## Configuration - -### Environment Variables - -Set in `.woodpecker.yml`: - -```yaml -environment: - # URL of gitops-status-server ClusterIP service - GITOPS_STATUS_SERVER_URL: http://gitops-status-server.observability-stack.svc.cluster.local:80 - - # Repository identifier (for multi-repo dashboards) - REPO_NAME: rsyslog - - # Server identifier (for multi-server dashboards) - SERVER_NAME: rsyslog-lab - - # SSH private key (for Ansible to connect to server) - SSH_PRIVATE_KEY: from_secret: SSH_PRIVATE_KEY - - # Ansible config location - ANSIBLE_CONFIG: ansible.cfg -``` - -### Customization - -To support multiple servers: -```yaml -gitops_sync_check: - commands: - - bash update-gitops-status.sh # Uses env vars SERVER_NAME - # Or: - - SERVER_NAME=rsyslog-lab-1 bash update-gitops-status.sh - - SERVER_NAME=rsyslog-lab-2 bash update-gitops-status.sh -``` - ---- - -## Advantages Over Pushgateway - -| Aspect | Pushgateway | gitops-status-server | -|--------|-------------|----------------------| -| **Infrastructure** | Prometheus + Pushgateway | Single Kubernetes service | -| **Data richness** | Only 0/1 metric | Full JSON with file names, timestamps | -| **Query language** | PromQL (complex) | Simple JSON API (easy) | -| **Grafana integration** | Prometheus datasource | Infinity datasource (JSON) | -| **Multi-repository** | Complex labels | Built-in support | -| **File-level details** | Not available | Full list of changed files | -| **Audit trail** | Metrics only | JSON snapshot with metadata | - ---- - -## Troubleshooting - -### JSON not being sent to gitops-status-server - -**Check:** -1. Verify gitops-status-server is running: - ```bash - kubectl get pod -n observability-stack | grep gitops-status - kubectl logs -n observability-stack - ``` - -2. Test connectivity from Woodpecker: - ```bash - # In Woodpecker log, should see: - # ==> Sending status to gitops-status-server... - # ==> Response: HTTP 200 - ``` - -3. Check network connectivity: - ```bash - # From Woodpecker container, test the endpoint: - curl http://gitops-status-server.observability-stack.svc.cluster.local:80/api/status - ``` - -### Drift not being detected - -**Check:** -1. Review drift-check.yml output in Woodpecker logs: - - Should see "✓ SYNCED" or "✗ OUT OF SYNC" - - Should see "DRIFTED_FILES=" line - -2. Manual test on server: - ```bash - ansible-playbook -i ansible/inventory/hosts.yml ansible/playbooks/drift-check.yml -v - ``` - -3. Check SSH connectivity: - ```bash - # Verify SSH key is properly set in Woodpecker - # Check server's rsyslog files are readable: - ssh user@server ls -la /etc/rsyslog.conf /etc/rsyslog.d/ - ``` - -### Cron job not running - -**Check:** -1. In Woodpecker UI: - - Go to repository settings - - Click "Cron" or look for cron job list - - Verify `gitops_sync_check` is listed with `*/2 * * * *` schedule - -2. Check cron execution history: - - Woodpecker should show execution log - - Look for "Step 1/4: Running drift-check playbook..." - -3. Manual trigger: - - In Woodpecker UI, try to manually trigger the cron job - - Check logs for errors - -### Grafana not showing data - -**Check:** -1. Verify gitops-status-server is serving JSON: - ```bash - curl http://gitops-status-server.observability-stack.svc.cluster.local:80/status.json - ``` - -2. Check Grafana Infinity datasource configuration: - - Datasource URL should point to gitops-status-server - - Test button should show "Data source is working" - -3. In dashboard: - - Edit panel - - Query should be: `GET /status.json` (or similar) - - Click "Run query" to test - - Should see JSON response with data - ---- - -## Security Considerations - -- ✓ SSH credentials in Woodpecker secrets (not exposed) -- ✓ JSON contains only file names and metadata (not config contents) -- ✓ gitops-status-server only accepts POST from authorized CI/CD -- ✓ No actual rsyslog config files are exposed -- ✓ Network communication is internal (ClusterIP) - ---- - -## Monitoring - -### Key Metrics -- **Cron execution:** Every 2 minutes -- **JSON update frequency:** Every 2 minutes (or after deployment) -- **Time to detection:** ≤ 2 minutes for manual drift -- **Time to verification:** Immediate after deployment (post-deploy step) - -### Grafana Dashboard -- **Status panel:** Shows SYNCED / OUT_OF_SYNC -- **Drift count:** Number of changed files -- **Changed files table:** Lists affected files -- **Last check:** Timestamp of last check -- **Repo/Server info:** Identifies which repo/server - ---- - -## Examples - -### Example 1: Server is synced with Git -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "SYNCED", - "drift_count": 0, - "files": [], - "last_check": "2026-04-21T10:30:00Z" -} -``` - -### Example 2: Manual edit on server -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 1, - "files": [ - { "name": "rsyslog.conf" } - ], - "last_check": "2026-04-21T10:32:00Z" -} -``` - -### Example 3: Multiple files changed -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 3, - "files": [ - { "name": "rsyslog.conf" }, - { "name": "rsyslog.d/30-lab.conf" }, - { "name": "rsyslog.d/31-remote.conf" } - ], - "last_check": "2026-04-21T10:34:00Z" -} -``` - ---- - -## Summary - -This integration provides: -- ✓ Real-time drift detection via cron (every 2 minutes) -- ✓ Post-deployment verification -- ✓ File-level granularity (which files changed) -- ✓ Integration with gitops-status-server (centralized service) -- ✓ Grafana Infinity datasource support -- ✓ Clean JSON-based architecture -- ✓ No Pushgateway dependency for this use case - -The rsyslog repository is now responsible for producing clean, structured JSON snapshots of its GitOps status, which are consumed by gitops-status-server and displayed in Grafana. diff --git a/MIGRATION_SUMMARY.md b/MIGRATION_SUMMARY.md deleted file mode 100644 index c6e3bec..0000000 --- a/MIGRATION_SUMMARY.md +++ /dev/null @@ -1,168 +0,0 @@ -# Migration Summary: Pushgateway → gitops-status-server - -This document summarizes the changes made to migrate the rsyslog repository from Pushgateway metrics to JSON status updates for gitops-status-server. - -## What Changed - -### 1. New Files Created - -#### `update-gitops-status.sh` -- Main script that orchestrates the status update flow -- Runs Ansible drift-check playbook -- Parses output to extract changed file names -- Builds structured JSON payload -- Sends JSON to gitops-status-server via HTTP POST -- Handles both SYNCED and OUT_OF_SYNC states - -#### `GITOPS_STATUS_INTEGRATION.md` -- Documentation explaining the integration with gitops-status-server -- API endpoint specification -- JSON payload format examples -- Architecture diagram -- Error handling details - -### 2. Modified Files - -#### `.woodpecker.yml` -**Changes:** -- Updated header comments to reflect new JSON status flow -- Renamed step: `update-sync-metric` → `update-gitops-status` -- Removed Pushgateway environment variable (`PUSHGATEWAY_URL`) -- Added new environment variables: - - `GITOPS_STATUS_SERVER_URL` - - `REPO_NAME` - - `SERVER_NAME` -- Added `jq` package installation for JSON formatting -- Added `bash` package to cron step (required by update-gitops-status.sh) -- Updated both `update-gitops-status` and `gitops_sync_check` steps to call new script -- Removed Pushgateway metric push logic -- Added JSON status update logic - -**Step: `update-gitops-status` (formerly `update-sync-metric`)** -- Runs after successful deployment -- Calls `update-gitops-status.sh` -- Always succeeds (status sent regardless of drift) - -**Step: `gitops_sync_check`** -- Runs on cron schedule (every 2 minutes) -- Calls `update-gitops-status.sh` to send JSON -- Then checks drift status to determine pipeline success/failure -- Pipeline fails if drift detected (for visibility in Woodpecker UI) -- JSON status always sent before checking drift - -#### `README.md` -**Changes:** -- Updated pipeline flow diagrams -- Replaced "Prometheus" with "gitops-status-server" in diagrams -- Removed "sync metric" section -- Added "GitOps status JSON format" section with examples -- Updated pipeline step descriptions to mention JSON status -- Added `update-gitops-status.sh` to repository structure -- Added optional environment variables table -- Updated flow descriptions to explain file-level drift details - -### 3. Unchanged Files - -The following files remain unchanged and continue to work as before: -- `ansible/playbooks/drift-check.yml` - Still works as-is -- `ansible/playbooks/apply.yml` - Deploy logic unchanged -- `ansible/playbooks/validate.yml` - Validation logic unchanged -- `ansible/inventory/hosts.yml` - Inventory unchanged -- `ansible/inventory/group_vars/all.yml` - Variables unchanged -- `ansible.cfg` - Ansible config unchanged -- `apply.sh` - Local apply script unchanged -- `drift-check.sh` - Local drift check script unchanged -- `files/rsyslog.conf` - Config files unchanged -- `files/rsyslog.d/30-lab.conf` - Config files unchanged - -## Behavior Changes - -### Before (Pushgateway) - -``` -drift-check → calculate status (0 or 1) → push to Pushgateway → Prometheus scrapes -``` - -Output: -``` -gitops_sync_status{repo="rsyslog",server="rsyslog-lab"} 1 -``` - -### After (gitops-status-server) - -``` -drift-check → extract changed files → build JSON → POST to gitops-status-server → Grafana queries -``` - -Output: -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 2, - "files": [ - { "name": "rsyslog.conf" }, - { "name": "rsyslog.d/30-lab.conf" } - ], - "last_check": "2026-04-21T10:32:15Z" -} -``` - -## Benefits - -1. **Richer data**: File-level drift information instead of just a binary status -2. **Better visualization**: Grafana can display which specific files have drifted -3. **Detailed tracking**: Know exactly what changed, not just that something changed -4. **Timestamp tracking**: Last check time included in JSON -5. **Drift count**: Quick numeric indicator of severity -6. **Extensible**: JSON format can be easily extended with additional fields - -## Migration Checklist - -- [x] Create `update-gitops-status.sh` script -- [x] Update `.woodpecker.yml` pipeline -- [x] Update `README.md` documentation -- [x] Create `GITOPS_STATUS_INTEGRATION.md` integration docs -- [x] Remove Pushgateway environment variables -- [x] Add gitops-status-server environment variables -- [x] Update pipeline step names -- [x] Add required packages (jq, bash) -- [x] Test JSON generation logic -- [x] Update flow diagrams - -## Testing Recommendations - -1. **Test the script locally:** - ```bash - export GITOPS_STATUS_SERVER_URL="http://localhost:80" - export REPO_NAME="rsyslog" - export SERVER_NAME="rsyslog-lab" - ./update-gitops-status.sh - ``` - -2. **Test in Woodpecker:** - - Trigger a push to master → check `update-gitops-status` step - - Wait for cron run → check `gitops_sync_check` step - - Manually edit a file on server → wait for next cron → verify OUT_OF_SYNC status - -3. **Verify gitops-status-server:** - - Check that JSON is received at POST endpoint - - Verify `/status.json` serves the latest data - - Confirm Grafana dashboard displays drift details - -## Rollback Plan - -If needed, the old Pushgateway approach can be restored by: -1. Reverting `.woodpecker.yml` to previous version -2. Removing `update-gitops-status.sh` -3. Restoring Pushgateway environment variables - -All Ansible playbooks remain unchanged, so they will work with either approach. - -## Notes - -- The rsyslog repo now focuses only on status generation and sending -- gitops-status-server is responsible for serving data to Grafana -- No changes to observability-stack app are needed on the rsyslog side -- This migration is specific to rsyslog repo; other repos can follow same pattern diff --git a/QUICK_REFERENCE.md b/QUICK_REFERENCE.md deleted file mode 100644 index 5b0d783..0000000 --- a/QUICK_REFERENCE.md +++ /dev/null @@ -1,227 +0,0 @@ -# Quick Reference: GitOps Status Server Implementation - -## What Changed - -✅ **Removed:** Pushgateway-based metrics push for sync status -✅ **Added:** JSON-based status updates to gitops-status-server -✅ **Kept:** All existing deploy/apply/drift-check logic - -## Files Modified/Created - -### Modified -- **`.woodpecker.yml`** – Updated to use `update-gitops-status.sh` instead of Pushgateway -- **`ansible/playbooks/drift-check.yml`** – Added structured file output (`DRIFTED_FILES=...`) - -### Created/Used -- **`update-gitops-status.sh`** – Main script that generates JSON and POSTs to gitops-status-server -- **`GITOPS_STATUS_SERVER_INTEGRATION.md`** – Full documentation - -## How It Works - -``` -Cron (every 2 min) or Post-Deploy - ↓ -update-gitops-status.sh - 1. Run drift-check.yml - 2. Parse output (DRIFTED_FILES=...) - 3. Generate JSON with metadata - 4. POST to gitops-status-server/api/status - ↓ -gitops-status-server receives JSON - └─ Updates /status.json internally - ↓ -Grafana Infinity datasource - └─ Queries /status.json - └─ Displays sync status, drift count, changed files -``` - -## JSON Payload Example - -### When Synced: -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "SYNCED", - "drift_count": 0, - "files": [], - "last_check": "2026-04-21T10:30:00Z" -} -``` - -### When Out of Sync: -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 2, - "files": [ - { "name": "rsyslog.conf" }, - { "name": "rsyslog.d/30-lab.conf" } - ], - "last_check": "2026-04-21T10:30:00Z" -} -``` - -## Testing - -### Test the script locally: -```bash -# Make script executable -chmod +x update-gitops-status.sh - -# Run it (requires SSH key and Ansible configured) -./update-gitops-status.sh - -# You should see: -# ==> Running drift-check playbook... -# Step 1/4: Running drift-check playbook... -# Step 2/4: Analyzing drift detection results... -# Step 3/4: Building JSON payload... -# Step 4/4: Sending status to gitops-status-server... -``` - -### Test drift-check.yml output: -```bash -# Run drift-check playbook to see new structured output -ansible-playbook -i ansible/inventory/hosts.yml ansible/playbooks/drift-check.yml - -# You should see debug output like: -# DRIFTED_FILES=/etc/rsyslog.conf,/etc/rsyslog.d/30-lab.conf -# SYNC_STATUS=OUT_OF_SYNC -``` - -### Test gitops-status-server connectivity: -```bash -# From Woodpecker container or CI environment, test endpoint -curl -X POST \ - -H "Content-Type: application/json" \ - -d '{"repo":"rsyslog","server":"rsyslog-lab","sync_status":"SYNCED","drift_count":0,"files":[],"last_check":"2026-04-21T10:30:00Z"}' \ - http://gitops-status-server.observability-stack.svc.cluster.local:80/api/status -``` - -## Deployment Steps - -### 1. Push changes to Git -```bash -git add .woodpecker.yml ansible/playbooks/drift-check.yml update-gitops-status.sh -git commit -m "refactor: replace pushgateway with gitops-status-server integration" -git push -``` - -### 2. Verify Woodpecker pipeline -- Pipeline should run automatically on push -- Check logs for successful syntax-check, validate, deploy steps -- New post-deploy step `update-gitops-status` should run after deploy - -### 3. Set up Woodpecker cron job -In Woodpecker UI: -1. Go to repository settings -2. Add Cron job: - - Name: `gitops_sync_check` - - Branch: `master` - - Schedule: `*/2 * * * *` (every 2 minutes) - -### 4. Verify cron execution -- Wait for next cron trigger (within 2 minutes) -- Check Woodpecker logs for cron execution -- Look for: "Step 1/4: Running drift-check playbook..." -- Should show: "✓ Status update successful (HTTP 200)" - -### 5. Verify gitops-status-server receives JSON -```bash -# Check gitops-status-server logs -kubectl logs -n observability-stack -l app=gitops-status-server -f - -# Should show POST requests like: -# POST /api/status from Woodpecker -``` - -### 6. Verify Grafana dashboard -- Open Grafana -- Check Infinity datasource can query gitops-status-server -- Dashboard panel should display: - - Sync status (SYNCED / OUT_OF_SYNC) - - Drift count - - List of changed files - - Last check timestamp - -## Environment Variables - -In `.woodpecker.yml`: -```yaml -environment: - GITOPS_STATUS_SERVER_URL: http://gitops-status-server.observability-stack.svc.cluster.local:80 - REPO_NAME: rsyslog - SERVER_NAME: rsyslog-lab - SSH_PRIVATE_KEY: from_secret: SSH_PRIVATE_KEY - ANSIBLE_CONFIG: ansible.cfg -``` - -## Troubleshooting - -### "HTTP 500" when posting to gitops-status-server -- Check gitops-status-server logs: - `kubectl logs -n observability-stack -l app=gitops-status-server` -- Verify gitops-status-server's API expects POST /api/status -- Check JSON format matches expected schema - -### Cron job not running -- In Woodpecker UI, check cron job list in repository settings -- Verify schedule is `*/2 * * * *` -- Check repository has write access to cron jobs - -### Drift not detected -- Run drift-check.yml manually: - `ansible-playbook -i ansible/inventory/hosts.yml ansible/playbooks/drift-check.yml -v` -- Check SSH key is properly set in Woodpecker secrets -- Verify server files are readable via SSH - -### JSON not being sent -- Check update-gitops-status.sh script is executable: - `ls -la update-gitops-status.sh` -- Check Woodpecker logs for HTTP response code -- Verify gitops-status-server URL is correct: - `curl http://gitops-status-server.observability-stack.svc.cluster.local:80` - -## Key Differences from Previous Architecture - -| Old (Pushgateway) | New (gitops-status-server) | -|-------------------|---------------------------| -| POST metric to Pushgateway | POST JSON to gitops-status-server | -| Only sync/out-of-sync (0/1) | Rich JSON with file names, count, timestamp | -| Prometheus dependency | Pure JSON API | -| Pushgateway metric format | Grafana Infinity datasource | -| Manual file-level details | Automatic file list in JSON | - -## Success Indicators - -After deployment: -- ✓ Cron job runs every 2 minutes -- ✓ update-gitops-status.sh script executes -- ✓ JSON POST to gitops-status-server returns HTTP 200 -- ✓ gitops-status-server logs show POST requests -- ✓ Grafana dashboard displays sync status and file names -- ✓ Manual changes on server detected within 2 minutes -- ✓ Deployment status updated immediately after push - -## Rollback - -If needed, revert to Pushgateway: -```bash -# Checkout old .woodpecker.yml version -git checkout HEAD~N .woodpecker.yml - -# (Where N is number of commits back) -# Or manually restore Pushgateway step -``` - -## Monitoring - -After deployment, monitor: -1. Cron execution frequency (should be every 2 minutes) -2. HTTP response codes (should be 200) -3. JSON schema consistency -4. Grafana dashboard updates -5. Time to drift detection (should be ≤ 2 minutes) diff --git a/README.md b/README.md index 23c01d8..7b621a4 100644 --- a/README.md +++ b/README.md @@ -1,228 +1,348 @@ -# rsyslog GitOps +# File Deployment & GitOps Management -Manage rsyslog configuration on Linux servers using Git as the single source of truth. -If it's not in Git, it doesn't belong on the server. +A simple, generic Ansible-based system to deploy and manage files on multiple servers using Git as the single source of truth. --- -## How it works in one sentence +## Overview -Every change goes through Git. The pipeline makes sure the server always matches what's in Git — and if someone changes the server directly, the system detects it automatically. +This repository uses **Ansible** to: +- **Deploy** files from Git to target servers +- **Check Drift** to ensure servers stay in sync with the repository +- **Validate** that deployed files are correct + +No rsyslog-specific code. Just simple file deployment that works for any file or service. --- -## The three pipelines - -### 1. Pull Request — "Is this config safe?" - -Triggered when you open or update a pull request. -Does **not** touch the live server beyond a basic reachability check. -Does **not** compare the PR content to the server (they're expected to differ before merge). +## Project Structure ``` -Open PR - │ - ├─► syntax-check Check the YAML/Ansible syntax is valid - │ - └─► validate Connect to the server and verify rsyslog is running - and the current config is loadable -``` - -**Pass** = safe to review and merge. -**Fail** = syntax error or server is unreachable / config is broken. - ---- - -### 2. Push to master — "Deploy and verify" - -Triggered when a PR is merged into master. - -``` -Merge to master - │ - ├─► syntax-check Same lint check as PR - │ - ├─► validate Same server check as PR - │ - ├─► deploy Copy the new config files from Git to the server - │ and restart rsyslog - │ - └─► update-gitops-status Run a diff between Git and the live server - │ - ├─ Matches? → send JSON (SYNCED, drift_count: 0) - └─ Differs? → send JSON (OUT_OF_SYNC, drift_count: N, files: [...]) - │ - └─ Update gitops-status-server for Grafana visualization -``` - -**Pass** = new config is live and the server matches Git. -The sync status JSON is always sent to gitops-status-server regardless of outcome. - ---- - -### 3. Cron — "Is the server still synced?" - -Runs automatically every 2 minutes, **even with no new push**. -This is the ArgoCD-style continuous check. -It only reads — never deploys anything. - -``` -Every 2 minutes (cron) - │ - └─► gitops_sync_check SSH to the server, compare every managed config - file against the latest Git commit - │ - ├─ Matches? → send JSON (SYNCED, drift_count: 0, files: []) - └─ Differs? → send JSON (OUT_OF_SYNC, drift_count: N, files: [...]) - │ - └─ Update gitops-status-server for Grafana visualization -``` - -**Why this matters:** if someone edits `/etc/rsyslog.conf` directly on the server -(bypassing Git), the next cron run catches it within 2 minutes and marks OUT_OF_SYNC -with detailed information about which specific files have drifted. - ---- - -## Full flow diagramgitops-status-server - │ │ │ │ - │── open PR ───────────────►│ │ │ - │ │── syntax-check │ │ - │ │── validate ─────────────►│ │ - │◄── PR ok / failed ────────│ │ │ - │ │ │ │ - │── merge to master ───────►│ │ │ - │ │── syntax-check │ │ - │ │── validate ─────────────►│ │ - │ │── deploy ───────────────►│ write config │ - │ │ │ restart rsyslog │ - │ │── drift-check ──────────►│ compare files │ - │ │ │◄────────────────────│ - │ │── JSON status ───────────────────────────────►│ - │ │ │ │ - │ │ [every 2 min, no push] │ │ - │ │── drift-check ──────────►│ compare files │ - │ │── JSON status ───────────────────────────────►│ - │ │ │ │ - - -Someone edits the server directly (bad): - - rogue admin Woodpecker CI Linux Server gitops-status-server - │ │ │ │ - │── ssh rsyslog-lab │ │ │ - │── vim /etc/rsyslog.conf ──────────────────────────► │ file changed │ - │ │ │ │ - │ [2 min later, cron runs] │ │ - │ │── drift-check ──────────►│ diff detected │ - │ │── JSON status (OUT_OF_SYNC)─────────────────►│ - │ │ drift_count: 1 │ Grafana shows - │ │ files: [rsyslog.conf] OUT_OF_SYNC - │ │── drift-check ──────────►│ diff detected │ - │ │── metric 0 (OUT_OF_SYNC)────────────────────►│ - │ │ │ alert fires -``` - ---- -GitOps status JSON format - -Instead of simple numeric metrics, this repo now sends rich JSON status data to gitops-status-server: - -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "SYNCED", - "drift_count": 0, - "files": [], - "last_check": "2026-04-21T10:30:00Z" -} -``` - -When drift is detected: - -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 2, - "files": [ - { "name": "rsyslog.conf" }, - { "name": "rsyslog.d/30-lab.conf" } - ], - "last_check": "2026-04-21T10:32:15Z" -} -``` - -update-gitops-status.sh Script to generate and send JSON status to gitops-status-server -This JSON is sent to `gitops-status-server` at: -- `http://gitops-status-server.observability-stack.svc.cluster.local:80/api/status` - -The gitops-status-server app serves this data via `/status.json` for Grafana Infinity datasource, -providing rich visualization with file-level drift details instead of just a numeric metric -Alert on `gitops_sync_status == 0` in Grafana/Alertmanager. - ---- - -## What drift-check actually compares - -The drift-check playbook compares files **from the Woodpecker CI container** (always the latest Git commit) against the live server. It checks: - -1. `/etc/rsyslog.conf` — must match `files/rsyslog.conf` in Git -2. `/etc/rsyslog.d/30-lab.conf` — must match `files/rsyslog.d/30-lab.conf` in Git -3. Any file managed by Git must not be missing from the server - -Files on the server that are **not** in Git (e.g. `50-default.conf`, `20-ufw.conf`) are ignored — they are owned by the OS and are not our concern. - ---- - -## Repository structure - -``` -.woodpecker.yml CI pipeline definition -ansible/ - inventory/ - hosts.yml Server list - group_vars/all.yml Variables (paths, user, etc.) - playbooks/ - validate.yml Check rsyslog is running and config loads - apply.yml Deploy config files from Git to server - drift-check.yml Compare Git files to live server (read-only) -files/ - rsyslog.conf Main rsyslog config (source of truth) - rsyslog.d/ - 30-lab.conf Drop-in config for lab logging +. +├── README.md # This file +├── ansible.cfg # Ansible configuration +├── .woodpecker.yml # CI/CD pipeline configuration +│ +├── ansible/ +│ ├── inventory/ +│ │ ├── hosts.yml # Define target servers +│ │ └── group_vars/ +│ │ └── all.yml # Global variables (SSH credentials, etc.) +│ │ +│ └── playbooks/ +│ ├── apply.yml # Deploy file to servers +│ ├── drift-check.yml # Check if servers are in sync with repo +│ └── validate.yml # Verify file exists on server +│ +└── files/ + └── dvir.txt # The file to deploy (edit this to your needs) ``` --- -## Woodpecker cron setup +## How It Works (Simple Version) -Go to **Repository Settings → Crons → Add cron**: +1. **You edit the file** in `files/dvir.txt` +2. **You commit to Git** (the source of truth) +3. **Run `apply.yml`** to deploy to all servers +4. **Run `drift-check.yml`** anytime to verify servers match Git +5. **If drift is detected**, run `apply.yml` again to fix it -| Field | Value | -|----------|---------------------| -| Name | `gitops_sync_check` | -| Branch | `master` | -| Schedule | `*/2 * * * *` | +That's it! --- -## Required secrets +## Quick Start -Go to **Repository Settings → Secrets**: +### 1. Edit the file you want to deploy -| Name | Description | -|----------------------------|-------------------------------------------------------| -| `SSH_PRIVATE_KEY` | Private key to SSH into the server | +Open `files/dvir.txt` and add your content: -## Optional environment variables +```bash +nano files/dvir.txt +``` -These can be overridden in the Woodpecker pipeline or `.woodpecker.yml`: +To deploy a **different file**, rename it or update the paths in the playbooks. -| Variable | Default | Description | -|------------------------------|--------------------------------------------------------------------------|---------------------------------------| -| `GITOPS_STATUS_SERVER_URL` | `http://gitops-status-server.observability-stack.svc.cluster.local:80` | URL of gitops-status-server API | -| `REPO_NAME` | `rsyslog` | Repository name for JSON status | -| `SERVER_NAME` | `rsyslog-lab` | Server name for JSON status | +### 2. Add target servers + +Edit `ansible/inventory/hosts.yml`: + +```yaml +all: + children: + servers: + hosts: + server1: + ansible_host: 192.168.10.161 + server2: + ansible_host: 192.168.10.162 +``` + +### 3. Run the playbooks + +#### Deploy the file: +```bash +ansible-playbook ansible/playbooks/apply.yml +``` + +#### Check if servers are in sync: +```bash +ansible-playbook ansible/playbooks/drift-check.yml +``` + +#### Validate file on server: +```bash +ansible-playbook ansible/playbooks/validate.yml +``` + +--- + +## Playbook Breakdown + +### `apply.yml` - Deploy Files +**What it does:** +- Copies `files/dvir.txt` to `/tmp/dvir.txt` on all target servers +- Sets file ownership to `root:root` with permissions `0644` + +**When to use:** +- Initial deployment of the file +- After updating the file in Git + +**Example:** +```bash +ansible-playbook ansible/playbooks/apply.yml +``` + +--- + +### `drift-check.yml` - Detect Configuration Drift +**What it does:** +- Reads the file from the Git repository (local) +- Reads the file from each target server (`/tmp/dvir.txt`) +- Compares the content byte-for-byte +- Reports `SYNCED` or `OUT_OF_SYNC` + +**When to use:** +- Verify servers match the repository state +- Detect if someone manually changed the file on the server +- Run periodically (via cron or CI/CD) to monitor compliance + +**Example:** +```bash +ansible-playbook ansible/playbooks/drift-check.yml +``` + +**Output:** +``` +✓ dvir.txt is synced # Files match +✗ dvir.txt is out of sync # Files differ or file missing +``` + +--- + +### `validate.yml` - Validate Deployment +**What it does:** +- Checks that `/tmp/dvir.txt` exists on the server +- Verifies the file is readable +- Fails if the file is missing or not readable + +**When to use:** +- After running `apply.yml` to verify success +- To confirm the deployment was successful + +**Example:** +```bash +ansible-playbook ansible/playbooks/validate.yml +``` + +--- + +## Configuration + +### Inventory: `ansible/inventory/hosts.yml` +Define which servers to manage: + +```yaml +all: + children: + servers: + hosts: + server1: + ansible_host: 192.168.10.161 + server2: + ansible_host: 192.168.10.162 +``` + +### Global Variables: `ansible/inventory/group_vars/all.yml` +SSH and connection settings: + +```yaml +ansible_user: root +ansible_ssh_private_key_file: "~/.ssh/id_rsa" +ansible_ssh_common_args: "-o StrictHostKeyChecking=no" +``` + +### Ansible Config: `ansible.cfg` +Global Ansible settings (host checking, plugins, etc.) + +--- + +## Workflow Examples + +### Example 1: Deploy a new file + +```bash +# 1. Edit the file +echo "new config content" > files/dvir.txt + +# 2. Commit to Git +git add files/dvir.txt +git commit -m "Update deployment file" +git push + +# 3. Deploy to servers +ansible-playbook ansible/playbooks/apply.yml + +# 4. Verify success +ansible-playbook ansible/playbooks/validate.yml +``` + +### Example 2: Monitor for drift (continuous compliance) + +```bash +# Run drift check periodically (cron job) +0 */6 * * * cd /path/to/repo && ansible-playbook ansible/playbooks/drift-check.yml +``` + +### Example 3: Detect and fix manual changes + +```bash +# Someone manually edited /tmp/dvir.txt on server1 +# Check for drift +ansible-playbook ansible/playbooks/drift-check.yml +# Output: ✗ dvir.txt is out of sync + +# Restore from Git +ansible-playbook ansible/playbooks/apply.yml + +# Verify it's fixed +ansible-playbook ansible/playbooks/drift-check.yml +# Output: ✓ dvir.txt is synced +``` + +--- + +## Prerequisites + +- **Ansible** installed on your machine +- **SSH access** to all target servers (password or key-based) +- **Root or sudo access** on target servers (for writing to `/tmp`) + +### Install Ansible + +```bash +# macOS +brew install ansible + +# Ubuntu/Debian +apt-get install ansible + +# RHEL/CentOS +yum install ansible + +# Python pip +pip install ansible +``` + +--- + +## Customization + +### Deploy to a different path + +Edit `ansible/playbooks/apply.yml`: + +```yaml +- name: Copy file to destination + copy: + src: ../../files/dvir.txt + dest: /your/custom/path/filename.txt # ← Change this + owner: root + group: root + mode: "0644" +``` + +### Deploy multiple files + +Add more tasks to `apply.yml`: + +```yaml +- name: Copy file 1 + copy: + src: ../../files/dvir.txt + dest: /tmp/dvir.txt + +- name: Copy file 2 + copy: + src: ../../files/another.txt + dest: /tmp/another.txt +``` + +### Use different target servers + +Edit `hosts.yml` and use: + +```bash +ansible-playbook ansible/playbooks/apply.yml -i ansible/inventory/hosts.yml --limit "server2" +``` + +--- + +## Troubleshooting + +### "SSH connection refused" +- Check `ansible_host` is correct in `hosts.yml` +- Verify SSH key in `group_vars/all.yml` +- Test manually: `ssh -i ~/.ssh/id_rsa root@192.168.10.161` + +### "Permission denied" on deployment +- Ensure `become: true` is in the playbook +- Verify user has sudo access +- Check SSH key has correct permissions: `chmod 600 ~/.ssh/id_rsa` + +### Drift check shows "out of sync" but I didn't change anything +- File permissions or ownership might have changed +- Line endings (CRLF vs LF) might differ +- The server file might be missing + +### Can't read file on server +- Check `/tmp/dvir.txt` exists: `ls -la /tmp/dvir.txt` +- Verify permissions: `stat /tmp/dvir.txt` +- Ensure file is readable: `cat /tmp/dvir.txt` + +--- + +## Tips & Best Practices + +1. **Always commit before deploying** + - Use Git as the single source of truth + - Never manually edit `/tmp/dvir.txt` on servers + +2. **Run drift-check regularly** + - Use cron or CI/CD to monitor compliance + - Alert on `OUT_OF_SYNC` status + +3. **Test in dev first** + - Add a `dev` group in `hosts.yml` + - Test playbooks on dev servers before prod + +4. **Use version control for everything** + - Keep all changes in Git + - Easy rollback: just revert and re-run `apply.yml` + +--- + +## License + +MIT (or your preferred license) diff --git a/README_GITOPS_STATUS.md b/README_GITOPS_STATUS.md deleted file mode 100644 index 183d304..0000000 --- a/README_GITOPS_STATUS.md +++ /dev/null @@ -1,462 +0,0 @@ -# rsyslog GitOps: gitops-status-server Integration - -## Overview - -This repository now uses **gitops-status-server** (a Kubernetes-based service) for GitOps status monitoring, replacing Prometheus Pushgateway. - -**Status:** ✅ Ready for deployment - ---- - -## What This Means - -### For You (Operator) -- Configure Woodpecker cron job once (5 minutes) -- Cron runs every 2 minutes and checks if rsyslog config on server matches Git -- If changes detected, JSON is automatically sent to gitops-status-server -- Grafana shows: - - **Sync status** (green=SYNCED, red=OUT_OF_SYNC) - - **Drift count** (how many files changed) - - **Changed files** (list of which files are different) - - **Last check** (when drift was last checked) - -### For Grafana -- Connects to gitops-status-server via Infinity datasource -- Reads JSON status snapshots -- Displays rich metadata about rsyslog configuration state - -### For the System -- Detects manual edits on the server within 2 minutes -- Verifies deployment succeeded immediately after push -- No Pushgateway required -- Clean JSON-based architecture - ---- - -## Quick Start (5 Steps) - -### Step 1: Verify Files Are In Place -```bash -# Check these files exist: -ls -la .woodpecker.yml # ✓ Updated -ls -la ansible/playbooks/drift-check.yml # ✓ Enhanced -ls -la update-gitops-status.sh # ✓ New script -``` - -### Step 2: Make Script Executable -```bash -chmod +x update-gitops-status.sh -git add update-gitops-status.sh -git commit -m "add: gitops-status-server integration script" -git push -``` - -### Step 3: Verify Pipeline Runs -- Push should trigger Woodpecker pipeline -- New step `update-gitops-status` should execute after `deploy` -- Check logs for: "✓ Status update successful" - -### Step 4: Configure Woodpecker Cron Job -In Woodpecker UI: -1. Go to repository -2. Click **Settings** → **Cron** -3. Click **Add Cron** -4. Fill in: - - **Name:** `gitops_sync_check` - - **Branch:** `master` - - **Schedule:** `*/2 * * * *` -5. Click **Save** - -### Step 5: Test Cron Job -- Wait max 2 minutes for cron to trigger -- Check Woodpecker logs for cron execution -- Verify gitops-status-server received POST: - ```bash - kubectl logs -n observability-stack -l app=gitops-status-server | grep POST - ``` - ---- - -## How It Works - -### Architecture Diagram -``` -Every 2 minutes (or after deployment): - -Woodpecker -├─ gitops_sync_check or update-gitops-status step -└─ Runs: update-gitops-status.sh - ├─ 1. Execute: ansible/playbooks/drift-check.yml - │ └─ Compare Git files vs server files (read-only) - │ └─ Output: DRIFTED_FILES=file1,file2,file3 - ├─ 2. Parse: Extract changed files and sync status - ├─ 3. Generate: JSON payload with metadata - └─ 4. POST: JSON to gitops-status-server/api/status - -gitops-status-server (K8s Service) -└─ Receives JSON -└─ Updates /status.json -└─ Serves via HTTP - -Grafana -└─ Infinity datasource -└─ Polls /status.json (periodic refresh) -└─ Displays in dashboard panel -``` - -### Data Flow - -**Input:** Ansible drift-check output -``` -DRIFTED_FILES=/etc/rsyslog.conf,/etc/rsyslog.d/30-lab.conf -SYNC_STATUS=OUT_OF_SYNC -``` - -**Output:** JSON sent to gitops-status-server -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 2, - "files": [ - { "name": "rsyslog.conf" }, - { "name": "rsyslog.d/30-lab.conf" } - ], - "last_check": "2026-04-21T10:30:00Z" -} -``` - -**Display:** Grafana dashboard -- 🔴 OUT_OF_SYNC (red card) -- Drift count: 2 -- Files: rsyslog.conf, rsyslog.d/30-lab.conf -- Last check: 2026-04-21 10:30 UTC - ---- - -## Files Changed - -### `.woodpecker.yml` -**Changes:** -- Renamed step: `update-sync-metric` → `update-gitops-status` -- Changed command: replaced Pushgateway push → `update-gitops-status.sh` script call -- Added environment variables: - - `GITOPS_STATUS_SERVER_URL` - - `REPO_NAME` - - `SERVER_NAME` -- Both `update-gitops-status` (post-deploy) and `gitops_sync_check` (cron) now use the script - -### `ansible/playbooks/drift-check.yml` -**Changes:** -- Added file collection: builds list of changed files in `drifted_files` fact -- Added debug output: prints `DRIFTED_FILES=file1,file2,file3` for script parsing -- Added status markers: prints `SYNC_STATUS=SYNCED` or `SYNC_STATUS=OUT_OF_SYNC` -- **No changes to drift detection logic** (fully backward compatible) - -### `update-gitops-status.sh` (NEW) -**Purpose:** Orchestrates status generation and delivery -**4 Steps:** -1. Run drift-check.yml -2. Parse output to extract changed files -3. Build JSON payload -4. POST to gitops-status-server/api/status - ---- - -## Testing - -### Test 1: Verify script works locally -```bash -# From repo root, with SSH key configured -./update-gitops-status.sh - -# Expected output: -# ═══════════════════════════════════════════════════ -# Step 1/4: Running drift-check playbook... -# [playbook output...] -# Step 2/4: Analyzing drift detection results... -# Step 3/4: Building JSON payload... -# Generated JSON: -# { -# "repo": "rsyslog", -# ... -# } -# Step 4/4: Sending status to gitops-status-server... -# Response: HTTP 200 -# ✓ Status update successful -``` - -### Test 2: Verify Woodpecker pipeline runs -1. Make a change to a file -2. Push to master branch -3. Woodpecker pipeline should: - - Run syntax-check ✓ - - Run validate ✓ - - Run deploy ✓ - - Run update-gitops-status ✓ -4. Check logs for: "✓ Status update successful" - -### Test 3: Verify cron job triggers -1. Woodpecker cron job configured for `*/2 * * * *` -2. Wait 2 minutes -3. Check Woodpecker UI for cron execution -4. Check logs for drift-check output - -### Test 4: Verify gitops-status-server receives JSON -```bash -# Check gitops-status-server logs -kubectl logs -n observability-stack -l app=gitops-status-server -f - -# Should show POST requests: -# POST /api/status - 200 OK -``` - -### Test 5: Verify Grafana dashboard -1. Open Grafana -2. Check Infinity datasource: - - Should show "Data source is working" -3. Check dashboard panel: - - Should display sync status - - Should show drift count - - Should list changed files (if any) - ---- - -## Troubleshooting - -### Issue: Cron job not running -**Check:** -1. Is cron job configured in Woodpecker? - - Go to repo settings → Cron - - Should see `gitops_sync_check` with `*/2 * * * *` schedule -2. Is the schedule active? - - Cron should have triggered at least once - -**Fix:** -- Create cron job if missing -- Verify schedule is `*/2 * * * *` -- Check branch is `master` - -### Issue: "HTTP 500" or "HTTP 503" when posting -**Check:** -1. Is gitops-status-server running? - ```bash - kubectl get pod -n observability-stack | grep gitops-status - ``` -2. Is it ready? - ```bash - kubectl get pod -n observability-stack -o wide | grep gitops-status - ``` -3. Check logs: - ```bash - kubectl logs -n observability-stack -l app=gitops-status-server - ``` - -**Fix:** -- Restart gitops-status-server if needed -- Check error logs for API issues -- Verify /api/status endpoint exists - -### Issue: Drift not detected -**Check:** -1. Run drift-check manually: - ```bash - ansible-playbook -i ansible/inventory/hosts.yml ansible/playbooks/drift-check.yml -v - ``` -2. Does it report the correct status? -3. SSH connectivity to server? - -**Fix:** -- Check SSH key is set in Woodpecker secrets -- Verify server files are readable: `ssh user@server ls -la /etc/rsyslog.conf` -- Check Ansible inventory is correct - -### Issue: JSON not sent (HTTP 000) -**Check:** -1. Is gitops-status-server URL correct? - ```bash - curl http://gitops-status-server.observability-stack.svc.cluster.local:80 - ``` -2. Can Woodpecker reach it? - - May be network/DNS issue - -**Fix:** -- Check gitops-status-server hostname/port -- Test from Woodpecker container: `curl http://gitops-status-server:80` -- Check Woodpecker network policies - -### Issue: Grafana shows "No data" -**Check:** -1. Does Infinity datasource work? - - Go to Data Sources → test -2. Can Grafana reach gitops-status-server? -3. What query is the panel using? - -**Fix:** -- Verify datasource URL is correct -- Check query in panel: should be `/status.json` or similar -- Ensure gitops-status-server is returning JSON - ---- - -## Examples - -### Example 1: Server is synced -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "SYNCED", - "drift_count": 0, - "files": [], - "last_check": "2026-04-21T10:32:00Z" -} -``` - -**Grafana display:** -- 🟢 SYNCED -- Drift: 0 -- Files: (empty) - -### Example 2: Manual edit on server -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 1, - "files": [ - { "name": "rsyslog.conf" } - ], - "last_check": "2026-04-21T10:34:00Z" -} -``` - -**Grafana display:** -- 🔴 OUT OF SYNC -- Drift: 1 -- Files: rsyslog.conf - -### Example 3: Multiple files changed after deployment -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "SYNCED", - "drift_count": 0, - "files": [], - "last_check": "2026-04-21T10:36:00Z" -} -``` - -**Timeline:** -1. 10:30 - Push to master triggers deploy -2. 10:31 - Deploy completes, files changed -3. 10:31 - update-gitops-status runs, verifies sync -4. 10:31 - JSON sent: SYNCED -5. 10:36 - Grafana shows ✓ SYNCED (5 min later) - ---- - -## Environment Configuration - -The following environment variables are set in `.woodpecker.yml`: - -```yaml -GITOPS_STATUS_SERVER_URL: http://gitops-status-server.observability-stack.svc.cluster.local:80 -REPO_NAME: rsyslog -SERVER_NAME: rsyslog-lab -SSH_PRIVATE_KEY: from_secret: SSH_PRIVATE_KEY -ANSIBLE_CONFIG: ansible.cfg -``` - -**To customize:** -- Edit `.woodpecker.yml` -- Change environment variables under `update-gitops-status` and `gitops_sync_check` steps -- Push and re-run - ---- - -## Key Features - -✅ **Automatic drift detection** – Every 2 minutes -✅ **Post-deployment verification** – Immediate after deploy -✅ **File-level details** – Shows which files changed -✅ **No Pushgateway** – Simplified infrastructure -✅ **Grafana integration** – Infinity datasource (native) -✅ **Audit trail** – JSON snapshots with timestamps -✅ **Multi-server ready** – Structured for scale - ---- - -## Documentation - -| Document | Purpose | -|----------|---------| -| `GITOPS_STATUS_SERVER_INTEGRATION.md` | Comprehensive architecture & flow | -| `QUICK_REFERENCE.md` | Quick start & troubleshooting | -| `REFACTOR_SUMMARY.md` | Before/after comparison | -| This file | Overview & quick start | - ---- - -## Next Steps - -1. **Verify files are in place:** - ```bash - git status - ``` - -2. **Push changes:** - ```bash - git push - ``` - -3. **Monitor first pipeline run:** - - Check Woodpecker logs - - Look for `update-gitops-status` step - - Verify HTTP 200 response - -4. **Configure cron job:** - - Go to Woodpecker UI - - Add cron: `gitops_sync_check` at `*/2 * * * *` - -5. **Test cron execution:** - - Wait 2 minutes - - Check Woodpecker logs - - Verify gitops-status-server receives JSON - -6. **Verify Grafana:** - - Check dashboard displays sync status - - Test with manual file edit on server - - Verify detection within 2 minutes - ---- - -## Support - -For issues or questions: -1. Check `QUICK_REFERENCE.md` troubleshooting section -2. Review Woodpecker pipeline logs -3. Check gitops-status-server application logs: - ```bash - kubectl logs -n observability-stack -l app=gitops-status-server -f - ``` -4. Test connectivity manually: - ```bash - curl http://gitops-status-server.observability-stack.svc.cluster.local:80/api/status - ``` - ---- - -## Summary - -You now have a clean, production-ready GitOps status monitoring system that: -- Detects configuration drift every 2 minutes -- Sends rich metadata (file names, timestamps) to gitops-status-server -- Integrates with Grafana via Infinity datasource -- Requires minimal infrastructure (no Pushgateway) -- Works reliably for multi-server deployments - -**Status:** ✅ Ready to deploy and use diff --git a/REFACTOR_SUMMARY.md b/REFACTOR_SUMMARY.md deleted file mode 100644 index 07ea59d..0000000 --- a/REFACTOR_SUMMARY.md +++ /dev/null @@ -1,382 +0,0 @@ -# Implementation Summary: Pushgateway → gitops-status-server - -## Status: ✅ Complete - -This document summarizes the refactoring of the rsyslog GitOps monitoring flow to use a centralized gitops-status-server instead of Pushgateway. - ---- - -## What Was Replaced - -### Old Architecture (Pushgateway-based) -``` -Drift-check runs - ↓ -Exit code: 0 (synced) or 1 (drift) - ↓ -Send metric to Pushgateway - ↓ -Prometheus scrapes Pushgateway - ↓ -gitops_sync_status{repo="rsyslog",server="rsyslog-lab"} = 0 or 1 - ↓ -Grafana queries Prometheus - ↓ -Dashboard shows only: SYNCED or OUT_OF_SYNC -``` - -**Limitations:** -- Only 0/1 metric (no file-level details) -- Requires Pushgateway, Prometheus infrastructure -- Cannot show which files changed - ---- - -### New Architecture (gitops-status-server) -``` -Drift-check runs + outputs DRIFTED_FILES=... - ↓ -update-gitops-status.sh script: - 1. Parse changed files - 2. Generate JSON - 3. POST to gitops-status-server - ↓ -gitops-status-server - ↓ -Serves /status.json with rich metadata - ↓ -Grafana Infinity datasource reads /status.json - ↓ -Dashboard shows: - - Sync status - - Drift count - - List of changed files - - Last check timestamp -``` - -**Advantages:** -- ✓ Rich metadata (file-level details) -- ✓ No Pushgateway/Prometheus for this use case -- ✓ Centralized gitops-status-server -- ✓ Easier to audit (JSON snapshot) -- ✓ Better for multi-server/multi-repo - ---- - -## Files Changed - -### 1. `.woodpecker.yml` (MAJOR UPDATE) - -#### Before (Pushgateway): -```yaml -update-sync-metric: - commands: - - printf 'gitops_sync_status{repo="rsyslog",server="rsyslog-lab"} %s\n' "$STATUS" | \ - curl ... --data-binary @- "$PUSHGATEWAY_URL/metrics/job/gitops_rsyslog/..." -``` - -#### After (gitops-status-server): -```yaml -update-gitops-status: - commands: - - chmod +x update-gitops-status.sh - - ./update-gitops-status.sh - environment: - GITOPS_STATUS_SERVER_URL: http://gitops-status-server.observability-stack.svc.cluster.local:80 - REPO_NAME: rsyslog - SERVER_NAME: rsyslog-lab -``` - -**Changes:** -- Removed `PUSHGATEWAY_URL` environment variable -- Removed metric push command -- Added script execution -- Added `GITOPS_STATUS_SERVER_URL` configuration -- Both `update-gitops-status` and `gitops_sync_check` steps now use the script - ---- - -### 2. `ansible/playbooks/drift-check.yml` (ADDED OUTPUT) - -#### Before: -```yaml -- name: Fail if drift detected - ansible.builtin.fail: - msg: "Configuration drift detected..." - when: drift_detected -``` - -#### After (ADDED before the fail task): -```yaml -# New: Build structured list of changed files -- name: Initialize list of drifted files - ansible.builtin.set_fact: - drifted_files: [] - -- name: Add main config to drifted files if changed - ansible.builtin.set_fact: - drifted_files: "{{ drifted_files + ['/etc/rsyslog.conf'] }}" - when: main_config_check.changed - -# ... (more file collection tasks) - -# New: Output structured markers for parsing -- name: Output structured list of drifted files - ansible.builtin.debug: - msg: "DRIFTED_FILES={{ drifted_files | join(',') }}" - when: drift_detected - -- name: Output sync status marker - ansible.builtin.debug: - msg: "SYNC_STATUS=OUT_OF_SYNC" - when: drift_detected -``` - -**Changes:** -- Builds list of drifted files in `drifted_files` fact -- Outputs `DRIFTED_FILES=file1,file2,file3` for script parsing -- Outputs `SYNC_STATUS=SYNCED` or `SYNC_STATUS=OUT_OF_SYNC` markers -- Original drift detection logic unchanged - ---- - -### 3. `update-gitops-status.sh` (CORE SCRIPT) - -**New file created:** Orchestrates the entire flow - -**Key functionality:** -1. Runs `drift-check.yml` playbook -2. Captures output to temp file -3. Parses `DRIFTED_FILES=...` and `SYNC_STATUS=...` markers -4. Extracts changed file names -5. Converts `/etc/rsyslog.conf` → `rsyslog.conf` (relative paths) -6. Generates JSON with metadata -7. POSTs JSON to gitops-status-server API - -**4-step process:** -``` -Step 1/4: Running drift-check playbook... -Step 2/4: Analyzing drift detection results... -Step 3/4: Building JSON payload... -Step 4/4: Sending status to gitops-status-server... -``` - ---- - -## Generated JSON Format - -### Synced State: -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "SYNCED", - "drift_count": 0, - "files": [], - "last_check": "2026-04-21T10:30:00Z" -} -``` - -### Out of Sync State: -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 2, - "files": [ - { "name": "rsyslog.conf" }, - { "name": "rsyslog.d/30-lab.conf" } - ], - "last_check": "2026-04-21T10:30:00Z" -} -``` - ---- - -## Data Flow Example - -### Scenario: Manual edit on server - -1. **Manual change:** Someone edits `/etc/rsyslog.conf` directly on server -2. **Cron trigger:** Scheduled cron job runs (every 2 minutes) -3. **Woodpecker step:** `gitops_sync_check` executes `update-gitops-status.sh` -4. **Drift detection:** `drift-check.yml` runs and detects change -5. **Output parsing:** Script extracts: - - `DRIFTED_FILES=/etc/rsyslog.conf` - - `SYNC_STATUS=OUT_OF_SYNC` -6. **JSON generation:** - ```json - { - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 1, - "files": [{ "name": "rsyslog.conf" }], - "last_check": "2026-04-21T10:32:00Z" - } - ``` -7. **API POST:** Script POSTs JSON to: - - URL: `http://gitops-status-server.observability-stack.svc.cluster.local:80/api/status` - - Method: POST - - Content-Type: application/json -8. **Server update:** gitops-status-server receives JSON and updates `/status.json` -9. **Grafana update:** Infinity datasource refreshes and displays new status -10. **Result:** Dashboard shows OUT_OF_SYNC with rsyslog.conf listed - -**Time to detection:** ≤ 2 minutes - ---- - -## Integration Points - -### Woodpecker Events Handled - -1. **Pull Request:** - - syntax-check → validate (no drift check) - - No gitops-status update - -2. **Push to Master:** - - syntax-check → validate → deploy → **update-gitops-status** - - After deployment, immediately verify sync and update status - -3. **Scheduled Cron:** - - **gitops_sync_check** (every 2 minutes by default) - - Continuous drift monitoring - ---- - -## Configuration - -### Required Environment Variables -```yaml -GITOPS_STATUS_SERVER_URL: http://gitops-status-server.observability-stack.svc.cluster.local:80 -REPO_NAME: rsyslog -SERVER_NAME: rsyslog-lab -SSH_PRIVATE_KEY: from_secret: SSH_PRIVATE_KEY -ANSIBLE_CONFIG: ansible.cfg -``` - -### Cron Job Setup (Woodpecker UI) -- Name: `gitops_sync_check` -- Branch: `master` -- Schedule: `*/2 * * * *` - ---- - -## Backward Compatibility - -- ✓ **Existing deploy logic:** Unchanged (apply.yml still used) -- ✓ **Existing drift detection:** Enhanced (now outputs file names) -- ✓ **PR validation:** Unchanged (syntax-check, validate still used) -- ✓ **Server files:** No changes needed - ---- - -## Security - -- ✓ SSH credentials in Woodpecker secrets (not exposed) -- ✓ JSON contains only metadata (file names, counts, timestamps) -- ✓ No actual rsyslog config contents exposed -- ✓ Internal Kubernetes communication (ClusterIP) -- ✓ No Pushgateway exposure - ---- - -## Testing Checklist - -- [ ] Cron job is created in Woodpecker -- [ ] Cron job runs on schedule (every 2 minutes) -- [ ] `update-gitops-status.sh` script is executable -- [ ] Script runs successfully (HTTP 200 response) -- [ ] gitops-status-server receives JSON POSTs -- [ ] JSON format matches expected schema -- [ ] Grafana dashboard displays sync status -- [ ] Changed files appear in Grafana panel -- [ ] Manual file edit on server is detected -- [ ] Post-deployment status updates correctly - ---- - -## Migration Steps - -1. **Commit and push changes:** - ```bash - git add .woodpecker.yml ansible/playbooks/drift-check.yml update-gitops-status.sh - git commit -m "refactor: replace pushgateway with gitops-status-server" - git push - ``` - -2. **Verify pipeline runs successfully** - - Check Woodpecker logs for new steps - -3. **Create Woodpecker cron job** - - Name: gitops_sync_check - - Schedule: */2 * * * * - -4. **Test cron execution** - - Wait for cron trigger (within 2 minutes) - - Verify JSON is sent to gitops-status-server - -5. **Verify Grafana dashboard** - - Confirm Infinity datasource can read gitops-status-server - - Dashboard shows sync status and changed files - -6. **Monitor for 24 hours** - - Verify cron runs consistently - - Check for any HTTP errors - - Confirm drift detection works - -7. **Decommission Pushgateway** (when confident) - - Stop sending metrics to Pushgateway - - Remove Pushgateway from infrastructure - ---- - -## Rollback Plan - -If issues arise: - -1. **Revert Woodpecker changes:** - ```bash - git revert - git push - ``` - -2. **Remove cron job:** - - Delete gitops_sync_check from Woodpecker UI - -3. **Restore Pushgateway metric push** (if keeping Prometheus monitoring) - ---- - -## Key Improvements - -| Metric | Old | New | -|--------|-----|-----| -| Data richness | 0/1 only | JSON with file names | -| Setup complexity | Pushgateway + Prometheus | Single service call | -| Audit trail | Basic | Structured snapshots | -| File-level visibility | None | Complete list | -| Update frequency | After deployment | Every 2 minutes + post-deploy | -| Infrastructure | 2+ services | 1 service (gitops-status-server) | - ---- - -## Documentation Files - -1. **`GITOPS_STATUS_SERVER_INTEGRATION.md`** – Comprehensive documentation -2. **`QUICK_REFERENCE.md`** – Quick start and troubleshooting -3. **`IMPLEMENTATION_SUMMARY.md`** – This file - ---- - -## Support - -For issues, consult: -1. `.woodpecker.yml` comments -2. `update-gitops-status.sh` comments -3. `drift-check.yml` comments -4. Full documentation in GITOPS_STATUS_SERVER_INTEGRATION.md -5. Woodpecker pipeline logs -6. gitops-status-server application logs diff --git a/ansible/inventory/group_vars/all.yml b/ansible/inventory/group_vars/all.yml index 9403f44..94acacf 100644 --- a/ansible/inventory/group_vars/all.yml +++ b/ansible/inventory/group_vars/all.yml @@ -1,17 +1,7 @@ --- -# Global variables for rsyslog configuration management +# Global variables for deployment # Ansible connection settings ansible_user: root ansible_ssh_private_key_file: "~/.ssh/id_rsa" ansible_ssh_common_args: "-o StrictHostKeyChecking=no" - -# Root directory of the rsyslog repository -repo_root: /root/rsyslog - -# rsyslog service name -rsyslog_service: rsyslog - -# Configuration paths -rsyslog_main_config: /etc/rsyslog.conf -rsyslog_config_dir: /etc/rsyslog.d diff --git a/ansible/inventory/hosts.yml b/ansible/inventory/hosts.yml index 90bbcc5..0d45f59 100644 --- a/ansible/inventory/hosts.yml +++ b/ansible/inventory/hosts.yml @@ -1,12 +1,12 @@ --- all: children: - rsyslog_servers: + servers: hosts: - rsyslog-lab: + server1: ansible_host: 192.168.10.161 - # Future servers can be added here: - # rsyslog-prod: + # Add more servers here: + # server2: # ansible_host: 192.168.10.162 - # rsyslog-backup: + # server3: # ansible_host: 192.168.10.163 diff --git a/ansible/playbooks/apply.yml b/ansible/playbooks/apply.yml index f830835..a2e6a63 100644 --- a/ansible/playbooks/apply.yml +++ b/ansible/playbooks/apply.yml @@ -1,151 +1,31 @@ --- -- name: Apply rsyslog configuration (safe staged deployment) - hosts: rsyslog_servers +# ============================================================================= +# APPLY PLAYBOOK +# Purpose: Deploy dvir.txt file to target servers at /tmp/dvir.txt +# Usage: ansible-playbook apply.yml +# ============================================================================= + +- name: Deploy file to servers + hosts: all become: true - vars: - backup_dir: /var/backups/rsyslog-ansible - backup_conf: "{{ backup_dir }}/rsyslog.conf.bak" - backup_confd: "{{ backup_dir }}/rsyslog.d.bak" - tasks: - - # ------------------------------------------------------------------------- - # STAGE 1 — Backup current working configuration - # ------------------------------------------------------------------------- - - - name: Ensure backup directory exists - file: - path: "{{ backup_dir }}" - state: directory - mode: "0700" - - - name: Backup current rsyslog.conf + # ───────────────────────────────────────────────────────────────────── + # TASK 1: Copy file to destination + # Copies the dvir.txt from the repo to /tmp/dvir.txt on target servers + # ───────────────────────────────────────────────────────────────────── + - name: Copy file to destination copy: - src: "{{ rsyslog_main_config }}" - dest: "{{ backup_conf }}" - remote_src: true - mode: "0600" - - - name: Remove stale rsyslog.d backup - file: - path: "{{ backup_confd }}" - state: absent - - - name: Backup current rsyslog.d directory - copy: - src: "{{ rsyslog_config_dir }}/" - dest: "{{ backup_confd }}/" - remote_src: true - - # ------------------------------------------------------------------------- - # STAGE 2 — Deploy new configuration files from repo - # ------------------------------------------------------------------------- - - - name: Copy new rsyslog.conf from repo - copy: - src: ../../files/rsyslog.conf - dest: "{{ rsyslog_main_config }}" + src: ../../files/dvir.txt + dest: /tmp/dvir.txt owner: root group: root mode: "0644" - # ------------------------------------------------------------------------- - # Clean rsyslog.d directory to ensure exact Git sync (remove old files) - # ------------------------------------------------------------------------- - - name: Remove all existing rsyslog.d config files - shell: rm -f {{ rsyslog_config_dir }}/*.conf - - - name: Copy new rsyslog.d configs from repo - copy: - src: ../../files/rsyslog.d/ - dest: "{{ rsyslog_config_dir }}/" - owner: root - group: root - mode: "0644" - - # ------------------------------------------------------------------------- - # STAGE 3 — Validate against the full real config tree on the remote host - # Runs rsyslogd -N1 against the actual /etc/rsyslog.conf so all includes, - # modules, and templates are resolved in the real environment. - # ------------------------------------------------------------------------- - - - name: Validate new configuration on remote host - command: rsyslogd -N1 -f "{{ rsyslog_main_config }}" - register: validation_result - changed_when: false - failed_when: false # We handle failure manually below - - # ------------------------------------------------------------------------- - # STAGE 4a — Validation FAILED: restore backup and abort - # ------------------------------------------------------------------------- - - - name: Print validation error output + # ───────────────────────────────────────────────────────────────────── + # TASK 2: Confirm deployment success + # Displays success message with the hostname for verification + # ───────────────────────────────────────────────────────────────────── + - name: Confirm deployment debug: - msg: | - ################################################## - ❌ RSYSLOG VALIDATION FAILED - ################################################## - - HOST : {{ inventory_hostname }} ({{ ansible_host }}) - RC : {{ validation_result.rc }} - - --- STDOUT ---------------------------------- - {{ validation_result.stdout | default('(empty)') }} - - --- STDERR ---------------------------------- - {{ validation_result.stderr | default('(empty)') }} - - ################################################## - ⚠ Rolling back to previous working configuration - ################################################## - when: validation_result.rc != 0 - - - name: Restore rsyslog.conf from backup - copy: - src: "{{ backup_conf }}" - dest: "{{ rsyslog_main_config }}" - remote_src: true - mode: "0644" - when: validation_result.rc != 0 - - - name: Restore rsyslog.d from backup - copy: - src: "{{ backup_confd }}/" - dest: "{{ rsyslog_config_dir }}/" - remote_src: true - when: validation_result.rc != 0 - - - name: Fail pipeline — config restored to previous working state - fail: - msg: | - ################################################## - ❌ PIPELINE FAILED — rsyslog validation error - ################################################## - Previous working config has been restored. - rsyslog was NOT restarted. - rc={{ validation_result.rc }} - stderr: {{ validation_result.stderr | default('(empty)') }} - ################################################## - when: validation_result.rc != 0 - - # ------------------------------------------------------------------------- - # STAGE 4b — Validation PASSED: restart rsyslog and report success - # ------------------------------------------------------------------------- - - - name: Restart rsyslog service - service: - name: "{{ rsyslog_service }}" - state: restarted - when: validation_result.rc == 0 - - - name: Print success status - debug: - msg: | - ################################################## - ✅ RSYSLOG CONFIGURATION DEPLOYED SUCCESSFULLY - ################################################## - HOST : {{ inventory_hostname }} ({{ ansible_host }}) - STATUS : Validation passed. Service restarted. - ################################################## - when: validation_result.rc == 0 \ No newline at end of file + msg: "✅ File deployed successfully to /tmp/dvir.txt on {{ inventory_hostname }}" \ No newline at end of file diff --git a/ansible/playbooks/drift-check.yml b/ansible/playbooks/drift-check.yml index 2d4db99..a9d805b 100644 --- a/ansible/playbooks/drift-check.yml +++ b/ansible/playbooks/drift-check.yml @@ -1,151 +1,72 @@ --- -- name: Check rsyslog configuration drift - hosts: rsyslog_servers +# ============================================================================= +# DRIFT-CHECK PLAYBOOK +# Purpose: Compare file on repo vs server to detect if they're in sync +# Usage: ansible-playbook drift-check.yml +# Output: SYNCED or OUT_OF_SYNC status +# ============================================================================= + +- name: Check file drift + hosts: all gather_facts: false - # SIMPLE FILE CONTENT COMPARISON - No permission checks - # Uses slurp to read files directly (no stat/watchers) - # Exit code: 0 = SYNCED, non-zero = OUT_OF_SYNC - tasks: - - name: Initialize variables - set_fact: - drift_detected: false - drifted_files: [] - - # ───────────────────────────────────────────────────────────────────────── - # Compare rsyslog.conf content (with line ending normalization) - # ───────────────────────────────────────────────────────────────────────── - - name: Read Git rsyslog.conf + # ───────────────────────────────────────────────────────────────────── + # TASK 1: Read local file from repo + # Reads dvir.txt from the local repository using base64 encoding + # ───────────────────────────────────────────────────────────────────── + - name: Read local file slurp: - src: "{{ playbook_dir }}/../../files/rsyslog.conf" + src: "{{ playbook_dir }}/../../files/dvir.txt" delegate_to: localhost - register: git_main_conf + register: local_file - - name: Read server rsyslog.conf + # ───────────────────────────────────────────────────────────────────── + # TASK 2: Read file from server + # Attempts to read dvir.txt from /tmp on the target server + # Failure is allowed here (we'll handle it in next task) + # ───────────────────────────────────────────────────────────────────── + - name: Read server file slurp: - src: "{{ rsyslog_main_config }}" - register: server_main_conf + src: /tmp/dvir.txt + register: server_file + failed_when: false - - name: Normalize line endings and compare rsyslog.conf + # ───────────────────────────────────────────────────────────────────── + # TASK 3: Compare file contents (if server file exists) + # Decodes base64 and compares content between repo and server + # Sets drift_detected to true if content differs + # ───────────────────────────────────────────────────────────────────── + - name: Compare file contents set_fact: - # Decode base64, normalize line endings (CRLF -> LF), compare - git_main_content: "{{ (git_main_conf.content | b64decode | replace('\r\n', '\n')) }}" - server_main_content: "{{ (server_main_conf.content | b64decode | replace('\r\n', '\n')) }}" + drift_detected: "{{ (local_file.content | b64decode) != (server_file.content | b64decode) }}" + when: server_file.rc == 0 - - name: Check rsyslog.conf content match - set_fact: - main_conf_match: "{{ git_main_content == server_main_content }}" - - - name: Debug rsyslog.conf comparison - debug: - msg: | - Git rsyslog.conf size: {{ git_main_content | length }} chars - Server rsyslog.conf size: {{ server_main_content | length }} chars - Match: {{ main_conf_match }} - when: not main_conf_match - - - name: Mark drift if rsyslog.conf differs + # ───────────────────────────────────────────────────────────────────── + # TASK 4: Mark as drift if server file is missing + # If the server file doesn't exist, it's also considered drift + # ───────────────────────────────────────────────────────────────────── + - name: Mark as drift if server file missing set_fact: drift_detected: true - drifted_files: "{{ drifted_files + ['rsyslog.conf'] }}" - when: not main_conf_match - - # ───────────────────────────────────────────────────────────────────────── - # Compare rsyslog.d directory files (filenames and content) - # ───────────────────────────────────────────────────────────────────────── - - name: List Git rsyslog.d files - find: - paths: "{{ playbook_dir }}/../../files/rsyslog.d" - patterns: "*.conf" - recurse: false - delegate_to: localhost - register: git_confd_list - - - name: List server rsyslog.d files - find: - paths: "{{ rsyslog_config_dir }}" - patterns: "*.conf" - recurse: false - register: server_confd_list - - - name: Extract file names - set_fact: - git_confd_names: "{{ git_confd_list.files | map(attribute='path') | map('basename') | sort }}" - server_confd_names: "{{ server_confd_list.files | map(attribute='path') | map('basename') | sort }}" - - - name: Check rsyslog.d file list match - set_fact: - confd_match: "{{ git_confd_names == server_confd_names }}" - - - name: Mark drift if rsyslog.d file list differs - set_fact: - drift_detected: true - drifted_files: "{{ drifted_files + ['rsyslog.d/'] }}" - when: not confd_match - - # Compare content of each file in rsyslog.d (only if filenames match) - - name: Read Git rsyslog.d files content - slurp: - src: "{{ playbook_dir }}/../../files/rsyslog.d/{{ item }}" - delegate_to: localhost - register: git_confd_contents - loop: "{{ git_confd_names }}" - when: confd_match - - - name: Read server rsyslog.d files content - slurp: - src: "{{ rsyslog_config_dir }}/{{ item }}" - register: server_confd_contents - loop: "{{ git_confd_names }}" - when: confd_match - - - name: Compare rsyslog.d file contents and detect drift - set_fact: - drift_detected: true - drifted_files: "{{ drifted_files + ['rsyslog.d/' + item.item] }}" - loop: "{{ git_confd_contents.results }}" - when: - - confd_match - - item.content is defined - - server_confd_contents.results[ansible_loop.index0].content is defined - - (item.content | b64decode | replace('\r\n', '\n')) != (server_confd_contents.results[ansible_loop.index0].content | b64decode | replace('\r\n', '\n')) - loop_control: - extended: yes - - # ───────────────────────────────────────────────────────────────────────── - # Output markers for update-gitops-status.sh parsing - # ───────────────────────────────────────────────────────────────────────── - - name: Output drifted files list - debug: - msg: "DRIFTED_FILES={{ drifted_files | join(',') if drifted_files | length > 0 else '' }}" + when: server_file.rc != 0 + # ───────────────────────────────────────────────────────────────────── + # TASK 5: Output SYNCED status + # Displayed when file on server matches repo file exactly + # ───────────────────────────────────────────────────────────────────── - name: Output SYNCED status debug: - msg: "SYNC_STATUS=SYNCED" + msg: "✓ dvir.txt is synced" when: not drift_detected + # ───────────────────────────────────────────────────────────────────── + # TASK 6: Output OUT_OF_SYNC status + # Displayed when file on server differs from repo or is missing + # ───────────────────────────────────────────────────────────────────── - name: Output OUT_OF_SYNC status debug: - msg: "SYNC_STATUS=OUT_OF_SYNC" - when: drift_detected - - - name: Display SYNCED - debug: - msg: | - ╭─────────────────────────────╮ - │ ✓ SYNCED │ - │ Configuration is up-to-date │ - ╰─────────────────────────────╯ - when: not drift_detected - - - name: Display OUT_OF_SYNC - debug: - msg: | - ╭─────────────────────────────╮ - │ ✗ OUT OF SYNC │ - │ Configuration has drifted │ - ╰─────────────────────────────╯ + msg: "✗ dvir.txt is out of sync" when: drift_detected - name: Fail if drift detected diff --git a/ansible/playbooks/validate.yml b/ansible/playbooks/validate.yml index 90e9cac..83b188d 100644 --- a/ansible/playbooks/validate.yml +++ b/ansible/playbooks/validate.yml @@ -1,48 +1,41 @@ --- -- name: Validate rsyslog configuration - hosts: rsyslog_servers +# ============================================================================= +# VALIDATE PLAYBOOK +# Purpose: Verify that dvir.txt exists and is readable on target servers +# Usage: ansible-playbook validate.yml +# Output: Success if file exists and is readable, Failure if not +# ============================================================================= + +- name: Validate file exists + hosts: all gather_facts: false - vars: - validate_dir: /tmp/rsyslog-validate - validate_conf: /tmp/rsyslog-validate/rsyslog.conf - tasks: - - name: Create temp validation directory - file: - path: "{{ validate_dir }}/rsyslog.d" - state: directory - mode: "0700" + # ───────────────────────────────────────────────────────────────────── + # TASK 1: Check file status + # Gathers file stats (exists, readable, permissions, etc.) + # ───────────────────────────────────────────────────────────────────── + - name: Check if file is readable + stat: + path: /tmp/dvir.txt + register: file_stat - - name: Copy main config to temp location - copy: - src: "{{ repo_root }}/files/rsyslog.conf" - dest: "{{ validate_conf }}" - remote_src: true - - - name: Copy rsyslog.d configs to temp location - copy: - src: "{{ repo_root }}/files/rsyslog.d/" - dest: "{{ validate_dir }}/rsyslog.d/" - remote_src: true - - - name: Update IncludeConfig path to point to temp dir - replace: - path: "{{ validate_conf }}" - regexp: '^\$IncludeConfig\s+.*$' - replace: '$IncludeConfig {{ validate_dir }}/rsyslog.d/*.conf' - - - name: Validate repo rsyslog configuration - command: rsyslogd -N1 -f "{{ validate_conf }}" - register: validate_result - failed_when: validate_result.rc != 0 - changed_when: false + # ───────────────────────────────────────────────────────────────────── + # TASK 2: Assert file requirements + # Verifies that the file exists and is readable + # Fails the playbook if either condition is false + # ───────────────────────────────────────────────────────────────────── + - name: Verify file exists and is readable + assert: + that: + - file_stat.stat.exists + - file_stat.stat.readable + msg: "dvir.txt not found or not readable" + # ───────────────────────────────────────────────────────────────────── + # TASK 3: Display validation result + # Shows success message if all checks passed + # ───────────────────────────────────────────────────────────────────── - name: Display validation result debug: - msg: "✓ rsyslog configuration is valid" - - - name: Clean up temp validation directory - file: - path: "{{ validate_dir }}" - state: absent + msg: "✓ dvir.txt is valid and readable" diff --git a/apply.sh b/apply.sh deleted file mode 100755 index 4fd93a1..0000000 --- a/apply.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -set -e - -# Load configuration -CONFIG_FILE="${CONFIG_FILE:-config.local.env}" -if [ ! -f "$CONFIG_FILE" ]; then - CONFIG_FILE="config.env" -fi - -if [ ! -f "$CONFIG_FILE" ]; then - echo "ERROR: Configuration file not found. Please create config.local.env or config.env" - exit 1 -fi - -# shellcheck disable=SC1090 -source "$CONFIG_FILE" - -echo "Applying rsyslog config from git repo..." -echo " Main config: $GIT_RSYSLOG_MAIN_CONFIG → $RSYSLOG_MAIN_CONFIG" -echo " Config dir: $GIT_RSYSLOG_CONFIG_DIR → $RSYSLOG_CONFIG_DIR" - -if [ ! -f "$GIT_RSYSLOG_MAIN_CONFIG" ]; then - echo "ERROR: Source file not found: $GIT_RSYSLOG_MAIN_CONFIG" - exit 1 -fi - -cp "$GIT_RSYSLOG_MAIN_CONFIG" "$RSYSLOG_MAIN_CONFIG" -mkdir -p "$RSYSLOG_CONFIG_DIR" -cp "$GIT_RSYSLOG_CONFIG_DIR"/*.conf "$RSYSLOG_CONFIG_DIR/" - -echo "Validating config..." -rsyslogd -N1 - -echo "Restarting rsyslog..." -systemctl restart rsyslog - -echo "Done." - diff --git a/drift-check.sh b/drift-check.sh deleted file mode 100755 index 965d1a4..0000000 --- a/drift-check.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -set -e - -# Load configuration -CONFIG_FILE="${CONFIG_FILE:-config.local.env}" -if [ ! -f "$CONFIG_FILE" ]; then - CONFIG_FILE="config.env" -fi - -if [ ! -f "$CONFIG_FILE" ]; then - echo "ERROR: Configuration file not found. Please create config.local.env or config.env" - exit 1 -fi - -# shellcheck disable=SC1090 -source "$CONFIG_FILE" - -echo "Checking drift between git repo and live server..." -echo " Comparing: $GIT_RSYSLOG_MAIN_CONFIG ↔ $RSYSLOG_MAIN_CONFIG" -echo " Comparing: $GIT_RSYSLOG_CONFIG_DIR ↔ $RSYSLOG_CONFIG_DIR" - -DIFF_FOUND=0 - -echo -echo "Comparing $RSYSLOG_MAIN_CONFIG" -if ! diff -u "$GIT_RSYSLOG_MAIN_CONFIG" "$RSYSLOG_MAIN_CONFIG"; then - DIFF_FOUND=1 -fi - -echo -echo "Comparing $RSYSLOG_CONFIG_DIR configs" -for file in "$GIT_RSYSLOG_CONFIG_DIR"/*.conf; do - base=$(basename "$file") - target="$RSYSLOG_CONFIG_DIR/$base" - - if [ ! -f "$target" ]; then - echo "Missing on server: $target" - DIFF_FOUND=1 - continue - fi - - if ! diff -u "$file" "$target"; then - DIFF_FOUND=1 - fi -done - -if [ "$DIFF_FOUND" -eq 0 ]; then - echo - echo "No drift detected." - exit 0 -else - echo - echo "Drift detected!" - exit 1 -fi diff --git a/files/dvir.txt b/files/dvir.txt new file mode 100644 index 0000000..6e1dd77 --- /dev/null +++ b/files/dvir.txt @@ -0,0 +1,3 @@ +This is a generic deployment file. +It can be used for any service or configuration. +Simply replace this with your own content as needed. diff --git a/files/rsyslog.conf b/files/rsyslog.conf deleted file mode 100644 index 959866e..0000000 --- a/files/rsyslog.conf +++ /dev/null @@ -1,60 +0,0 @@ -# /etc/rsyslog.conf configuration file for rsyslog -# -# For more information install rsyslog-doc and see -# /usr/share/doc/rsyslog-doc/html/configuration/index.html -# -# Default logging rules can be found in /etc/rsyslog.d/50-default.conf - -# TEST -################# -#### MODULES #### -################# - -module(load="imuxsock") # provides support for local system logging -#module(load="immark") # provides --MARK-- message capability - -# provides UDP syslog reception -#module(load="imudp") -#input(type="imudp" port="514") - -# provides TCP syslog reception -#module(load="imtcp") -#input(type="imtcp" port="514") - -# provides kernel logging support and enable non-kernel klog messages -module(load="imklog" permitnonkernelfacility="on") - -########################### -#### GLOBAL DIRECTIVES #### -########################### - -# Use traditional timestamp format. -# To enable high precision timestamps, comment out the following line. - -$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat - -# Filter duplicated messages -$RepeatedMsgReduction on - -# -# Set the default permissions for all log files. -# -$FileOwner syslog -$FileGroup adm -$FileCreateMode 0640 -$DirCreateMode 0755 -$Umask 0022 -$PrivDropToUser syslog -$PrivDropToGroup syslog - -# -# Where to place spool and state files -# -$WorkDirectory /var/spool/rsyslog - -# -# Include all config files in /etc/rsyslog.d/ -# -$IncludeConfig /etc/rsyslog.d/*.conf - -# Dvir was here diff --git a/files/rsyslog.d/30-lab.conf b/files/rsyslog.d/30-lab.conf deleted file mode 100644 index 7b46f46..0000000 --- a/files/rsyslog.d/30-lab.conf +++ /dev/null @@ -1,2 +0,0 @@ -local0.* /var/log/lab.log -# dvir test \ No newline at end of file diff --git a/gitops-status-server/.helmignore b/gitops-status-server/.helmignore deleted file mode 100644 index 0e8a0eb..0000000 --- a/gitops-status-server/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/gitops-status-server/Chart.yaml b/gitops-status-server/Chart.yaml deleted file mode 100644 index 43fc4d3..0000000 --- a/gitops-status-server/Chart.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v2 -name: gitops-status-server -description: A minimal HTTP server that serves GitOps status information as JSON -type: application -version: 1.0.0 -appVersion: "1.25.5" -keywords: - - gitops - - status - - monitoring - - nginx -maintainers: - - name: DevOps Team -home: https://github.com/your-org/observability-stack diff --git a/gitops-status-server/README.md b/gitops-status-server/README.md deleted file mode 100644 index 785d491..0000000 --- a/gitops-status-server/README.md +++ /dev/null @@ -1,478 +0,0 @@ -# GitOps Status Server Helm Chart - -A dual-container HTTP server that receives GitOps status updates via POST API and serves status information as JSON for monitoring and observability purposes. - -## Overview - -This chart deploys a two-container pod: -1. **Nginx** - Serves `/status.json` endpoint for monitoring tools and handles API routing -2. **Flask API** - Processes POST requests to `/api/status` and updates the status JSON - -It's designed to be consumed by Grafana's Infinity datasource or other monitoring tools, and to receive updates from CI/CD pipelines like Woodpecker. - -## Architecture - -``` -CI/CD Pipeline (Woodpecker) - ↓ - POST /api/status - ↓ -Kubernetes Service (port 80) - ↓ -Nginx (port 8080) - ├─→ /api/status → Proxies to Flask (localhost:5000) - └─→ /status.json → Serves static file - ↓ -Shared Volume (emptyDir) - ├─→ status.json (updated by Flask API) - └─→ Read by Nginx - ↓ -Grafana Infinity Datasource - Reads /status.json -``` - -## Features - -- **API-driven updates**: POST endpoint for CI/CD pipelines to update status -- **Read-only serving**: Grafana-friendly JSON endpoint -- **Minimal footprint**: nginx-unprivileged + Python-Alpine with minimal resources -- **Secure by default**: Runs as non-root with restricted filesystems -- **Internal only**: ClusterIP service for cluster-internal access -- **ArgoCD compatible**: Init container auto-initializes status from ConfigMap -- **Production-ready**: Includes health checks, security contexts, and resource limits - -## Installation - -### Using Helm - -```bash -# Install with default values -helm install gitops-status ./gitops-status-server - -# Install with custom namespace -helm install gitops-status ./gitops-status-server -n observability-stack --create-namespace - -# Install with custom values -helm install gitops-status ./gitops-status-server -f custom-values.yaml -``` - -### Using ArgoCD - -Create an Application manifest: - -```yaml -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: gitops-status-server - namespace: argocd -spec: - project: default - source: - repoURL: https://github.com/your-org/observability-stack - targetRevision: main - path: gitops-status-server - helm: - values: | - replicaCount: 1 - statusJson: - repo: "rsyslog" - server: "rsyslog-lab" - sync_status: "UNKNOWN" -``` - -## API Endpoints - -### GET /status.json -Returns the current status JSON - -```bash -curl http://gitops-status-server.observability-stack.svc.cluster.local:80/status.json -``` - -Response: -```json -{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "SYNCED", - "drift_count": 0, - "files": [], - "last_check": "2026-04-21T10:30:00Z" -} -``` - -### POST /api/status -Updates the status with new data - -```bash -curl -X POST http://gitops-status-server.observability-stack.svc.cluster.local:80/api/status \ - -H "Content-Type: application/json" \ - -d '{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "OUT_OF_SYNC", - "drift_count": 2, - "files": [ - {"name": "rsyslog.conf"}, - {"name": "rsyslog.d/30-lab.conf"} - ], - "last_check": "2026-04-21T10:30:00Z" - }' -``` - -Response (HTTP 200): -```json -{ - "success": true, - "message": "Status updated successfully", - "status": { ... } -} -``` - -### GET /health -Health check endpoint (returns HTTP 200) - -```bash -curl http://gitops-status-server.observability-stack.svc.cluster.local:80/health -``` - -### GET /ready -Readiness check (verifies status file is readable) - -```bash -curl http://gitops-status-server.observability-stack.svc.cluster.local:80/ready -``` - -## Integration with Woodpecker - -The rsyslog CI/CD pipeline can update status by POSTing to the `/api/status` endpoint: - -```bash -#!/bin/bash - -GITOPS_STATUS_SERVER_URL="http://gitops-status-server.observability-stack.svc.cluster.local:80" - -STATUS_JSON='{ - "repo": "rsyslog", - "server": "rsyslog-lab", - "sync_status": "SYNCED", - "drift_count": 0, - "files": [], - "last_check": "2026-04-21T10:30:00Z" -}' - -curl -X POST "$GITOPS_STATUS_SERVER_URL/api/status" \ - -H "Content-Type: application/json" \ - -d "$STATUS_JSON" -``` - -## Service Discovery - -### Internal Kubernetes URL -``` -http://gitops-status-server.observability-stack.svc.cluster.local:80/status.json -``` - -### Port Forwarding (for local testing) -```bash -kubectl port-forward -n observability-stack svc/gitops-status-server 8080:80 -# Then access at http://localhost:8080/status.json -``` - -### NodePort (if service type is changed) -```bash -kubectl patch service -n observability-stack gitops-status-server -p '{"spec":{"type":"NodePort"}}' -# Then access at http://:/status.json -``` - -## Configuration - -See `values.yaml` for all configuration options: - -- `replicaCount`: Number of replicas -- `image.repository`: Container image -- `image.tag`: Image tag -- `service.type`: Service type (ClusterIP, NodePort, LoadBalancer) -- `service.port`: Service port (default 80) -- `service.targetPort`: Container port (default 8080) -- `resources`: CPU/memory limits and requests -- `statusJson`: Default status JSON values -- `api.image.*`: Python/Flask image configuration - -## Grafana Integration - -### Infinity Datasource Configuration - -1. Install Infinity datasource plugin: - ```bash - grafana-cli plugins install yesoreyeram-infinity-datasource - ``` - -2. Add datasource with URL: - ``` - http://gitops-status-server.observability-stack.svc.cluster.local:80/status.json - ``` - -3. Create panels to visualize: - - `sync_status`: Current synchronization state - - `drift_count`: Number of drifted files - - `files[]`: List of changed files - - `last_check`: Timestamp of last check - -### Example Query - -```json -{ - "url": "http://gitops-status-server.observability-stack.svc.cluster.local:80/status.json", - "format": "json" -} -``` - -## Security - -- Runs as non-root user (UID 101) -- Read-only root filesystem (except for /tmp, /var/cache/nginx, /var/run) -- No privileged capabilities -- Network policies recommended for production -- Service Account with minimal RBAC - -## Troubleshooting - -### POST Request Returns 400 Error - -**Issue**: "Invalid JSON" error - -**Solution**: Verify JSON formatting with: -```bash -echo '{...}' | jq '.' -``` - -### POST Updates Not Appearing in GET Response - -**Issue**: Update endpoint returns 200 but status.json isn't updated - -**Possible causes**: -- Shared volume permission issue -- API container crashed after POST -- Status file permissions - -**Debug**: -```bash -# Check logs -kubectl logs -f deployment/gitops-status-server -c api -kubectl logs -f deployment/gitops-status-server -c nginx - -# Check shared volume -kubectl exec deployment/gitops-status-server -c nginx -- ls -la /usr/share/nginx/html/ - -# Test API directly (port-forward to 5000 first) -kubectl port-forward deployment/gitops-status-server 5000:5000 -curl -X POST http://localhost:5000/api/status -H "Content-Type: application/json" -d '{...}' -``` - -### Connection Refused to gitops-status-server - -**Issue**: Woodpecker can't reach the service - -**Possible causes**: -- Service in different namespace -- Network policies blocking traffic -- Woodpecker outside cluster -- Service DNS name incorrect - -**Solutions**: -- Verify service exists: `kubectl get svc gitops-status-server -n observability-stack` -- Use NodePort for external access (update service type in values) -- Use port-forward as a temporary solution -- Verify network policies allow traffic - -## Performance - -- **CPU**: 150m limit (100m nginx + 100m API) -- **Memory**: 192Mi limit (64Mi nginx + 128Mi API) -- **Startup time**: ~5 seconds (Flask app install + startup) -- **Update latency**: <100ms (direct file write) -- **Read performance**: <10ms (static file serving) - -## License - -Same as observability-stack repository - statusJson: - repo: "my-repo" - server: "my-server" - sync_status: "SYNCED" - drift_count: 0 - files: [] - last_check: "2026-04-21T10:00:00Z" - destination: - server: https://kubernetes.default.svc - namespace: monitoring - syncPolicy: - automated: - prune: true - selfHeal: true -``` - -## Configuration - -### Key Values - -| Parameter | Description | Default | -|-----------|-------------|---------| -| `replicaCount` | Number of replicas | `1` | -| `image.repository` | Container image repository | `nginxinc/nginx-unprivileged` | -| `image.tag` | Container image tag | `1.25-alpine` | -| `service.type` | Kubernetes service type | `ClusterIP` | -| `service.port` | Service port | `80` | -| `service.targetPort` | Container target port | `8080` | -| `resources.limits.cpu` | CPU limit | `100m` | -| `resources.limits.memory` | Memory limit | `64Mi` | -| `statusJson` | JSON content to serve | See values.yaml | - -### Custom Status JSON - -Override the status JSON content in your values: - -```yaml -statusJson: - repo: "production-apps" - server: "prod-cluster-01" - sync_status: "SYNCED" - drift_count: 2 - files: - - "deployment.yaml" - - "service.yaml" - last_check: "2026-04-21T12:30:00Z" -``` - -## Usage - -### Access the Status Endpoint - -From inside the cluster: - -```bash -# Using the service DNS name -curl http://gitops-status-server/status.json - -# With namespace -curl http://gitops-status-server.monitoring.svc.cluster.local/status.json -``` - -### Grafana Infinity Datasource Configuration - -1. Add an Infinity datasource in Grafana -2. Configure URL: `http://gitops-status-server.monitoring.svc.cluster.local/status.json` -3. Parser: JSON -4. Use fields from the JSON response in your dashboard - -Example query fields: -- `sync_status` - Current sync status -- `drift_count` - Number of drifted resources -- `files` - List of changed files -- `last_check` - Timestamp of last check - -## Updating Status Data - -### Manual Update - -Edit the ConfigMap directly: - -```bash -kubectl edit configmap gitops-status-server -n monitoring -``` - -The deployment will automatically roll out with the new content due to the ConfigMap checksum annotation. - -### Automated Update via Pipeline - -Use `kubectl` in your CI/CD pipeline: - -```bash -kubectl create configmap gitops-status-server \ - --from-file=status.json=./status.json \ - --dry-run=client -o yaml | kubectl apply -f - -``` - -### ArgoCD Hook (Advanced) - -Create a PostSync hook that updates the ConfigMap with current sync status: - -```yaml -apiVersion: batch/v1 -kind: Job -metadata: - name: update-status - annotations: - argocd.argoproj.io/hook: PostSync -spec: - template: - spec: - containers: - - name: update - image: bitnami/kubectl - command: - - /bin/sh - - -c - - | - # Update status.json with current sync status - kubectl patch configmap gitops-status-server \ - --patch '{"data":{"status.json":"..."}}' - restartPolicy: Never -``` - -## Security Considerations - -- Runs as non-root user (UID 101) -- Read-only root filesystem -- No privilege escalation -- Minimal capabilities (all dropped) -- No external network access required -- ClusterIP only (no external exposure) - -## Resource Requirements - -Minimal resource footprint suitable for small clusters: -- CPU: 50m request / 100m limit -- Memory: 32Mi request / 64Mi limit - -## Troubleshooting - -### Check pod status - -```bash -kubectl get pods -l app.kubernetes.io/name=gitops-status-server -``` - -### View logs - -```bash -kubectl logs -l app.kubernetes.io/name=gitops-status-server -``` - -### Test endpoint - -```bash -kubectl run -it --rm curl --image=curlimages/curl --restart=Never -- \ - curl http://gitops-status-server/status.json -``` - -### Common Issues - -**Pod not starting**: Check security context compatibility with your cluster's PSP/PSA policies. - -**Empty response**: Verify the ConfigMap is mounted correctly: -```bash -kubectl describe pod -l app.kubernetes.io/name=gitops-status-server -``` - -**Service not accessible**: Ensure you're accessing from within the cluster and using the correct namespace. - -## License - -This chart is part of the observability-stack project. - -## Maintainers - -- DevOps Team diff --git a/gitops-status-server/templates/_helpers.tpl b/gitops-status-server/templates/_helpers.tpl deleted file mode 100644 index 7585170..0000000 --- a/gitops-status-server/templates/_helpers.tpl +++ /dev/null @@ -1,63 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "gitops-status-server.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -*/}} -{{- define "gitops-status-server.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "gitops-status-server.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "gitops-status-server.labels" -}} -helm.sh/chart: {{ include "gitops-status-server.chart" . }} -{{ include "gitops-status-server.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- with .Values.labels }} -{{ toYaml . }} -{{- end }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "gitops-status-server.selectorLabels" -}} -app.kubernetes.io/name: {{ include "gitops-status-server.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "gitops-status-server.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "gitops-status-server.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/gitops-status-server/templates/api-app.yaml b/gitops-status-server/templates/api-app.yaml deleted file mode 100644 index 25a47fd..0000000 --- a/gitops-status-server/templates/api-app.yaml +++ /dev/null @@ -1,142 +0,0 @@ -{{/* -ConfigMap containing the API backend Python script -Handles POST requests to /api/status and updates the status.json file -*/}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "gitops-status-server.fullname" . }}-api - labels: - {{- include "gitops-status-server.labels" . | nindent 4 }} -data: - app.py: | - #!/usr/bin/env python3 - """ - Simple Flask API for updating status.json - Listens on port 5000 and handles POST requests to /api/status - """ - import os - import json - import logging - from flask import Flask, request, jsonify - from datetime import datetime - - app = Flask(__name__) - - # Configuration - STATUS_FILE = '/usr/share/nginx/html/status.json' - API_PORT = int(os.environ.get('API_PORT', 5000)) - API_HOST = os.environ.get('API_HOST', '127.0.0.1') - - # Setup logging - logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' - ) - logger = logging.getLogger(__name__) - - def load_status(): - """Load the current status from file""" - try: - if os.path.exists(STATUS_FILE): - with open(STATUS_FILE, 'r') as f: - return json.load(f) - else: - # Default status if file doesn't exist - return { - "repo": "unknown", - "server": "unknown", - "sync_status": "UNKNOWN", - "drift_count": 0, - "files": [], - "last_check": "" - } - except Exception as e: - logger.error(f"Error loading status: {e}") - return {} - - def save_status(status): - """Save the status to file""" - try: - # Ensure directory exists (should already exist from mount) - os.makedirs(os.path.dirname(STATUS_FILE), exist_ok=True) - - # Write with proper formatting - with open(STATUS_FILE, 'w') as f: - json.dump(status, f, indent=2) - - logger.info(f"Status saved successfully: {status['repo']}/{status['server']} -> {status['sync_status']}") - return True - except Exception as e: - logger.error(f"Error saving status: {e}") - return False - - @app.route('/api/status', methods=['GET', 'POST', 'OPTIONS']) - def api_status(): - """ - GET: Retrieve current status - POST: Update status with new data - """ - if request.method == 'OPTIONS': - return '', 204 - - if request.method == 'GET': - status = load_status() - return jsonify(status), 200 - - if request.method == 'POST': - try: - # Parse incoming JSON - incoming_data = request.get_json() - if not incoming_data: - return jsonify({"error": "No JSON data provided"}), 400 - - # Load current status - status = load_status() - - # Update with incoming data (merge) - status.update(incoming_data) - - # Ensure required fields exist - if 'last_check' not in status or not status['last_check']: - status['last_check'] = datetime.utcnow().isoformat() + 'Z' - - # Save updated status - if save_status(status): - return jsonify({ - "success": True, - "message": "Status updated successfully", - "status": status - }), 200 - else: - return jsonify({ - "error": "Failed to save status" - }), 500 - - except json.JSONDecodeError: - return jsonify({"error": "Invalid JSON"}), 400 - except Exception as e: - logger.error(f"Error processing POST request: {e}") - return jsonify({"error": str(e)}), 500 - - @app.route('/health', methods=['GET']) - def health(): - """Health check endpoint""" - return jsonify({"status": "healthy"}), 200 - - @app.route('/ready', methods=['GET']) - def ready(): - """Readiness check - verify status file is accessible""" - try: - status = load_status() - if status: - return jsonify({"status": "ready"}), 200 - else: - return jsonify({"status": "not_ready", "reason": "status file empty"}), 503 - except Exception as e: - return jsonify({"status": "not_ready", "error": str(e)}), 503 - - if __name__ == '__main__': - logger.info(f"Starting gitops-status-server API on {API_HOST}:{API_PORT}") - logger.info(f"Status file: {STATUS_FILE}") - app.run(host=API_HOST, port=API_PORT, debug=False) diff --git a/gitops-status-server/templates/configmap.yaml b/gitops-status-server/templates/configmap.yaml deleted file mode 100644 index 7b4cc5c..0000000 --- a/gitops-status-server/templates/configmap.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{/* -ConfigMap for default status.json values -Used by init container to set up initial status if file doesn't exist -*/}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "gitops-status-server.fullname" . }} - labels: - {{- include "gitops-status-server.labels" . | nindent 4 }} - {{- with .Values.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -data: - # Default status.json values (used for initialization) - # This is not mounted directly; instead it's used by the init container - # to set up the initial status.json in the shared emptyDir volume. - # The actual status.json is stored on the emptyDir and updated via the API. - status.json: | - {{- .Values.statusJson | toJson | nindent 4 }} - diff --git a/gitops-status-server/templates/deployment.yaml b/gitops-status-server/templates/deployment.yaml deleted file mode 100644 index c96ad1e..0000000 --- a/gitops-status-server/templates/deployment.yaml +++ /dev/null @@ -1,190 +0,0 @@ -{{/* -Deployment for the gitops-status-server -Runs nginx-unprivileged to serve the status.json file -*/}} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "gitops-status-server.fullname" . }} - labels: - {{- include "gitops-status-server.labels" . | nindent 4 }} - {{- with .Values.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - {{- include "gitops-status-server.selectorLabels" . | nindent 6 }} - template: - metadata: - annotations: - # Automatically roll deployment when ConfigMap changes - checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} - {{- with .Values.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "gitops-status-server.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "gitops-status-server.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - - # Init container to set up initial status.json from ConfigMap - initContainers: - - name: init-status - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - command: - - sh - - -c - - | - if [ ! -f /usr/share/nginx/html/status.json ]; then - cat > /usr/share/nginx/html/status.json <<'EOF' - {{- .Values.statusJson | toJson | nindent 10 }} - EOF - fi - volumeMounts: - - name: shared-data - mountPath: /usr/share/nginx/html - - containers: - - name: api - image: "{{ .Values.api.image.repository }}:{{ .Values.api.image.tag }}" - imagePullPolicy: {{ .Values.api.image.pullPolicy }} - command: - - sh - - -c - - | - pip install --no-cache-dir Flask==2.3.2 >/dev/null 2>&1 - exec python3 /app/app.py - ports: - - name: api - containerPort: 5000 - protocol: TCP - env: - - name: API_HOST - value: "127.0.0.1" - - name: API_PORT - value: "5000" - - name: FLASK_ENV - value: "production" - livenessProbe: - httpGet: - path: /health - port: api - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /ready - port: api - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 2 - failureThreshold: 2 - resources: - limits: - cpu: 100m - memory: 128Mi - requests: - cpu: 50m - memory: 64Mi - volumeMounts: - - name: shared-data - mountPath: /usr/share/nginx/html - - name: api-code - mountPath: /app - readOnly: true - - - name: nginx - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - securityContext: - {{- toYaml .Values.securityContext | nindent 10 }} - ports: - - name: http - containerPort: {{ .Values.service.targetPort }} - protocol: TCP - # Health checks - livenessProbe: - httpGet: - path: /status.json - port: http - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /status.json - port: http - initialDelaySeconds: 2 - periodSeconds: 5 - timeoutSeconds: 2 - failureThreshold: 2 - resources: - {{- toYaml .Values.resources | nindent 10 }} - volumeMounts: - # Mount the nginx config - - name: nginx-config - mountPath: /etc/nginx/nginx.conf - subPath: nginx.conf - readOnly: true - # Mount the shared data directory (status.json is writable here) - - name: shared-data - mountPath: /usr/share/nginx/html - # nginx-unprivileged needs writable directories for cache and run - - name: cache - mountPath: /var/cache/nginx - - name: run - mountPath: /var/run - # nginx needs writable /tmp for proxy buffers - - name: tmp - mountPath: /tmp - volumes: - # ConfigMap volume containing the nginx configuration - - name: nginx-config - configMap: - name: {{ include "gitops-status-server.fullname" . }}-nginx-config - items: - - key: nginx.conf - path: nginx.conf - # ConfigMap volume containing the API application code - - name: api-code - configMap: - name: {{ include "gitops-status-server.fullname" . }}-api - defaultMode: 0755 - items: - - key: app.py - path: app.py - # Shared data volume for status.json (writable emptyDir) - - name: shared-data - emptyDir: - sizeLimit: 1Mi - # Empty directories for nginx runtime - - name: cache - emptyDir: {} - - name: run - emptyDir: {} - - name: tmp - emptyDir: {} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/gitops-status-server/templates/nginx-config.yaml b/gitops-status-server/templates/nginx-config.yaml deleted file mode 100644 index 0f6b2dc..0000000 --- a/gitops-status-server/templates/nginx-config.yaml +++ /dev/null @@ -1,96 +0,0 @@ -{{/* -ConfigMap containing the nginx configuration -Enables serving status.json via GET and updating via POST requests -*/}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "gitops-status-server.fullname" . }}-nginx-config - labels: - {{- include "gitops-status-server.labels" . | nindent 4 }} -data: - nginx.conf: | - # Minimal nginx config for serving and updating status.json - user nginx; - worker_processes auto; - error_log /var/log/nginx/error.log warn; - pid /var/run/nginx.pid; - - events { - worker_connections 1024; - } - - http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - client_max_body_size 1M; - - # Gzip compression - gzip on; - gzip_vary on; - gzip_types text/plain text/css text/xml text/javascript - application/x-javascript application/xml+rss - application/json; - - upstream api_backend { - server 127.0.0.1:5000; - keepalive 32; - } - - server { - listen 8080 default_server; - server_name _; - - # Serve status.json as read-only - location /status.json { - alias /usr/share/nginx/html/status.json; - add_header Cache-Control "no-cache, no-store, must-revalidate"; - add_header Pragma "no-cache"; - add_header Expires "0"; - } - - # Health check endpoint - location /health { - access_log off; - return 200 "healthy\n"; - add_header Content-Type text/plain; - } - - # Proxy POST requests to the API backend (Python Flask) - location /api/ { - proxy_pass http://api_backend; - proxy_http_version 1.1; - proxy_set_header Connection ""; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - - # Buffer settings for POST requests - proxy_request_buffering off; - proxy_buffering off; - - # Timeouts - proxy_connect_timeout 30s; - proxy_send_timeout 30s; - proxy_read_timeout 30s; - } - - # Catch-all for root - location / { - return 301 /status.json; - } - } - } diff --git a/gitops-status-server/templates/service.yaml b/gitops-status-server/templates/service.yaml deleted file mode 100644 index 6de2897..0000000 --- a/gitops-status-server/templates/service.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{{/* -Service for the gitops-status-server -Exposes the nginx server inside the cluster (ClusterIP) -This allows Grafana to query the status.json endpoint -*/}} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "gitops-status-server.fullname" . }} - labels: - {{- include "gitops-status-server.labels" . | nindent 4 }} - {{- with .Values.service.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: {{ .Values.service.targetPort }} - protocol: TCP - name: http - selector: - {{- include "gitops-status-server.selectorLabels" . | nindent 4 }} diff --git a/gitops-status-server/templates/serviceaccount.yaml b/gitops-status-server/templates/serviceaccount.yaml deleted file mode 100644 index 48517d8..0000000 --- a/gitops-status-server/templates/serviceaccount.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{/* -ServiceAccount for the gitops-status-server -*/}} -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "gitops-status-server.serviceAccountName" . }} - labels: - {{- include "gitops-status-server.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/gitops-status-server/values.yaml b/gitops-status-server/values.yaml deleted file mode 100644 index cadb501..0000000 --- a/gitops-status-server/values.yaml +++ /dev/null @@ -1,103 +0,0 @@ -# Default values for gitops-status-server -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -# Number of replicas for the deployment -replicaCount: 1 - -# Container image configuration -image: - # Use nginx-unprivileged for better security (runs as non-root) - repository: nginxinc/nginx-unprivileged - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion - tag: "1.25-alpine" - -# API backend container configuration (handles POST requests) -api: - image: - # Python Flask API for handling status updates - repository: python - pullPolicy: IfNotPresent - tag: "3.11-alpine" - # Pre-install Flask via pip before running the app - pip_packages: "Flask==2.3.2" - -# Image pull secrets for private registries -imagePullSecrets: [] - -# Override the name of the chart -nameOverride: "" -fullnameOverride: "" - -# Service configuration -service: - # Service type - ClusterIP for internal-only access - type: ClusterIP - # Port where the service will be exposed - port: 80 - # Target port on the container (nginx default) - targetPort: 8080 - # Annotations to add to the service - annotations: {} - -# Resource limits and requests -resources: - limits: - cpu: 100m - memory: 64Mi - requests: - cpu: 50m - memory: 32Mi - -# Node selector for pod assignment -nodeSelector: {} - -# Tolerations for pod assignment -tolerations: [] - -# Affinity rules for pod assignment -affinity: {} - -# Security context for the pod -podSecurityContext: - runAsNonRoot: true - runAsUser: 101 - fsGroup: 101 - -# Security context for the container -securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - -# Status JSON content -# This can be overridden in your values to customize the status information -statusJson: - repo: "rsyslog" - server: "rsyslog-lab" - sync_status: "UNKNOWN" - drift_count: 0 - files: [] - last_check: "" - -# Labels to add to all resources -labels: {} - -# Annotations to add to all resources -annotations: {} - -# Pod annotations -podAnnotations: {} - -# Service account configuration -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" diff --git a/update-gitops-status.sh b/update-gitops-status.sh deleted file mode 100644 index 65bbbb1..0000000 --- a/update-gitops-status.sh +++ /dev/null @@ -1,414 +0,0 @@ -#!/bin/bash -# ============================================================================= -# update-gitops-status.sh -# -# Purpose: -# Runs drift-check playbook and generates a JSON status snapshot for -# gitops-status-server. This replaces Pushgateway metric updates with -# richer JSON status suitable for Grafana visualization via Infinity DS. -# -# Flow: -# 1. Execute ansible/playbooks/drift-check.yml (check mode, read-only) -# 2. Capture exit code to determine sync status -# 3. Parse playbook output to extract changed files -# 4. Build structured JSON with metadata -# 5. POST JSON to gitops-status-server API -# -# Usage: -# ./update-gitops-status.sh -# -# Environment Variables: -# GITOPS_STATUS_SERVER_URL - URL of gitops-status-server API -# (default: http://gitops-status-server.observability-stack.svc.cluster.local:80) -# REPO_NAME - Repository name (default: rsyslog) -# SERVER_NAME - Server name (default: rsyslog-lab) -# -# Generated JSON Structure: -# { -# "repo": "rsyslog", -# "server": "rsyslog-lab", -# "sync_status": "SYNCED" or "OUT_OF_SYNC", -# "drift_count": , -# "files": [{"name": "rsyslog.conf"}, {"name": "rsyslog.d/30-lab.conf"}], -# "last_check": "2026-04-21T10:30:00Z" -# } -# -# Exit Codes: -# 0 - Success (JSON posted to gitops-status-server regardless of sync status) -# 1 - Failure (playbook error, network error, JSON post failure, etc.) -# -# ============================================================================= - -set -euo pipefail - -# ───────────────────────────────────────────────────────────────────────────── -# Configuration -# ───────────────────────────────────────────────────────────────────────────── -GITOPS_STATUS_SERVER_URL="${GITOPS_STATUS_SERVER_URL:-http://gitops-status-server.observability-stack.svc.cluster.local:5000}" -REPO_NAME="${REPO_NAME:-rsyslog}" -SERVER_NAME="${SERVER_NAME:-rsyslog-lab}" -INVENTORY_FILE="ansible/inventory/hosts.yml" -PLAYBOOK="ansible/playbooks/drift-check.yml" -MODE="${MODE:-drift-check}" # drift-check or post-deploy -PLAYBOOK_LOG="" # Initialize to avoid unbound variable error - -echo "═══════════════════════════════════════════════════════════════════════════════" -echo " GitOps Status Update" -echo " Repository: $REPO_NAME | Server: $SERVER_NAME" -echo " Target: $GITOPS_STATUS_SERVER_URL" -echo " Mode: $MODE" -echo "═══════════════════════════════════════════════════════════════════════════════" -echo "" - -CHANGED_FILES=() -DRIFT_COUNT=0 -SYNC_STATUS="SYNCED" - -# ───────────────────────────────────────────────────────────────────────────────── -# MODE 1: post-deploy - Report what files were deployed from Git -# ───────────────────────────────────────────────────────────────────────────────── -if [ "$MODE" = "post-deploy" ]; then - echo "Step 1/4: Analyzing Git changes (what was just deployed)..." - - # Check what files changed in the last commit in files/ directory - if command -v git >/dev/null 2>&1 && [ -d .git ]; then - # Try multiple methods to detect changed files (Woodpecker might do shallow clone) - - # Method 1: Use Woodpecker CI environment variables (most reliable in CI) - if [ -n "${CI_PREV_COMMIT_SHA:-}" ] && [ -n "${CI_COMMIT_SHA:-}" ]; then - echo " DEBUG: Using Woodpecker commit range: ${CI_PREV_COMMIT_SHA}..${CI_COMMIT_SHA}" - CHANGED_FILE_PATHS=$(git diff --name-only "${CI_PREV_COMMIT_SHA}" "${CI_COMMIT_SHA}" -- files/ 2>/dev/null || echo "") - # Method 2: Use git diff with HEAD~1 - elif git rev-parse HEAD~1 >/dev/null 2>&1; then - echo " DEBUG: Using git diff HEAD~1..HEAD" - CHANGED_FILE_PATHS=$(git diff --name-only HEAD~1 HEAD -- files/ 2>/dev/null || echo "") - # Method 3: Use git show (less accurate - shows all files in commit) - else - echo " DEBUG: Using git show HEAD (may show all files in commit)" - CHANGED_FILE_PATHS=$(git show --name-only --pretty="" HEAD -- files/ 2>/dev/null || echo "") - fi - - echo " DEBUG: Git detection method results: '$CHANGED_FILE_PATHS'" - - if [ -n "$CHANGED_FILE_PATHS" ]; then - echo " Files changed in last commit:" - while IFS= read -r filepath; do - if [ -n "$filepath" ]; then - # Strip "files/" prefix to get the config file name - filename="${filepath#files/}" - - # Skip if it's just "files/" (directory change) - if [ "$filename" != "" ] && [ "$filename" != "files/" ]; then - CHANGED_FILES+=("$filename") - echo " - $filename" - fi - fi - done <<< "$CHANGED_FILE_PATHS" - - DRIFT_COUNT=${#CHANGED_FILES[@]} - else - echo " No files changed in files/ directory" - fi - else - echo " Git not available, cannot determine deployed files" - fi - - # Always SYNCED after successful deploy - SYNC_STATUS="SYNCED" - echo " ✓ Status: SYNCED - files were deployed successfully" - echo " Total deployed files: $DRIFT_COUNT" - echo "" - -# ───────────────────────────────────────────────────────────────────────────────── -# MODE 2: drift-check - Check for manual changes on server (drift detection) -# ───────────────────────────────────────────────────────────────────────────────── -else - echo "Step 1/4: Running drift-check playbook..." - - # Capture playbook output to a temp file for parsing - PLAYBOOK_LOG=$(mktemp) - KEEP_LOG="${KEEP_PLAYBOOK_LOG:-false}" - if [ "$KEEP_LOG" = "true" ]; then - PLAYBOOK_LOG="./drift-check-output.log" - echo " Playbook output will be saved to: $PLAYBOOK_LOG" - fi - - # Set up cleanup trap (will be updated later with RESPONSE_BODY) - trap "rm -f $PLAYBOOK_LOG" EXIT - - # Run playbook (no -v flag to avoid file descriptor exhaustion in containers) - # Exit code: 0 = synced, non-zero = drift detected (expected) - # Limit forks to 1 to reduce file descriptor usage - set +e - ANSIBLE_FORCE_COLOR=false \ - ANSIBLE_FORKS=1 \ - ansible-playbook \ - -i "$INVENTORY_FILE" \ - "$PLAYBOOK" \ - > "$PLAYBOOK_LOG" 2>&1 - DRIFT_RC=$? - set -e - - # Show playbook output for debugging (compact) - echo "Playbook output (last 25 lines):" - cat "$PLAYBOOK_LOG" | tail -25 - echo "" - echo "DEBUG: Full playbook output saved to: $PLAYBOOK_LOG" - echo "" - - # ───────────────────────────────────────────────────────────────────────────────── - # Step 2: Determine sync status and collect changed files - # ───────────────────────────────────────────────────────────────────────────────── - echo "Step 2/4: Analyzing drift detection results..." - - # Exit code 0 = synced (all tasks succeeded) - # Exit code non-zero = drift detected (fail task was reached) - if [ "$DRIFT_RC" -eq 0 ]; then - SYNC_STATUS="SYNCED" - echo " ✓ Status: SYNCED - server configuration matches Git" - else - SYNC_STATUS="OUT_OF_SYNC" - echo " ✗ Status: OUT OF SYNC - configuration drift detected" - fi - - # Extract structured drifted files from playbook output - # The drift-check.yml playbook outputs: DRIFTED_FILES=file1,file2,file3 - echo " DEBUG: Searching for DRIFTED_FILES in playbook output..." - if grep -q "DRIFTED_FILES=" "$PLAYBOOK_LOG"; then - echo " DEBUG: Found DRIFTED_FILES pattern" - DRIFTED_FILES_LINE=$(grep "DRIFTED_FILES=" "$PLAYBOOK_LOG" | tail -1) - echo " DEBUG: Raw line: $DRIFTED_FILES_LINE" - - # Extract value after DRIFTED_FILES= (handles both YAML and default callback formats) - # Format: "msg: DRIFTED_FILES=file1,file2" or "DRIFTED_FILES=file1,file2" - DRIFTED_FILES_STR=$(echo "$DRIFTED_FILES_LINE" | sed 's/.*DRIFTED_FILES=//' | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//' | tr -d '"') - echo " DEBUG: Extracted value: '$DRIFTED_FILES_STR'" - - # Check if the value is an empty list ([] or empty string) - if [ -n "$DRIFTED_FILES_STR" ] && [ "$DRIFTED_FILES_STR" != "[]" ] && [ "$DRIFTED_FILES_STR" != "" ]; then - # Parse comma-separated list into array - IFS=',' read -ra CHANGED_FILES <<<"$DRIFTED_FILES_STR" - - echo " DEBUG: Parsed ${#CHANGED_FILES[@]} files" - - # Clean up whitespace and normalize paths - for i in "${!CHANGED_FILES[@]}"; do - CHANGED_FILES[$i]=$(echo "${CHANGED_FILES[$i]}" | xargs) - - # Convert full paths to relative paths for cleaner output - if [[ "${CHANGED_FILES[$i]}" == /etc/rsyslog.conf ]]; then - CHANGED_FILES[$i]="rsyslog.conf" - elif [[ "${CHANGED_FILES[$i]}" == /etc/rsyslog.d/* ]]; then - CHANGED_FILES[$i]=$(echo "${CHANGED_FILES[$i]}" | sed 's|^/etc/||') - fi - - if [ "$SYNC_STATUS" = "OUT_OF_SYNC" ]; then - echo " - Drift detected in: ${CHANGED_FILES[$i]}" - fi - done - - DRIFT_COUNT=${#CHANGED_FILES[@]} - else - echo " DEBUG: DRIFTED_FILES is empty or []" - fi - else - echo " DEBUG: DRIFTED_FILES not found in playbook output" - fi - - # Additional validation: If OUT_OF_SYNC but no files found, show warning - if [ "$SYNC_STATUS" = "OUT_OF_SYNC" ] && [ "$DRIFT_COUNT" -eq 0 ]; then - echo " ⚠️ WARNING: Status is OUT_OF_SYNC but no drifted files were extracted" - echo " ⚠️ This might indicate a parsing issue. Check the playbook output above." - fi - - echo " Total drift count: $DRIFT_COUNT" - echo "" -fi - -# ───────────────────────────────────────────────────────────────────────────────── -# Step 3: Build JSON payload -# ───────────────────────────────────────────────────────────────────────────────── -echo "Step 3/4: Building JSON payload..." - -# Determine which field to populate based on mode -if [ "$MODE" = "post-deploy" ]; then - # Post-deploy: populate deployed_files, keep drifted_files empty - DEPLOYED_FILES_JSON="[" - DRIFTED_FILES_JSON="[]" - - if [ ${#CHANGED_FILES[@]} -gt 0 ]; then - for i in "${!CHANGED_FILES[@]}"; do - if [ "$i" -gt 0 ]; then - DEPLOYED_FILES_JSON+="," - fi - escaped_name="${CHANGED_FILES[$i]//\\/\\\\}" - escaped_name="${escaped_name//\"/\\\"}" - DEPLOYED_FILES_JSON+="{\"name\":\"$escaped_name\"}" - done - fi - DEPLOYED_FILES_JSON+="]" -else - # Drift-check mode: Keep existing deployed_files, only update drifted_files - # First fetch current status to preserve deployed_files - echo " Fetching current deployed_files from API..." - CURRENT_JSON=$(curl -s "$GITOPS_STATUS_SERVER_URL/api/status" 2>/dev/null || echo "{}") - - # Extract deployed_files from current JSON (use jq if available) - if command -v jq >/dev/null 2>&1; then - DEPLOYED_FILES_JSON=$(echo "$CURRENT_JSON" | jq -c '.deployed_files // []' 2>/dev/null || echo "[]") - echo " Preserving deployed_files: $DEPLOYED_FILES_JSON" - else - DEPLOYED_FILES_JSON="[]" - echo " WARNING: jq not available, cannot preserve deployed_files" - fi - - # Build drifted_files from detected drift - DRIFTED_FILES_JSON="[" - if [ ${#CHANGED_FILES[@]} -gt 0 ]; then - for i in "${!CHANGED_FILES[@]}"; do - if [ "$i" -gt 0 ]; then - DRIFTED_FILES_JSON+="," - fi - escaped_name="${CHANGED_FILES[$i]//\\/\\\\}" - escaped_name="${escaped_name//\"/\\\"}" - DRIFTED_FILES_JSON+="{\"name\":\"$escaped_name\"}" - done - fi - DRIFTED_FILES_JSON+="]" -fi - -# Generate ISO 8601 timestamp -TIMESTAMP=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - -# Build complete JSON status with both fields -STATUS_JSON=$(cat </dev/null || echo "$STATUS_JSON" -echo "" - -# ───────────────────────────────────────────────────────────────────────────────── -# Step 4: Send JSON to gitops-status-server -# ───────────────────────────────────────────────────────────────────────────────── -echo "Step 4/4: Sending status to gitops-status-server..." -echo " URL: $GITOPS_STATUS_SERVER_URL/api/status" -echo " Method: POST" -echo " Request Payload:" -echo "$STATUS_JSON" | jq '.' 2>/dev/null | sed 's/^/ /' || echo "$STATUS_JSON" | sed 's/^/ /' -echo "" -echo "Step 4/4: Sending status to gitops-status-server..." -echo " URL: $GITOPS_STATUS_SERVER_URL/api/status" -echo " Method: POST" -echo " Payload size: $(echo "$STATUS_JSON" | wc -c) bytes" -echo "" -echo " ========== JSON PAYLOAD BEING SENT ==========" -echo "$STATUS_JSON" | jq '.' 2>/dev/null || echo "$STATUS_JSON" -echo " ==========================================" -echo "" - -# Test connectivity first -echo " Testing connectivity to gitops-status-server..." -if ! curl -s -m 5 "$GITOPS_STATUS_SERVER_URL/health" > /dev/null 2>&1; then - echo " ✗ WARNING: Cannot reach $GITOPS_STATUS_SERVER_URL/health" - echo " Attempting DNS resolution..." - nslookup gitops-status-server.observability-stack.svc.cluster.local || true - echo "" -else - echo " ✓ Server is reachable" -fi -echo "" - -# Create temporary files for response -RESPONSE_BODY=$(mktemp) -trap "rm -f $RESPONSE_BODY $PLAYBOOK_LOG" EXIT - -echo " Sending POST request with curl..." -echo " Command: curl -X POST -H 'Content-Type: application/json' -d '' $GITOPS_STATUS_SERVER_URL/api/status" -echo "" - -# POST the JSON to the gitops-status-server API with full error reporting -# Capture both response code and body for debugging -set +e -HTTP_RESPONSE=$(curl -s -w "\n%{http_code}" \ - -X POST \ - -H "Content-Type: application/json" \ - -d "$STATUS_JSON" \ - "$GITOPS_STATUS_SERVER_URL/api/status" \ - 2>&1) -CURL_EXIT=$? -set -e - -if [ $CURL_EXIT -ne 0 ]; then - echo " ✗ CURL FAILED with exit code $CURL_EXIT" - echo " Error output: $HTTP_RESPONSE" - exit 1 -fi - -# Split response: body is everything except last line, code is last line -HTTP_CODE=$(echo "$HTTP_RESPONSE" | tail -1) -RESPONSE_CONTENT=$(echo "$HTTP_RESPONSE" | head -n -1) -echo "$RESPONSE_CONTENT" > "$RESPONSE_BODY" - -# Validate HTTP code is numeric -if ! [[ "$HTTP_CODE" =~ ^[0-9]+$ ]]; then - echo " ✗ ERROR: Invalid HTTP response code: $HTTP_CODE" - echo " Full response: $HTTP_RESPONSE" - exit 1 -fi - -echo " Response: HTTP $HTTP_CODE" - -# Show response body for debugging (especially on error) -if [ -s "$RESPONSE_BODY" ]; then - echo " Response Body:" - sed 's/^/ /' "$RESPONSE_BODY" -fi -echo "" - -if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then - echo "═══════════════════════════════════════════════════════════════════════════════" - echo " ✓ Status update successful (HTTP $HTTP_CODE)" - echo " JSON has been sent to gitops-status-server" - echo "" - - # Verify the JSON was actually received and stored - echo " Verifying JSON storage on gitops-status-server..." - sleep 1 # Brief delay to ensure server processed the POST - - VERIFY_JSON=$(curl -s "$GITOPS_STATUS_SERVER_URL/api/status" 2>&1 || true) - if echo "$VERIFY_JSON" | grep -q "\"repo\".*\"$REPO_NAME\""; then - echo " ✓ Verified: Latest JSON stored correctly on server" - echo "" - echo " Grafana Infinity datasource will now read the updated JSON from:" - echo " $GITOPS_STATUS_SERVER_URL/api/status" - else - echo " ⚠ Warning: Could not verify JSON storage" - echo " Response from /status.json:" - echo " $VERIFY_JSON" | sed 's/^/ /' - fi - - echo "═══════════════════════════════════════════════════════════════════════════════" - exit 0 -else - echo "═══════════════════════════════════════════════════════════════════════════════" - echo " ✗ ERROR: Status update failed with HTTP $HTTP_CODE" - echo " Debugging Information:" - echo " - Server URL: $GITOPS_STATUS_SERVER_URL" - echo " - Endpoint: /api/status" - echo " - Check gitops-status-server connectivity (ping/nslookup)" - echo " - Verify service port and internal port mapping" - echo " - Ensure /api/status endpoint accepts POST requests" - echo "═══════════════════════════════════════════════════════════════════════════════" - exit 1 -fi