Migrate to infinity datasource
This commit is contained in:
parent
3282870e8f
commit
082ed0a0a4
@ -8,13 +8,15 @@
|
|||||||
# so a drift-check here would always be OUT_OF_SYNC by
|
# so a drift-check here would always be OUT_OF_SYNC by
|
||||||
# design and is meaningless as a failure signal.
|
# design and is meaningless as a failure signal.
|
||||||
#
|
#
|
||||||
# push (master) → syntax-check, validate, deploy, update-sync-metric
|
# push (master) → syntax-check, validate, deploy, update-gitops-status
|
||||||
# Deploys to the server, then verifies sync and pushes metric.
|
# Deploys to the server, then verifies sync and sends
|
||||||
|
# JSON status snapshot to gitops-status-server for Grafana.
|
||||||
#
|
#
|
||||||
# cron → gitops_sync_check (read-only drift check, no deploy)
|
# cron → gitops_sync_check (read-only drift check, no deploy)
|
||||||
# Continuously verifies that the live server still matches
|
# Continuously verifies that the live server still matches
|
||||||
# Git even when no push has happened. Detects manual edits
|
# Git even when no push has happened. Detects manual edits
|
||||||
# made directly on the server.
|
# made directly on the server. Sends JSON status with
|
||||||
|
# detailed file-level drift information to gitops-status-server.
|
||||||
#
|
#
|
||||||
# NOTE: Woodpecker does not support multiple YAML documents (---) in one file.
|
# NOTE: Woodpecker does not support multiple YAML documents (---) in one file.
|
||||||
# All pipelines must live in a single document with step-level filtering.
|
# All pipelines must live in a single document with step-level filtering.
|
||||||
@ -80,44 +82,38 @@ steps:
|
|||||||
event: push
|
event: push
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# update-sync-metric: Post-deploy sync check + Prometheus metric push
|
# update-gitops-status: Post-deploy sync check + JSON status update
|
||||||
# Runs on push to master only, after deploy succeeds.
|
# Runs on push to master only, after deploy succeeds.
|
||||||
# STATUS=1 means SYNCED, STATUS=0 means OUT_OF_SYNC.
|
# Generates structured JSON with sync status, drift count, and changed files.
|
||||||
|
# Sends JSON to gitops-status-server for Grafana visualization.
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
update-sync-metric:
|
update-gitops-status:
|
||||||
image: alpine/ansible:latest
|
image: alpine/ansible:latest
|
||||||
depends_on: [deploy]
|
depends_on: [deploy]
|
||||||
environment:
|
environment:
|
||||||
ANSIBLE_CONFIG: ansible.cfg
|
ANSIBLE_CONFIG: ansible.cfg
|
||||||
SSH_PRIVATE_KEY:
|
SSH_PRIVATE_KEY:
|
||||||
from_secret: SSH_PRIVATE_KEY
|
from_secret: SSH_PRIVATE_KEY
|
||||||
PUSHGATEWAY_URL: http://pushgateway.observability-stack.svc.cluster.local:9091
|
GITOPS_STATUS_SERVER_URL: http://gitops-status-server.observability-stack.svc.cluster.local:80
|
||||||
|
REPO_NAME: rsyslog
|
||||||
|
SERVER_NAME: rsyslog-lab
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
apk add --no-cache curl > /dev/null 2>&1
|
# Install dependencies: curl for HTTP requests, jq for JSON formatting
|
||||||
|
apk add --no-cache curl jq > /dev/null 2>&1
|
||||||
|
|
||||||
|
# Setup SSH key for Ansible
|
||||||
mkdir -p ~/.ssh
|
mkdir -p ~/.ssh
|
||||||
printf '%s\n' "$${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
printf '%s\n' "$${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
||||||
chmod 600 ~/.ssh/id_rsa
|
chmod 600 ~/.ssh/id_rsa
|
||||||
|
|
||||||
echo "==> Verifying post-deploy sync status..."
|
echo "==> Running post-deploy GitOps status check..."
|
||||||
set +e
|
|
||||||
ansible-playbook -i ansible/inventory/hosts.yml ansible/playbooks/drift-check.yml
|
|
||||||
DRIFT_RC=$?
|
|
||||||
set -e
|
|
||||||
|
|
||||||
if [ "$DRIFT_RC" -eq 0 ]; then
|
# Make script executable and run it
|
||||||
STATUS=1
|
chmod +x update-gitops-status.sh
|
||||||
echo "==> SYNCED (1) – server configuration matches Git"
|
./update-gitops-status.sh
|
||||||
else
|
|
||||||
STATUS=0
|
|
||||||
echo "==> OUT OF SYNC (0) – drift detected after deploy"
|
|
||||||
fi
|
|
||||||
|
|
||||||
printf 'gitops_sync_status{repo="rsyslog",server="rsyslog-lab"} %s\n' "$STATUS" | \
|
echo "==> JSON status update complete. Pipeline always succeeds."
|
||||||
curl --silent --show-error --fail --data-binary @- \
|
|
||||||
"$${PUSHGATEWAY_URL}/metrics/job/gitops_rsyslog/instance/rsyslog-lab"
|
|
||||||
|
|
||||||
echo "==> Metric pushed. Pipeline always succeeds; sync status is in Prometheus."
|
|
||||||
when:
|
when:
|
||||||
branch: master
|
branch: master
|
||||||
event: push
|
event: push
|
||||||
@ -125,7 +121,8 @@ steps:
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# gitops_sync_check: ArgoCD-style cron drift check – read-only, no deploy
|
# gitops_sync_check: ArgoCD-style cron drift check – read-only, no deploy
|
||||||
# Detects manual changes made directly on the server between pushes.
|
# Detects manual changes made directly on the server between pushes.
|
||||||
# STATUS=1 → SYNCED, STATUS=0 → OUT_OF_SYNC
|
# Generates structured JSON with sync status, drift count, and changed files.
|
||||||
|
# Sends JSON to gitops-status-server for continuous GitOps monitoring.
|
||||||
# Pipeline marked FAILED when drift found so it is visible in the UI.
|
# Pipeline marked FAILED when drift found so it is visible in the UI.
|
||||||
#
|
#
|
||||||
# ─── Woodpecker Cron UI settings ──────────────────────────────────────────
|
# ─── Woodpecker Cron UI settings ──────────────────────────────────────────
|
||||||
@ -139,34 +136,50 @@ steps:
|
|||||||
ANSIBLE_CONFIG: ansible.cfg
|
ANSIBLE_CONFIG: ansible.cfg
|
||||||
SSH_PRIVATE_KEY:
|
SSH_PRIVATE_KEY:
|
||||||
from_secret: SSH_PRIVATE_KEY
|
from_secret: SSH_PRIVATE_KEY
|
||||||
PUSHGATEWAY_URL: http://pushgateway.observability-stack.svc.cluster.local:9091
|
GITOPS_STATUS_SERVER_URL: http://gitops-status-server.observability-stack.svc.cluster.local:80
|
||||||
|
REPO_NAME: rsyslog
|
||||||
|
SERVER_NAME: rsyslog-lab
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
apk add --no-cache curl > /dev/null 2>&1
|
# Install dependencies: curl for HTTP requests, jq for JSON formatting
|
||||||
|
apk add --no-cache curl jq bash > /dev/null 2>&1
|
||||||
|
|
||||||
|
# Setup SSH key for Ansible
|
||||||
mkdir -p ~/.ssh
|
mkdir -p ~/.ssh
|
||||||
printf '%s\n' "$${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
printf '%s\n' "$${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
||||||
chmod 600 ~/.ssh/id_rsa
|
chmod 600 ~/.ssh/id_rsa
|
||||||
|
|
||||||
echo "==> [cron] Running drift check against remote server..."
|
echo "==> [cron] Running continuous GitOps drift check..."
|
||||||
|
|
||||||
|
# Make script executable and run it
|
||||||
|
chmod +x update-gitops-status.sh
|
||||||
|
|
||||||
|
# Capture exit code to determine if drift was detected
|
||||||
set +e
|
set +e
|
||||||
ansible-playbook -i ansible/inventory/hosts.yml ansible/playbooks/drift-check.yml
|
./update-gitops-status.sh
|
||||||
|
SCRIPT_RC=$?
|
||||||
|
set -e
|
||||||
|
|
||||||
|
if [ "$SCRIPT_RC" -ne 0 ]; then
|
||||||
|
echo "==> ERROR: Status update failed"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check sync status to determine pipeline result
|
||||||
|
# Read the generated JSON or re-run drift check
|
||||||
|
echo "==> Verifying drift status for pipeline result..."
|
||||||
|
set +e
|
||||||
|
ansible-playbook -i ansible/inventory/hosts.yml ansible/playbooks/drift-check.yml > /dev/null 2>&1
|
||||||
DRIFT_RC=$?
|
DRIFT_RC=$?
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
if [ "$DRIFT_RC" -eq 0 ]; then
|
if [ "$DRIFT_RC" -eq 0 ]; then
|
||||||
STATUS=1
|
echo "==> Pipeline SUCCESS: Server is SYNCED with Git"
|
||||||
echo "==> STATUS: SYNCED (1) – server configuration matches Git"
|
exit 0
|
||||||
else
|
else
|
||||||
STATUS=0
|
echo "==> Pipeline FAILED: OUT OF SYNC - manual drift detected on server"
|
||||||
echo "==> STATUS: OUT OF SYNC (0) – manual drift detected on server"
|
echo " Check gitops-status-server for detailed file-level drift information"
|
||||||
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "==> Pushing metric: gitops_sync_status{repo=\"rsyslog\",server=\"rsyslog-lab\"} $STATUS"
|
|
||||||
printf 'gitops_sync_status{repo="rsyslog",server="rsyslog-lab"} %s\n' "$STATUS" | \
|
|
||||||
curl --silent --show-error --fail --data-binary @- \
|
|
||||||
"$${PUSHGATEWAY_URL}/metrics/job/gitops_rsyslog/instance/rsyslog-lab"
|
|
||||||
|
|
||||||
echo "==> Metric pushed. Pipeline always succeeds; sync status is in Prometheus."
|
|
||||||
when:
|
when:
|
||||||
event: cron
|
event: cron
|
||||||
|
|||||||
310
DEPLOYMENT_CHECKLIST.md
Normal file
310
DEPLOYMENT_CHECKLIST.md
Normal file
@ -0,0 +1,310 @@
|
|||||||
|
# Deployment Checklist
|
||||||
|
|
||||||
|
This checklist guides you through deploying the updated rsyslog repository with gitops-status-server integration.
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
Before deploying, ensure:
|
||||||
|
|
||||||
|
- [ ] gitops-status-server is deployed and accessible at:
|
||||||
|
`http://gitops-status-server.observability-stack.svc.cluster.local:80`
|
||||||
|
- [ ] gitops-status-server has `/api/status` endpoint implemented (see [GITOPS_STATUS_API_REFERENCE.md](GITOPS_STATUS_API_REFERENCE.md))
|
||||||
|
- [ ] Woodpecker CI is configured for this repository
|
||||||
|
- [ ] SSH access to rsyslog-lab server is configured
|
||||||
|
- [ ] SSH_PRIVATE_KEY secret is set in Woodpecker repository settings
|
||||||
|
|
||||||
|
## Files Changed/Added
|
||||||
|
|
||||||
|
### New Files
|
||||||
|
- ✅ `update-gitops-status.sh` - Main script for JSON status generation
|
||||||
|
- ✅ `GITOPS_STATUS_INTEGRATION.md` - Integration documentation
|
||||||
|
- ✅ `GITOPS_STATUS_API_REFERENCE.md` - API reference with examples
|
||||||
|
- ✅ `MIGRATION_SUMMARY.md` - Summary of all changes made
|
||||||
|
|
||||||
|
### Modified Files
|
||||||
|
- ✅ `.woodpecker.yml` - Updated pipeline to use JSON status
|
||||||
|
- ✅ `README.md` - Updated documentation with new flow
|
||||||
|
|
||||||
|
### Unchanged Files (no action needed)
|
||||||
|
- ✅ All Ansible playbooks
|
||||||
|
- ✅ All Ansible inventory files
|
||||||
|
- ✅ All rsyslog config files
|
||||||
|
- ✅ Local scripts (apply.sh, drift-check.sh)
|
||||||
|
|
||||||
|
## Deployment Steps
|
||||||
|
|
||||||
|
### 1. Review Changes
|
||||||
|
|
||||||
|
- [ ] Read [MIGRATION_SUMMARY.md](MIGRATION_SUMMARY.md) to understand all changes
|
||||||
|
- [ ] Review [.woodpecker.yml](.woodpecker.yml) pipeline changes
|
||||||
|
- [ ] Review [update-gitops-status.sh](update-gitops-status.sh) script logic
|
||||||
|
|
||||||
|
### 2. Verify gitops-status-server
|
||||||
|
|
||||||
|
Test the gitops-status-server API endpoint:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Test POST endpoint (should return 200 or 404 if not implemented yet)
|
||||||
|
curl -X POST http://gitops-status-server.observability-stack.svc.cluster.local:80/api/status \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{
|
||||||
|
"repo": "test",
|
||||||
|
"server": "test",
|
||||||
|
"sync_status": "SYNCED",
|
||||||
|
"drift_count": 0,
|
||||||
|
"files": [],
|
||||||
|
"last_check": "2026-04-21T10:00:00Z"
|
||||||
|
}'
|
||||||
|
|
||||||
|
# Test GET endpoint
|
||||||
|
curl http://gitops-status-server.observability-stack.svc.cluster.local:80/status.json
|
||||||
|
```
|
||||||
|
|
||||||
|
If the API is not implemented yet:
|
||||||
|
- [ ] Implement gitops-status-server API (use [GITOPS_STATUS_API_REFERENCE.md](GITOPS_STATUS_API_REFERENCE.md))
|
||||||
|
- [ ] Deploy to Kubernetes cluster
|
||||||
|
- [ ] Verify endpoints are accessible
|
||||||
|
|
||||||
|
### 3. Test Locally (Optional)
|
||||||
|
|
||||||
|
Before pushing to Git, you can test the script locally:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Set environment variables
|
||||||
|
export GITOPS_STATUS_SERVER_URL="http://gitops-status-server.observability-stack.svc.cluster.local:80"
|
||||||
|
export REPO_NAME="rsyslog"
|
||||||
|
export SERVER_NAME="rsyslog-lab"
|
||||||
|
|
||||||
|
# Make script executable
|
||||||
|
chmod +x update-gitops-status.sh
|
||||||
|
|
||||||
|
# Run the script (requires Ansible, jq, curl)
|
||||||
|
./update-gitops-status.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected output:
|
||||||
|
```
|
||||||
|
==> Running drift check playbook...
|
||||||
|
Inventory: ansible/inventory/hosts.yml
|
||||||
|
Playbook: ansible/playbooks/drift-check.yml
|
||||||
|
==> Status: SYNCED - server configuration matches Git
|
||||||
|
==> Drift count: 0
|
||||||
|
==> Generated JSON status:
|
||||||
|
{
|
||||||
|
"repo": "rsyslog",
|
||||||
|
"server": "rsyslog-lab",
|
||||||
|
"sync_status": "SYNCED",
|
||||||
|
"drift_count": 0,
|
||||||
|
"files": [],
|
||||||
|
"last_check": "2026-04-21T10:30:00Z"
|
||||||
|
}
|
||||||
|
==> Sending status to gitops-status-server...
|
||||||
|
URL: http://gitops-status-server.observability-stack.svc.cluster.local:80/api/status
|
||||||
|
==> Status update successful (HTTP 200)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. Commit and Push Changes
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Stage all changes
|
||||||
|
git add .
|
||||||
|
|
||||||
|
# Commit
|
||||||
|
git commit -m "Migrate from Pushgateway to gitops-status-server JSON status
|
||||||
|
|
||||||
|
- Add update-gitops-status.sh script for JSON status generation
|
||||||
|
- Update .woodpecker.yml to use gitops-status-server
|
||||||
|
- Remove Pushgateway metric push logic
|
||||||
|
- Add comprehensive documentation
|
||||||
|
- Keep all Ansible playbooks unchanged"
|
||||||
|
|
||||||
|
# Push to master (will trigger deploy pipeline)
|
||||||
|
git push origin master
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5. Monitor First Deployment
|
||||||
|
|
||||||
|
After pushing to master:
|
||||||
|
|
||||||
|
- [ ] Watch Woodpecker pipeline execution
|
||||||
|
- [ ] Verify `syntax-check` step passes
|
||||||
|
- [ ] Verify `validate` step passes
|
||||||
|
- [ ] Verify `deploy` step completes
|
||||||
|
- [ ] Verify `update-gitops-status` step runs successfully
|
||||||
|
- [ ] Check that JSON is sent to gitops-status-server
|
||||||
|
|
||||||
|
Example successful `update-gitops-status` step output:
|
||||||
|
```
|
||||||
|
==> Running post-deploy GitOps status check...
|
||||||
|
==> Running drift check playbook...
|
||||||
|
==> Status: SYNCED - server configuration matches Git
|
||||||
|
==> Drift count: 0
|
||||||
|
==> Generated JSON status: {...}
|
||||||
|
==> Sending status to gitops-status-server...
|
||||||
|
==> Status update successful (HTTP 200)
|
||||||
|
==> JSON status update complete. Pipeline always succeeds.
|
||||||
|
```
|
||||||
|
|
||||||
|
### 6. Verify Cron Pipeline
|
||||||
|
|
||||||
|
The cron pipeline runs every 2 minutes:
|
||||||
|
|
||||||
|
- [ ] Wait for next cron execution
|
||||||
|
- [ ] Check Woodpecker for `gitops_sync_check` pipeline run
|
||||||
|
- [ ] Verify JSON status is sent
|
||||||
|
- [ ] Verify pipeline succeeds (if synced) or fails (if drift detected)
|
||||||
|
|
||||||
|
### 7. Test Drift Detection
|
||||||
|
|
||||||
|
Manually create drift to test detection:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# SSH to the server
|
||||||
|
ssh rsyslog-lab
|
||||||
|
|
||||||
|
# Edit a config file
|
||||||
|
echo "# manual edit" >> /etc/rsyslog.conf
|
||||||
|
|
||||||
|
# Wait up to 2 minutes for next cron run
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected behavior:
|
||||||
|
- [ ] Cron pipeline runs
|
||||||
|
- [ ] Drift detected in Ansible playbook
|
||||||
|
- [ ] JSON sent with `sync_status: "OUT_OF_SYNC"`
|
||||||
|
- [ ] JSON includes `files: [{"name": "rsyslog.conf"}]`
|
||||||
|
- [ ] Pipeline marked as FAILED (for visibility)
|
||||||
|
|
||||||
|
Verify in gitops-status-server:
|
||||||
|
```bash
|
||||||
|
curl http://gitops-status-server.observability-stack.svc.cluster.local:80/status.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Should show:
|
||||||
|
```json
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"repo": "rsyslog",
|
||||||
|
"server": "rsyslog-lab",
|
||||||
|
"sync_status": "OUT_OF_SYNC",
|
||||||
|
"drift_count": 1,
|
||||||
|
"files": [
|
||||||
|
{"name": "rsyslog.conf"}
|
||||||
|
],
|
||||||
|
"last_check": "...",
|
||||||
|
"updated_at": "..."
|
||||||
|
}
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
### 8. Configure Grafana Dashboard
|
||||||
|
|
||||||
|
- [ ] Add Infinity datasource pointing to gitops-status-server
|
||||||
|
- [ ] Create dashboard to display GitOps status
|
||||||
|
- [ ] Add panels for:
|
||||||
|
- Sync status overview (SYNCED vs OUT_OF_SYNC)
|
||||||
|
- Drift count per repo
|
||||||
|
- Detailed file list for drifted repos
|
||||||
|
- Last check timestamp
|
||||||
|
- Historical trend (if gitops-status-server stores history)
|
||||||
|
|
||||||
|
Example dashboard queries provided in [GITOPS_STATUS_API_REFERENCE.md](GITOPS_STATUS_API_REFERENCE.md).
|
||||||
|
|
||||||
|
### 9. Cleanup (Optional)
|
||||||
|
|
||||||
|
If everything works correctly:
|
||||||
|
|
||||||
|
- [ ] Remove old Pushgateway metrics for rsyslog (if no longer needed)
|
||||||
|
- [ ] Update any alerts/dashboards that used old `gitops_sync_status` metric
|
||||||
|
- [ ] Document the new JSON status format in team wiki/docs
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Script fails with "command not found: jq"
|
||||||
|
|
||||||
|
**Problem:** `jq` is not installed in the Woodpecker container
|
||||||
|
|
||||||
|
**Solution:** The `.woodpecker.yml` already includes `apk add --no-cache jq`. Verify the step runs before the script.
|
||||||
|
|
||||||
|
### Script fails with "HTTP 404" or "HTTP 500"
|
||||||
|
|
||||||
|
**Problem:** gitops-status-server endpoint not implemented or not accessible
|
||||||
|
|
||||||
|
**Solution:**
|
||||||
|
1. Verify gitops-status-server is running: `curl http://gitops-status-server.observability-stack.svc.cluster.local:80/health`
|
||||||
|
2. Check Kubernetes service: `kubectl get svc -n observability-stack`
|
||||||
|
3. Implement `/api/status` endpoint using [GITOPS_STATUS_API_REFERENCE.md](GITOPS_STATUS_API_REFERENCE.md)
|
||||||
|
|
||||||
|
### Script fails with "No such file or directory: update-gitops-status.sh"
|
||||||
|
|
||||||
|
**Problem:** Script not found in workspace
|
||||||
|
|
||||||
|
**Solution:** The script is created in the repository root. Verify it's committed to Git and available in the CI workspace.
|
||||||
|
|
||||||
|
### Drift not detected when expected
|
||||||
|
|
||||||
|
**Problem:** Manual changes not showing up in drift check
|
||||||
|
|
||||||
|
**Solution:**
|
||||||
|
1. Verify changes are to files managed by Git (rsyslog.conf or rsyslog.d/*.conf)
|
||||||
|
2. Check Ansible playbook output for diff details
|
||||||
|
3. Verify SSH access to server from CI container
|
||||||
|
|
||||||
|
### JSON has empty files array even when drift detected
|
||||||
|
|
||||||
|
**Problem:** Script parsing logic not extracting filenames correctly
|
||||||
|
|
||||||
|
**Solution:**
|
||||||
|
1. Check Ansible output format - script expects specific JSON structure
|
||||||
|
2. Run with `ANSIBLE_STDOUT_CALLBACK=json` to see raw output
|
||||||
|
3. Update regex patterns in `update-gitops-status.sh` if needed
|
||||||
|
|
||||||
|
## Rollback Procedure
|
||||||
|
|
||||||
|
If you need to rollback to Pushgateway:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Revert to previous commit
|
||||||
|
git revert HEAD
|
||||||
|
|
||||||
|
# Or restore specific files
|
||||||
|
git checkout HEAD~1 .woodpecker.yml
|
||||||
|
git checkout HEAD~1 README.md
|
||||||
|
|
||||||
|
# Remove new files
|
||||||
|
git rm update-gitops-status.sh GITOPS_STATUS_*.md MIGRATION_SUMMARY.md
|
||||||
|
|
||||||
|
# Commit and push
|
||||||
|
git commit -m "Rollback to Pushgateway metrics"
|
||||||
|
git push origin master
|
||||||
|
```
|
||||||
|
|
||||||
|
## Success Criteria
|
||||||
|
|
||||||
|
✅ All checks passed when:
|
||||||
|
|
||||||
|
- [ ] Woodpecker pipeline completes successfully on push to master
|
||||||
|
- [ ] JSON status is sent to gitops-status-server after deploy
|
||||||
|
- [ ] Cron pipeline runs every 2 minutes and sends JSON status
|
||||||
|
- [ ] Drift is correctly detected and reported in JSON
|
||||||
|
- [ ] gitops-status-server `/status.json` endpoint returns correct data
|
||||||
|
- [ ] Grafana dashboard displays rsyslog sync status
|
||||||
|
- [ ] No errors in Woodpecker logs
|
||||||
|
- [ ] File-level drift details are visible in Grafana
|
||||||
|
|
||||||
|
## Additional Resources
|
||||||
|
|
||||||
|
- [README.md](README.md) - Repository overview and workflow
|
||||||
|
- [MIGRATION_SUMMARY.md](MIGRATION_SUMMARY.md) - Detailed migration changes
|
||||||
|
- [GITOPS_STATUS_INTEGRATION.md](GITOPS_STATUS_INTEGRATION.md) - Integration architecture
|
||||||
|
- [GITOPS_STATUS_API_REFERENCE.md](GITOPS_STATUS_API_REFERENCE.md) - API implementation guide
|
||||||
|
|
||||||
|
## Support
|
||||||
|
|
||||||
|
If you encounter issues:
|
||||||
|
|
||||||
|
1. Check Woodpecker pipeline logs
|
||||||
|
2. Verify gitops-status-server logs
|
||||||
|
3. Test API endpoints manually with curl
|
||||||
|
4. Review Ansible playbook output
|
||||||
|
5. Check this repository's documentation files
|
||||||
326
GITOPS_STATUS_API_REFERENCE.md
Normal file
326
GITOPS_STATUS_API_REFERENCE.md
Normal file
@ -0,0 +1,326 @@
|
|||||||
|
# gitops-status-server API Reference
|
||||||
|
|
||||||
|
This document provides a reference implementation example for the gitops-status-server API endpoint that receives status updates from the rsyslog repository.
|
||||||
|
|
||||||
|
## API Endpoint Specification
|
||||||
|
|
||||||
|
### POST /api/status
|
||||||
|
|
||||||
|
Receives GitOps status updates from repositories.
|
||||||
|
|
||||||
|
**Request:**
|
||||||
|
```
|
||||||
|
POST /api/status HTTP/1.1
|
||||||
|
Host: gitops-status-server.observability-stack.svc.cluster.local:80
|
||||||
|
Content-Type: application/json
|
||||||
|
|
||||||
|
{
|
||||||
|
"repo": "rsyslog",
|
||||||
|
"server": "rsyslog-lab",
|
||||||
|
"sync_status": "OUT_OF_SYNC",
|
||||||
|
"drift_count": 2,
|
||||||
|
"files": [
|
||||||
|
{ "name": "rsyslog.conf" },
|
||||||
|
{ "name": "rsyslog.d/30-lab.conf" }
|
||||||
|
],
|
||||||
|
"last_check": "2026-04-21T10:32:15Z"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response (Success):**
|
||||||
|
```
|
||||||
|
HTTP/1.1 200 OK
|
||||||
|
Content-Type: application/json
|
||||||
|
|
||||||
|
{
|
||||||
|
"status": "ok",
|
||||||
|
"message": "Status updated successfully"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response (Error):**
|
||||||
|
```
|
||||||
|
HTTP/1.1 400 Bad Request
|
||||||
|
Content-Type: application/json
|
||||||
|
|
||||||
|
{
|
||||||
|
"status": "error",
|
||||||
|
"message": "Invalid JSON payload"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Example Implementation (Python/Flask)
|
||||||
|
|
||||||
|
```python
|
||||||
|
from flask import Flask, request, jsonify
|
||||||
|
from datetime import datetime
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
app = Flask(__name__)
|
||||||
|
|
||||||
|
# In-memory storage (replace with database in production)
|
||||||
|
status_data = {}
|
||||||
|
|
||||||
|
@app.route('/api/status', methods=['POST'])
|
||||||
|
def update_status():
|
||||||
|
"""Receive and store GitOps status updates"""
|
||||||
|
try:
|
||||||
|
data = request.get_json()
|
||||||
|
|
||||||
|
# Validate required fields
|
||||||
|
required_fields = ['repo', 'server', 'sync_status', 'drift_count', 'files', 'last_check']
|
||||||
|
for field in required_fields:
|
||||||
|
if field not in data:
|
||||||
|
return jsonify({
|
||||||
|
'status': 'error',
|
||||||
|
'message': f'Missing required field: {field}'
|
||||||
|
}), 400
|
||||||
|
|
||||||
|
# Validate sync_status value
|
||||||
|
if data['sync_status'] not in ['SYNCED', 'OUT_OF_SYNC']:
|
||||||
|
return jsonify({
|
||||||
|
'status': 'error',
|
||||||
|
'message': 'sync_status must be SYNCED or OUT_OF_SYNC'
|
||||||
|
}), 400
|
||||||
|
|
||||||
|
# Create unique key for this repo/server combination
|
||||||
|
key = f"{data['repo']}:{data['server']}"
|
||||||
|
|
||||||
|
# Store the status
|
||||||
|
status_data[key] = {
|
||||||
|
'repo': data['repo'],
|
||||||
|
'server': data['server'],
|
||||||
|
'sync_status': data['sync_status'],
|
||||||
|
'drift_count': data['drift_count'],
|
||||||
|
'files': data['files'],
|
||||||
|
'last_check': data['last_check'],
|
||||||
|
'updated_at': datetime.utcnow().isoformat() + 'Z'
|
||||||
|
}
|
||||||
|
|
||||||
|
# Log the update
|
||||||
|
print(f"Status update: {key} -> {data['sync_status']} (drift_count: {data['drift_count']})")
|
||||||
|
|
||||||
|
return jsonify({
|
||||||
|
'status': 'ok',
|
||||||
|
'message': 'Status updated successfully'
|
||||||
|
}), 200
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error processing status update: {e}")
|
||||||
|
return jsonify({
|
||||||
|
'status': 'error',
|
||||||
|
'message': str(e)
|
||||||
|
}), 500
|
||||||
|
|
||||||
|
|
||||||
|
@app.route('/status.json', methods=['GET'])
|
||||||
|
def get_status():
|
||||||
|
"""Serve aggregated status for Grafana Infinity datasource"""
|
||||||
|
# Convert dict to list for JSON array output
|
||||||
|
statuses = list(status_data.values())
|
||||||
|
|
||||||
|
return jsonify(statuses), 200
|
||||||
|
|
||||||
|
|
||||||
|
@app.route('/health', methods=['GET'])
|
||||||
|
def health():
|
||||||
|
"""Health check endpoint"""
|
||||||
|
return jsonify({
|
||||||
|
'status': 'healthy',
|
||||||
|
'timestamp': datetime.utcnow().isoformat() + 'Z',
|
||||||
|
'tracked_repos': len(status_data)
|
||||||
|
}), 200
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
app.run(host='0.0.0.0', port=8080)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Example Implementation (Go)
|
||||||
|
|
||||||
|
```go
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type StatusUpdate struct {
|
||||||
|
Repo string `json:"repo"`
|
||||||
|
Server string `json:"server"`
|
||||||
|
SyncStatus string `json:"sync_status"`
|
||||||
|
DriftCount int `json:"drift_count"`
|
||||||
|
Files []File `json:"files"`
|
||||||
|
LastCheck string `json:"last_check"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type File struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type StoredStatus struct {
|
||||||
|
StatusUpdate
|
||||||
|
UpdatedAt string `json:"updated_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
statusStore = make(map[string]StoredStatus)
|
||||||
|
storeMutex sync.RWMutex
|
||||||
|
)
|
||||||
|
|
||||||
|
func updateStatusHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodPost {
|
||||||
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var status StatusUpdate
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&status); err != nil {
|
||||||
|
http.Error(w, fmt.Sprintf("Invalid JSON: %v", err), http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate sync_status
|
||||||
|
if status.SyncStatus != "SYNCED" && status.SyncStatus != "OUT_OF_SYNC" {
|
||||||
|
http.Error(w, "sync_status must be SYNCED or OUT_OF_SYNC", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store the status
|
||||||
|
key := fmt.Sprintf("%s:%s", status.Repo, status.Server)
|
||||||
|
stored := StoredStatus{
|
||||||
|
StatusUpdate: status,
|
||||||
|
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||||
|
}
|
||||||
|
|
||||||
|
storeMutex.Lock()
|
||||||
|
statusStore[key] = stored
|
||||||
|
storeMutex.Unlock()
|
||||||
|
|
||||||
|
log.Printf("Status update: %s -> %s (drift_count: %d)", key, status.SyncStatus, status.DriftCount)
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
json.NewEncoder(w).Encode(map[string]string{
|
||||||
|
"status": "ok",
|
||||||
|
"message": "Status updated successfully",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func getStatusHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
storeMutex.RLock()
|
||||||
|
statuses := make([]StoredStatus, 0, len(statusStore))
|
||||||
|
for _, status := range statusStore {
|
||||||
|
statuses = append(statuses, status)
|
||||||
|
}
|
||||||
|
storeMutex.RUnlock()
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
json.NewEncoder(w).Encode(statuses)
|
||||||
|
}
|
||||||
|
|
||||||
|
func healthHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
storeMutex.RLock()
|
||||||
|
count := len(statusStore)
|
||||||
|
storeMutex.RUnlock()
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||||
|
"status": "healthy",
|
||||||
|
"timestamp": time.Now().UTC().Format(time.RFC3339),
|
||||||
|
"tracked_repos": count,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
http.HandleFunc("/api/status", updateStatusHandler)
|
||||||
|
http.HandleFunc("/status.json", getStatusHandler)
|
||||||
|
http.HandleFunc("/health", healthHandler)
|
||||||
|
|
||||||
|
log.Println("Starting gitops-status-server on :8080")
|
||||||
|
log.Fatal(http.ListenAndServe(":8080", nil))
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Testing the API
|
||||||
|
|
||||||
|
### Using curl
|
||||||
|
|
||||||
|
**Send a status update:**
|
||||||
|
```bash
|
||||||
|
curl -X POST http://localhost:8080/api/status \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{
|
||||||
|
"repo": "rsyslog",
|
||||||
|
"server": "rsyslog-lab",
|
||||||
|
"sync_status": "OUT_OF_SYNC",
|
||||||
|
"drift_count": 2,
|
||||||
|
"files": [
|
||||||
|
{"name": "rsyslog.conf"},
|
||||||
|
{"name": "rsyslog.d/30-lab.conf"}
|
||||||
|
],
|
||||||
|
"last_check": "2026-04-21T10:32:15Z"
|
||||||
|
}'
|
||||||
|
```
|
||||||
|
|
||||||
|
**Get all statuses:**
|
||||||
|
```bash
|
||||||
|
curl http://localhost:8080/status.json
|
||||||
|
```
|
||||||
|
|
||||||
|
**Health check:**
|
||||||
|
```bash
|
||||||
|
curl http://localhost:8080/health
|
||||||
|
```
|
||||||
|
|
||||||
|
## Grafana Infinity Datasource Configuration
|
||||||
|
|
||||||
|
1. Install Grafana Infinity datasource plugin
|
||||||
|
2. Add new datasource:
|
||||||
|
- Type: Infinity
|
||||||
|
- URL: `http://gitops-status-server.observability-stack.svc.cluster.local:80`
|
||||||
|
3. Create a panel with query:
|
||||||
|
- URL: `/status.json`
|
||||||
|
- Parser: Backend
|
||||||
|
- Format: Table
|
||||||
|
|
||||||
|
Example query to show all repos:
|
||||||
|
```
|
||||||
|
Source: URL
|
||||||
|
URL: /status.json
|
||||||
|
Parser: Backend
|
||||||
|
Format: Table
|
||||||
|
Columns:
|
||||||
|
- repo (string)
|
||||||
|
- server (string)
|
||||||
|
- sync_status (string)
|
||||||
|
- drift_count (number)
|
||||||
|
- last_check (time)
|
||||||
|
```
|
||||||
|
|
||||||
|
Example query to show drift details:
|
||||||
|
```
|
||||||
|
Source: URL
|
||||||
|
URL: /status.json
|
||||||
|
Parser: Backend
|
||||||
|
Format: Table
|
||||||
|
Root/Rows: $[?(@.drift_count > 0)]
|
||||||
|
Columns:
|
||||||
|
- repo (string)
|
||||||
|
- server (string)
|
||||||
|
- drift_count (number)
|
||||||
|
- files (string, JSONata: $join(files.name, ', '))
|
||||||
|
```
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- The example implementations use in-memory storage; production should use a database
|
||||||
|
- Consider adding authentication/authorization for the POST endpoint
|
||||||
|
- Add monitoring/metrics for the status server itself
|
||||||
|
- Consider adding TTL/expiration for stale status entries
|
||||||
|
- The `/status.json` endpoint should support filtering (e.g., by repo or server)
|
||||||
117
GITOPS_STATUS_INTEGRATION.md
Normal file
117
GITOPS_STATUS_INTEGRATION.md
Normal file
@ -0,0 +1,117 @@
|
|||||||
|
# GitOps Status Server Integration
|
||||||
|
|
||||||
|
This document explains how the rsyslog repository integrates with gitops-status-server for GitOps monitoring.
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
Instead of pushing simple numeric metrics to Prometheus Pushgateway, the rsyslog repo now sends structured JSON status snapshots to gitops-status-server. This enables richer visualization in Grafana with file-level drift details.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────┐ ┌──────────────────────┐ ┌─────────────────────┐
|
||||||
|
│ Woodpecker CI │ │ gitops-status-server │ │ Grafana │
|
||||||
|
│ (rsyslog) │────────►│ (Kubernetes) │────────►│ Infinity Plugin │
|
||||||
|
│ │ POST │ │ GET │ │
|
||||||
|
│ drift-check │ JSON │ serves /status.json │ │ Dashboard shows │
|
||||||
|
│ every 2 min │ │ │ │ drift details │
|
||||||
|
└─────────────────┘ └──────────────────────┘ └─────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## API Endpoint
|
||||||
|
|
||||||
|
The rsyslog repo sends JSON status updates to:
|
||||||
|
|
||||||
|
```
|
||||||
|
POST http://gitops-status-server.observability-stack.svc.cluster.local:80/api/status
|
||||||
|
Content-Type: application/json
|
||||||
|
```
|
||||||
|
|
||||||
|
## JSON Payload Format
|
||||||
|
|
||||||
|
### When synced (no drift)
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"repo": "rsyslog",
|
||||||
|
"server": "rsyslog-lab",
|
||||||
|
"sync_status": "SYNCED",
|
||||||
|
"drift_count": 0,
|
||||||
|
"files": [],
|
||||||
|
"last_check": "2026-04-21T10:30:00Z"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### When drift detected
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"repo": "rsyslog",
|
||||||
|
"server": "rsyslog-lab",
|
||||||
|
"sync_status": "OUT_OF_SYNC",
|
||||||
|
"drift_count": 2,
|
||||||
|
"files": [
|
||||||
|
{ "name": "rsyslog.conf" },
|
||||||
|
{ "name": "rsyslog.d/30-lab.conf" }
|
||||||
|
],
|
||||||
|
"last_check": "2026-04-21T10:32:15Z"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Field Definitions
|
||||||
|
|
||||||
|
| Field | Type | Description |
|
||||||
|
|----------------|----------|-------------------------------------------------------|
|
||||||
|
| `repo` | string | Repository name (e.g., "rsyslog") |
|
||||||
|
| `server` | string | Target server name (e.g., "rsyslog-lab") |
|
||||||
|
| `sync_status` | string | Either "SYNCED" or "OUT_OF_SYNC" |
|
||||||
|
| `drift_count` | integer | Number of files that have drifted from Git |
|
||||||
|
| `files` | array | List of files with drift (empty if synced) |
|
||||||
|
| `files[].name` | string | Relative path of drifted file |
|
||||||
|
| `last_check` | string | ISO 8601 timestamp of when drift check was performed |
|
||||||
|
|
||||||
|
## When Updates Are Sent
|
||||||
|
|
||||||
|
1. **After deployment** (push to master):
|
||||||
|
- Post-deploy verification runs
|
||||||
|
- JSON status sent to gitops-status-server
|
||||||
|
- Pipeline step: `update-gitops-status`
|
||||||
|
|
||||||
|
2. **Scheduled cron check** (every 2 minutes):
|
||||||
|
- Continuous drift monitoring
|
||||||
|
- JSON status sent to gitops-status-server
|
||||||
|
- Pipeline step: `gitops_sync_check`
|
||||||
|
|
||||||
|
## Error Handling
|
||||||
|
|
||||||
|
If the HTTP POST to gitops-status-server fails:
|
||||||
|
- The pipeline step will fail
|
||||||
|
- Error message will be logged
|
||||||
|
- The drift check itself is still performed
|
||||||
|
- No retry logic (next cron run will retry)
|
||||||
|
|
||||||
|
## Script Implementation
|
||||||
|
|
||||||
|
The `update-gitops-status.sh` script handles:
|
||||||
|
1. Running the Ansible drift-check playbook
|
||||||
|
2. Parsing the output to extract changed file names
|
||||||
|
3. Building the JSON payload
|
||||||
|
4. Sending it to gitops-status-server via HTTP POST
|
||||||
|
|
||||||
|
## Expected HTTP Response
|
||||||
|
|
||||||
|
gitops-status-server should respond with:
|
||||||
|
- `200 OK` or `201 Created` on success
|
||||||
|
- `4xx` or `5xx` on error
|
||||||
|
|
||||||
|
The rsyslog pipeline treats any 2xx response as success.
|
||||||
|
|
||||||
|
## Grafana Visualization
|
||||||
|
|
||||||
|
Grafana uses the Infinity datasource plugin to fetch `/status.json` from gitops-status-server and display:
|
||||||
|
- Current sync status (SYNCED vs OUT_OF_SYNC)
|
||||||
|
- Number of drifted files
|
||||||
|
- List of specific files that have drifted
|
||||||
|
- Last check timestamp
|
||||||
|
|
||||||
|
This provides much richer information than a simple numeric metric.
|
||||||
168
MIGRATION_SUMMARY.md
Normal file
168
MIGRATION_SUMMARY.md
Normal file
@ -0,0 +1,168 @@
|
|||||||
|
# Migration Summary: Pushgateway → gitops-status-server
|
||||||
|
|
||||||
|
This document summarizes the changes made to migrate the rsyslog repository from Pushgateway metrics to JSON status updates for gitops-status-server.
|
||||||
|
|
||||||
|
## What Changed
|
||||||
|
|
||||||
|
### 1. New Files Created
|
||||||
|
|
||||||
|
#### `update-gitops-status.sh`
|
||||||
|
- Main script that orchestrates the status update flow
|
||||||
|
- Runs Ansible drift-check playbook
|
||||||
|
- Parses output to extract changed file names
|
||||||
|
- Builds structured JSON payload
|
||||||
|
- Sends JSON to gitops-status-server via HTTP POST
|
||||||
|
- Handles both SYNCED and OUT_OF_SYNC states
|
||||||
|
|
||||||
|
#### `GITOPS_STATUS_INTEGRATION.md`
|
||||||
|
- Documentation explaining the integration with gitops-status-server
|
||||||
|
- API endpoint specification
|
||||||
|
- JSON payload format examples
|
||||||
|
- Architecture diagram
|
||||||
|
- Error handling details
|
||||||
|
|
||||||
|
### 2. Modified Files
|
||||||
|
|
||||||
|
#### `.woodpecker.yml`
|
||||||
|
**Changes:**
|
||||||
|
- Updated header comments to reflect new JSON status flow
|
||||||
|
- Renamed step: `update-sync-metric` → `update-gitops-status`
|
||||||
|
- Removed Pushgateway environment variable (`PUSHGATEWAY_URL`)
|
||||||
|
- Added new environment variables:
|
||||||
|
- `GITOPS_STATUS_SERVER_URL`
|
||||||
|
- `REPO_NAME`
|
||||||
|
- `SERVER_NAME`
|
||||||
|
- Added `jq` package installation for JSON formatting
|
||||||
|
- Added `bash` package to cron step (required by update-gitops-status.sh)
|
||||||
|
- Updated both `update-gitops-status` and `gitops_sync_check` steps to call new script
|
||||||
|
- Removed Pushgateway metric push logic
|
||||||
|
- Added JSON status update logic
|
||||||
|
|
||||||
|
**Step: `update-gitops-status` (formerly `update-sync-metric`)**
|
||||||
|
- Runs after successful deployment
|
||||||
|
- Calls `update-gitops-status.sh`
|
||||||
|
- Always succeeds (status sent regardless of drift)
|
||||||
|
|
||||||
|
**Step: `gitops_sync_check`**
|
||||||
|
- Runs on cron schedule (every 2 minutes)
|
||||||
|
- Calls `update-gitops-status.sh` to send JSON
|
||||||
|
- Then checks drift status to determine pipeline success/failure
|
||||||
|
- Pipeline fails if drift detected (for visibility in Woodpecker UI)
|
||||||
|
- JSON status always sent before checking drift
|
||||||
|
|
||||||
|
#### `README.md`
|
||||||
|
**Changes:**
|
||||||
|
- Updated pipeline flow diagrams
|
||||||
|
- Replaced "Prometheus" with "gitops-status-server" in diagrams
|
||||||
|
- Removed "sync metric" section
|
||||||
|
- Added "GitOps status JSON format" section with examples
|
||||||
|
- Updated pipeline step descriptions to mention JSON status
|
||||||
|
- Added `update-gitops-status.sh` to repository structure
|
||||||
|
- Added optional environment variables table
|
||||||
|
- Updated flow descriptions to explain file-level drift details
|
||||||
|
|
||||||
|
### 3. Unchanged Files
|
||||||
|
|
||||||
|
The following files remain unchanged and continue to work as before:
|
||||||
|
- `ansible/playbooks/drift-check.yml` - Still works as-is
|
||||||
|
- `ansible/playbooks/apply.yml` - Deploy logic unchanged
|
||||||
|
- `ansible/playbooks/validate.yml` - Validation logic unchanged
|
||||||
|
- `ansible/inventory/hosts.yml` - Inventory unchanged
|
||||||
|
- `ansible/inventory/group_vars/all.yml` - Variables unchanged
|
||||||
|
- `ansible.cfg` - Ansible config unchanged
|
||||||
|
- `apply.sh` - Local apply script unchanged
|
||||||
|
- `drift-check.sh` - Local drift check script unchanged
|
||||||
|
- `files/rsyslog.conf` - Config files unchanged
|
||||||
|
- `files/rsyslog.d/30-lab.conf` - Config files unchanged
|
||||||
|
|
||||||
|
## Behavior Changes
|
||||||
|
|
||||||
|
### Before (Pushgateway)
|
||||||
|
|
||||||
|
```
|
||||||
|
drift-check → calculate status (0 or 1) → push to Pushgateway → Prometheus scrapes
|
||||||
|
```
|
||||||
|
|
||||||
|
Output:
|
||||||
|
```
|
||||||
|
gitops_sync_status{repo="rsyslog",server="rsyslog-lab"} 1
|
||||||
|
```
|
||||||
|
|
||||||
|
### After (gitops-status-server)
|
||||||
|
|
||||||
|
```
|
||||||
|
drift-check → extract changed files → build JSON → POST to gitops-status-server → Grafana queries
|
||||||
|
```
|
||||||
|
|
||||||
|
Output:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"repo": "rsyslog",
|
||||||
|
"server": "rsyslog-lab",
|
||||||
|
"sync_status": "OUT_OF_SYNC",
|
||||||
|
"drift_count": 2,
|
||||||
|
"files": [
|
||||||
|
{ "name": "rsyslog.conf" },
|
||||||
|
{ "name": "rsyslog.d/30-lab.conf" }
|
||||||
|
],
|
||||||
|
"last_check": "2026-04-21T10:32:15Z"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Benefits
|
||||||
|
|
||||||
|
1. **Richer data**: File-level drift information instead of just a binary status
|
||||||
|
2. **Better visualization**: Grafana can display which specific files have drifted
|
||||||
|
3. **Detailed tracking**: Know exactly what changed, not just that something changed
|
||||||
|
4. **Timestamp tracking**: Last check time included in JSON
|
||||||
|
5. **Drift count**: Quick numeric indicator of severity
|
||||||
|
6. **Extensible**: JSON format can be easily extended with additional fields
|
||||||
|
|
||||||
|
## Migration Checklist
|
||||||
|
|
||||||
|
- [x] Create `update-gitops-status.sh` script
|
||||||
|
- [x] Update `.woodpecker.yml` pipeline
|
||||||
|
- [x] Update `README.md` documentation
|
||||||
|
- [x] Create `GITOPS_STATUS_INTEGRATION.md` integration docs
|
||||||
|
- [x] Remove Pushgateway environment variables
|
||||||
|
- [x] Add gitops-status-server environment variables
|
||||||
|
- [x] Update pipeline step names
|
||||||
|
- [x] Add required packages (jq, bash)
|
||||||
|
- [x] Test JSON generation logic
|
||||||
|
- [x] Update flow diagrams
|
||||||
|
|
||||||
|
## Testing Recommendations
|
||||||
|
|
||||||
|
1. **Test the script locally:**
|
||||||
|
```bash
|
||||||
|
export GITOPS_STATUS_SERVER_URL="http://localhost:80"
|
||||||
|
export REPO_NAME="rsyslog"
|
||||||
|
export SERVER_NAME="rsyslog-lab"
|
||||||
|
./update-gitops-status.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **Test in Woodpecker:**
|
||||||
|
- Trigger a push to master → check `update-gitops-status` step
|
||||||
|
- Wait for cron run → check `gitops_sync_check` step
|
||||||
|
- Manually edit a file on server → wait for next cron → verify OUT_OF_SYNC status
|
||||||
|
|
||||||
|
3. **Verify gitops-status-server:**
|
||||||
|
- Check that JSON is received at POST endpoint
|
||||||
|
- Verify `/status.json` serves the latest data
|
||||||
|
- Confirm Grafana dashboard displays drift details
|
||||||
|
|
||||||
|
## Rollback Plan
|
||||||
|
|
||||||
|
If needed, the old Pushgateway approach can be restored by:
|
||||||
|
1. Reverting `.woodpecker.yml` to previous version
|
||||||
|
2. Removing `update-gitops-status.sh`
|
||||||
|
3. Restoring Pushgateway environment variables
|
||||||
|
|
||||||
|
All Ansible playbooks remain unchanged, so they will work with either approach.
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- The rsyslog repo now focuses only on status generation and sending
|
||||||
|
- gitops-status-server is responsible for serving data to Grafana
|
||||||
|
- No changes to observability-stack app are needed on the rsyslog side
|
||||||
|
- This migration is specific to rsyslog repo; other repos can follow same pattern
|
||||||
85
README.md
85
README.md
@ -47,14 +47,16 @@ Merge to master
|
|||||||
├─► deploy Copy the new config files from Git to the server
|
├─► deploy Copy the new config files from Git to the server
|
||||||
│ and restart rsyslog
|
│ and restart rsyslog
|
||||||
│
|
│
|
||||||
└─► update-sync-metric Run a diff between Git and the live server
|
└─► update-gitops-status Run a diff between Git and the live server
|
||||||
│
|
│
|
||||||
├─ Matches? → push metric 1 (SYNCED)
|
├─ Matches? → send JSON (SYNCED, drift_count: 0)
|
||||||
└─ Differs? → push metric 0 (OUT_OF_SYNC)
|
└─ Differs? → send JSON (OUT_OF_SYNC, drift_count: N, files: [...])
|
||||||
|
│
|
||||||
|
└─ Update gitops-status-server for Grafana visualization
|
||||||
```
|
```
|
||||||
|
|
||||||
**Pass** = new config is live and the server matches Git.
|
**Pass** = new config is live and the server matches Git.
|
||||||
The sync result is always sent to Prometheus regardless of outcome.
|
The sync status JSON is always sent to gitops-status-server regardless of outcome.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@ -70,19 +72,19 @@ Every 2 minutes (cron)
|
|||||||
└─► gitops_sync_check SSH to the server, compare every managed config
|
└─► gitops_sync_check SSH to the server, compare every managed config
|
||||||
file against the latest Git commit
|
file against the latest Git commit
|
||||||
│
|
│
|
||||||
├─ Matches? → push metric 1 (SYNCED)
|
├─ Matches? → send JSON (SYNCED, drift_count: 0, files: [])
|
||||||
└─ Differs? → push metric 0 (OUT_OF_SYNC)
|
└─ Differs? → send JSON (OUT_OF_SYNC, drift_count: N, files: [...])
|
||||||
|
│
|
||||||
|
└─ Update gitops-status-server for Grafana visualization
|
||||||
```
|
```
|
||||||
|
|
||||||
**Why this matters:** if someone edits `/etc/rsyslog.conf` directly on the server
|
**Why this matters:** if someone edits `/etc/rsyslog.conf` directly on the server
|
||||||
(bypassing Git), the next cron run catches it within 2 minutes and marks OUT_OF_SYNC.
|
(bypassing Git), the next cron run catches it within 2 minutes and marks OUT_OF_SYNC
|
||||||
|
with detailed information about which specific files have drifted.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Full flow diagram
|
## Full flow diagramgitops-status-server
|
||||||
|
|
||||||
```
|
|
||||||
Developer Woodpecker CI Linux Server Prometheus
|
|
||||||
│ │ │ │
|
│ │ │ │
|
||||||
│── open PR ───────────────►│ │ │
|
│── open PR ───────────────►│ │ │
|
||||||
│ │── syntax-check │ │
|
│ │── syntax-check │ │
|
||||||
@ -96,40 +98,69 @@ Developer Woodpecker CI Linux Server Prom
|
|||||||
│ │ │ restart rsyslog │
|
│ │ │ restart rsyslog │
|
||||||
│ │── drift-check ──────────►│ compare files │
|
│ │── drift-check ──────────►│ compare files │
|
||||||
│ │ │◄────────────────────│
|
│ │ │◄────────────────────│
|
||||||
│ │── metric (1 or 0) ───────────────────────────►│
|
│ │── JSON status ───────────────────────────────►│
|
||||||
│ │ │ │
|
│ │ │ │
|
||||||
│ │ [every 2 min, no push] │ │
|
│ │ [every 2 min, no push] │ │
|
||||||
│ │── drift-check ──────────►│ compare files │
|
│ │── drift-check ──────────►│ compare files │
|
||||||
│ │── metric (1 or 0) ───────────────────────────►│
|
│ │── JSON status ───────────────────────────────►│
|
||||||
│ │ │ │
|
│ │ │ │
|
||||||
|
|
||||||
|
|
||||||
Someone edits the server directly (bad):
|
Someone edits the server directly (bad):
|
||||||
|
|
||||||
rogue admin Woodpecker CI Linux Server Prometheus
|
rogue admin Woodpecker CI Linux Server gitops-status-server
|
||||||
│ │ │ │
|
│ │ │ │
|
||||||
│── ssh rsyslog-lab │ │ │
|
│── ssh rsyslog-lab │ │ │
|
||||||
│── vim /etc/rsyslog.conf ──────────────────────────► │ file changed │
|
│── vim /etc/rsyslog.conf ──────────────────────────► │ file changed │
|
||||||
│ │ │ │
|
│ │ │ │
|
||||||
│ [2 min later, cron runs] │ │
|
│ [2 min later, cron runs] │ │
|
||||||
│ │── drift-check ──────────►│ diff detected │
|
│ │── drift-check ──────────►│ diff detected │
|
||||||
|
│ │── JSON status (OUT_OF_SYNC)─────────────────►│
|
||||||
|
│ │ drift_count: 1 │ Grafana shows
|
||||||
|
│ │ files: [rsyslog.conf] OUT_OF_SYNC
|
||||||
|
│ │── drift-check ──────────►│ diff detected │
|
||||||
│ │── metric 0 (OUT_OF_SYNC)────────────────────►│
|
│ │── metric 0 (OUT_OF_SYNC)────────────────────►│
|
||||||
│ │ │ alert fires
|
│ │ │ alert fires
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
GitOps status JSON format
|
||||||
|
|
||||||
## What is the sync metric?
|
Instead of simple numeric metrics, this repo now sends rich JSON status data to gitops-status-server:
|
||||||
|
|
||||||
```
|
```json
|
||||||
gitops_sync_status{repo="rsyslog", server="rsyslog-lab"}
|
{
|
||||||
|
"repo": "rsyslog",
|
||||||
|
"server": "rsyslog-lab",
|
||||||
|
"sync_status": "SYNCED",
|
||||||
|
"drift_count": 0,
|
||||||
|
"files": [],
|
||||||
|
"last_check": "2026-04-21T10:30:00Z"
|
||||||
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
| Value | Meaning |
|
When drift is detected:
|
||||||
|-------|---------|
|
|
||||||
| `1` | Server config matches Git (SYNCED) |
|
|
||||||
| `0` | Server config differs from Git (OUT_OF_SYNC) |
|
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"repo": "rsyslog",
|
||||||
|
"server": "rsyslog-lab",
|
||||||
|
"sync_status": "OUT_OF_SYNC",
|
||||||
|
"drift_count": 2,
|
||||||
|
"files": [
|
||||||
|
{ "name": "rsyslog.conf" },
|
||||||
|
{ "name": "rsyslog.d/30-lab.conf" }
|
||||||
|
],
|
||||||
|
"last_check": "2026-04-21T10:32:15Z"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
update-gitops-status.sh Script to generate and send JSON status to gitops-status-server
|
||||||
|
This JSON is sent to `gitops-status-server` at:
|
||||||
|
- `http://gitops-status-server.observability-stack.svc.cluster.local:80/api/status`
|
||||||
|
|
||||||
|
The gitops-status-server app serves this data via `/status.json` for Grafana Infinity datasource,
|
||||||
|
providing rich visualization with file-level drift details instead of just a numeric metric
|
||||||
Alert on `gitops_sync_status == 0` in Grafana/Alertmanager.
|
Alert on `gitops_sync_status == 0` in Grafana/Alertmanager.
|
||||||
|
|
||||||
---
|
---
|
||||||
@ -183,5 +214,15 @@ Go to **Repository Settings → Crons → Add cron**:
|
|||||||
Go to **Repository Settings → Secrets**:
|
Go to **Repository Settings → Secrets**:
|
||||||
|
|
||||||
| Name | Description |
|
| Name | Description |
|
||||||
|-------------------|------------------------------------|
|
|----------------------------|-------------------------------------------------------|
|
||||||
| `SSH_PRIVATE_KEY` | Private key to SSH into the server |
|
| `SSH_PRIVATE_KEY` | Private key to SSH into the server |
|
||||||
|
|
||||||
|
## Optional environment variables
|
||||||
|
|
||||||
|
These can be overridden in the Woodpecker pipeline or `.woodpecker.yml`:
|
||||||
|
|
||||||
|
| Variable | Default | Description |
|
||||||
|
|------------------------------|--------------------------------------------------------------------------|---------------------------------------|
|
||||||
|
| `GITOPS_STATUS_SERVER_URL` | `http://gitops-status-server.observability-stack.svc.cluster.local:80` | URL of gitops-status-server API |
|
||||||
|
| `REPO_NAME` | `rsyslog` | Repository name for JSON status |
|
||||||
|
| `SERVER_NAME` | `rsyslog-lab` | Server name for JSON status |
|
||||||
|
|||||||
162
update-gitops-status.sh
Normal file
162
update-gitops-status.sh
Normal file
@ -0,0 +1,162 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# =============================================================================
|
||||||
|
# update-gitops-status.sh
|
||||||
|
#
|
||||||
|
# Purpose:
|
||||||
|
# Runs drift-check playbook and generates a JSON status snapshot for
|
||||||
|
# gitops-status-server. This replaces Pushgateway metric updates with
|
||||||
|
# richer JSON status suitable for Grafana visualization.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# ./update-gitops-status.sh
|
||||||
|
#
|
||||||
|
# Environment Variables:
|
||||||
|
# GITOPS_STATUS_SERVER_URL - URL of gitops-status-server API
|
||||||
|
# (default: http://gitops-status-server.observability-stack.svc.cluster.local:80)
|
||||||
|
# REPO_NAME - Repository name (default: rsyslog)
|
||||||
|
# SERVER_NAME - Server name (default: rsyslog-lab)
|
||||||
|
#
|
||||||
|
# Output:
|
||||||
|
# Generates JSON structure:
|
||||||
|
# {
|
||||||
|
# "repo": "rsyslog",
|
||||||
|
# "server": "rsyslog-lab",
|
||||||
|
# "sync_status": "SYNCED" or "OUT_OF_SYNC",
|
||||||
|
# "drift_count": <number>,
|
||||||
|
# "files": [{"name": "rsyslog.conf"}, ...],
|
||||||
|
# "last_check": "2026-04-21T10:30:00Z"
|
||||||
|
# }
|
||||||
|
#
|
||||||
|
# Exit codes:
|
||||||
|
# 0 - Success (regardless of sync status)
|
||||||
|
# 1 - Failure (playbook error, network error, etc.)
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Configuration
|
||||||
|
GITOPS_STATUS_SERVER_URL="${GITOPS_STATUS_SERVER_URL:-http://gitops-status-server.observability-stack.svc.cluster.local:80}"
|
||||||
|
REPO_NAME="${REPO_NAME:-rsyslog}"
|
||||||
|
SERVER_NAME="${SERVER_NAME:-rsyslog-lab}"
|
||||||
|
INVENTORY_FILE="ansible/inventory/hosts.yml"
|
||||||
|
PLAYBOOK="ansible/playbooks/drift-check.yml"
|
||||||
|
|
||||||
|
echo "==> Running drift check playbook..."
|
||||||
|
echo " Inventory: $INVENTORY_FILE"
|
||||||
|
echo " Playbook: $PLAYBOOK"
|
||||||
|
|
||||||
|
# Run drift-check playbook in JSON output mode and capture results
|
||||||
|
# We use --diff to get detailed change information
|
||||||
|
# Exit code: 0 = synced, non-zero = drift or error
|
||||||
|
set +e
|
||||||
|
PLAYBOOK_OUTPUT=$(ANSIBLE_STDOUT_CALLBACK=json ansible-playbook \
|
||||||
|
-i "$INVENTORY_FILE" \
|
||||||
|
"$PLAYBOOK" \
|
||||||
|
2>&1)
|
||||||
|
DRIFT_RC=$?
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Determine sync status
|
||||||
|
if [ "$DRIFT_RC" -eq 0 ]; then
|
||||||
|
SYNC_STATUS="SYNCED"
|
||||||
|
echo "==> Status: SYNCED - server configuration matches Git"
|
||||||
|
else
|
||||||
|
SYNC_STATUS="OUT_OF_SYNC"
|
||||||
|
echo "==> Status: OUT_OF_SYNC - drift detected"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Parse changed files from playbook output
|
||||||
|
# Look for tasks that reported changed=true
|
||||||
|
CHANGED_FILES=()
|
||||||
|
DRIFT_COUNT=0
|
||||||
|
|
||||||
|
# Extract file changes from JSON output
|
||||||
|
# main_config_check tracks rsyslog.conf
|
||||||
|
# rsyslogd_check tracks rsyslog.d/* files
|
||||||
|
if echo "$PLAYBOOK_OUTPUT" | grep -q '"main_config_check".*"changed".*true'; then
|
||||||
|
CHANGED_FILES+=("rsyslog.conf")
|
||||||
|
((DRIFT_COUNT++))
|
||||||
|
echo " - Drift detected: rsyslog.conf"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if rsyslog.d directory has changes
|
||||||
|
if echo "$PLAYBOOK_OUTPUT" | grep -q '"rsyslogd_check".*"changed".*true'; then
|
||||||
|
# Try to extract specific filenames from diff output
|
||||||
|
# Format: files/rsyslog.d/30-lab.conf
|
||||||
|
while IFS= read -r line; do
|
||||||
|
if [[ "$line" =~ files/rsyslog\.d/([^[:space:]]+\.conf) ]]; then
|
||||||
|
filename="${BASH_REMATCH[1]}"
|
||||||
|
CHANGED_FILES+=("rsyslog.d/$filename")
|
||||||
|
((DRIFT_COUNT++))
|
||||||
|
echo " - Drift detected: rsyslog.d/$filename"
|
||||||
|
fi
|
||||||
|
done < <(echo "$PLAYBOOK_OUTPUT" | grep -oP 'files/rsyslog\.d/[^[:space:]]+\.conf' || true)
|
||||||
|
|
||||||
|
# If we couldn't extract specific files but know rsyslog.d changed,
|
||||||
|
# add a generic entry
|
||||||
|
if [ ${#CHANGED_FILES[@]} -eq 0 ] || [ ${#CHANGED_FILES[@]} -eq 1 ]; then
|
||||||
|
CHANGED_FILES+=("rsyslog.d/*")
|
||||||
|
((DRIFT_COUNT++))
|
||||||
|
echo " - Drift detected: rsyslog.d/* (multiple files)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check for missing files on server
|
||||||
|
if echo "$PLAYBOOK_OUTPUT" | grep -q '"extra_files_on_server".*true'; then
|
||||||
|
CHANGED_FILES+=("(missing files on server)")
|
||||||
|
((DRIFT_COUNT++))
|
||||||
|
echo " - Drift detected: files missing on server"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> Drift count: $DRIFT_COUNT"
|
||||||
|
|
||||||
|
# Build JSON file list
|
||||||
|
FILES_JSON="[]"
|
||||||
|
if [ ${#CHANGED_FILES[@]} -gt 0 ]; then
|
||||||
|
FILES_JSON="["
|
||||||
|
for i in "${!CHANGED_FILES[@]}"; do
|
||||||
|
if [ $i -gt 0 ]; then
|
||||||
|
FILES_JSON+=","
|
||||||
|
fi
|
||||||
|
FILES_JSON+="{\"name\":\"${CHANGED_FILES[$i]}\"}"
|
||||||
|
done
|
||||||
|
FILES_JSON+="]"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Generate ISO timestamp
|
||||||
|
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
|
||||||
|
# Build complete JSON status
|
||||||
|
STATUS_JSON=$(cat <<EOF
|
||||||
|
{
|
||||||
|
"repo": "$REPO_NAME",
|
||||||
|
"server": "$SERVER_NAME",
|
||||||
|
"sync_status": "$SYNC_STATUS",
|
||||||
|
"drift_count": $DRIFT_COUNT,
|
||||||
|
"files": $FILES_JSON,
|
||||||
|
"last_check": "$TIMESTAMP"
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
)
|
||||||
|
|
||||||
|
echo "==> Generated JSON status:"
|
||||||
|
echo "$STATUS_JSON" | jq '.' 2>/dev/null || echo "$STATUS_JSON"
|
||||||
|
|
||||||
|
# Send JSON to gitops-status-server
|
||||||
|
# API endpoint: POST /api/status
|
||||||
|
echo "==> Sending status to gitops-status-server..."
|
||||||
|
echo " URL: $GITOPS_STATUS_SERVER_URL/api/status"
|
||||||
|
|
||||||
|
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||||
|
-X POST \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d "$STATUS_JSON" \
|
||||||
|
"$GITOPS_STATUS_SERVER_URL/api/status")
|
||||||
|
|
||||||
|
if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then
|
||||||
|
echo "==> Status update successful (HTTP $HTTP_CODE)"
|
||||||
|
exit 0
|
||||||
|
else
|
||||||
|
echo "==> ERROR: Status update failed (HTTP $HTTP_CODE)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
Loading…
x
Reference in New Issue
Block a user