diff --git a/.woodpecker.yml b/.woodpecker.yml index e5ca18b..097d058 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -6,7 +6,7 @@ steps: image: alpine commands: - apk add --no-cache git bash curl yq - - bash cloudflared-sync.sh + - bash automation/cloudflared-sync.sh environment: GIT_TOKEN: from_secret: GIT_TOKEN @@ -14,4 +14,16 @@ steps: from_secret: CLOUDFLARE_API_TOKEN CLOUDFLARE_ZONE_ID: from_secret: CLOUDFLARE_ZONE_ID - \ No newline at end of file + + sync-prometheus-scrapes: + when: + branch: [master] + name: Update Prometheus Additional Scrapes + image: python:3.11-alpine + commands: + - apk add --no-cache git bash + - pip install pyyaml + - bash automation/scrape-sync.sh + environment: + GIT_TOKEN: + from_secret: GIT_TOKEN diff --git a/cloudflared-sync.sh b/automation/cloudflared-sync.sh similarity index 100% rename from cloudflared-sync.sh rename to automation/cloudflared-sync.sh diff --git a/generate-scrape-config.py b/generate-scrape-config.py new file mode 100644 index 0000000..905099c --- /dev/null +++ b/generate-scrape-config.py @@ -0,0 +1,52 @@ +import os +import yaml +from pathlib import Path + +REPOS = { + "dev-tools": "https://git.dvirlabs.com/dvirlabs/dev-tools.git", + "infra": "https://git.dvirlabs.com/dvirlabs/infra.git", + "observability-stack": "https://git.dvirlabs.com/dvirlabs/observability-stack.git" +} + +TMP_DIR = ".tmp-repos" +OUTPUT_FILE = "observability-stack/manifests/prometheus-scrape-secret/additional-scrape-configs.yaml" + +os.makedirs(TMP_DIR, exist_ok=True) + +def collect_targets(): + jobs = {} + + for name, url in REPOS.items(): + repo_path = os.path.join(TMP_DIR, name) + if not os.path.exists(repo_path): + os.system(f"git clone --depth 1 {url} {repo_path}") + + for path in Path(repo_path, "manifests").glob("*/monitoring.yaml"): + with open(path) as f: + data = yaml.safe_load(f) + + if data.get("enabled") and "targets" in data: + if name not in jobs: + jobs[name] = [] + jobs[name].extend(data["targets"]) + + return jobs + +def write_scrape_config(jobs): + result = [] + for repo, targets in jobs.items(): + result.append({ + "job_name": repo, + "static_configs": [ + {"targets": targets} + ] + }) + + os.makedirs(os.path.dirname(OUTPUT_FILE), exist_ok=True) + with open(OUTPUT_FILE, "w") as f: + yaml.dump(result, f) + +if __name__ == "__main__": + jobs = collect_targets() + write_scrape_config(jobs) + print(f"✅ Generated: {OUTPUT_FILE}") diff --git a/scraoe-sync.sh b/scraoe-sync.sh new file mode 100644 index 0000000..d3a99aa --- /dev/null +++ b/scraoe-sync.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e + +echo "🔄 Cloning repositories..." +REPOS=("dev-tools" "infra" "observability-stack") +rm -rf .tmp-repos +mkdir -p .tmp-repos + +for REPO in "${REPOS[@]}"; do + git clone https://${GIT_TOKEN}@git.dvirlabs.com/dvirlabs/${REPO}.git .tmp-repos/${REPO} +done + +echo "⚙️ Generating additional-scrape-configs.yaml..." +python3 automation/generate-scrape-config.py + +echo "📦 Copying into observability-stack repo..." +cp observability-stack/manifests/prometheus-scrape-secret/additional-scrape-configs.yaml .tmp-repos/observability-stack/manifests/prometheus-scrape-secret/ + +cd .tmp-repos/observability-stack +git config user.name "auto-sync" +git config user.email "sync@dvirlabs.com" + +if git diff --quiet; then + echo "✅ No changes to commit." +else + git add manifests/prometheus-scrape-secret/additional-scrape-configs.yaml + git commit -m "auto: update Prometheus scrape config" + git push origin master +fi