Add automation to scrape targets for prometheus

.woodpecker.yml

@@ -6,7 +6,7 @@ steps:
     image: alpine
     commands:
       - apk add --no-cache git bash curl yq
-      - bash cloudflared-sync.sh
+      - bash automation/cloudflared-sync.sh
     environment:
       GIT_TOKEN:
         from_secret: GIT_TOKEN
@@ -15,3 +15,15 @@ steps:
       CLOUDFLARE_ZONE_ID:
         from_secret: CLOUDFLARE_ZONE_ID
 
+  sync-prometheus-scrapes:
+    when:
+      branch: [master]
+    name: Update Prometheus Additional Scrapes
+    image: python:3.11-alpine
+    commands:
+      - apk add --no-cache git bash
+      - pip install pyyaml
+      - bash automation/scrape-sync.sh
+    environment:
+      GIT_TOKEN:
+        from_secret: GIT_TOKEN

generate-scrape-config.py

@@ -0,0 +1,52 @@
+import os
+import yaml
+from pathlib import Path
+
+REPOS = {
+    "dev-tools": "https://git.dvirlabs.com/dvirlabs/dev-tools.git",
+    "infra": "https://git.dvirlabs.com/dvirlabs/infra.git",
+    "observability-stack": "https://git.dvirlabs.com/dvirlabs/observability-stack.git"
+}
+
+TMP_DIR = ".tmp-repos"
+OUTPUT_FILE = "observability-stack/manifests/prometheus-scrape-secret/additional-scrape-configs.yaml"
+
+os.makedirs(TMP_DIR, exist_ok=True)
+
+def collect_targets():
+    jobs = {}
+
+    for name, url in REPOS.items():
+        repo_path = os.path.join(TMP_DIR, name)
+        if not os.path.exists(repo_path):
+            os.system(f"git clone --depth 1 {url} {repo_path}")
+
+        for path in Path(repo_path, "manifests").glob("*/monitoring.yaml"):
+            with open(path) as f:
+                data = yaml.safe_load(f)
+
+            if data.get("enabled") and "targets" in data:
+                if name not in jobs:
+                    jobs[name] = []
+                jobs[name].extend(data["targets"])
+
+    return jobs
+
+def write_scrape_config(jobs):
+    result = []
+    for repo, targets in jobs.items():
+        result.append({
+            "job_name": repo,
+            "static_configs": [
+                {"targets": targets}
+            ]
+        })
+
+    os.makedirs(os.path.dirname(OUTPUT_FILE), exist_ok=True)
+    with open(OUTPUT_FILE, "w") as f:
+        yaml.dump(result, f)
+
+if __name__ == "__main__":
+    jobs = collect_targets()
+    write_scrape_config(jobs)
+    print(f"✅ Generated: {OUTPUT_FILE}")

scraoe-sync.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+
+echo "🔄 Cloning repositories..."
+REPOS=("dev-tools" "infra" "observability-stack")
+rm -rf .tmp-repos
+mkdir -p .tmp-repos
+
+for REPO in "${REPOS[@]}"; do
+  git clone https://${GIT_TOKEN}@git.dvirlabs.com/dvirlabs/${REPO}.git .tmp-repos/${REPO}
+done
+
+echo "⚙️ Generating additional-scrape-configs.yaml..."
+python3 automation/generate-scrape-config.py
+
+echo "📦 Copying into observability-stack repo..."
+cp observability-stack/manifests/prometheus-scrape-secret/additional-scrape-configs.yaml .tmp-repos/observability-stack/manifests/prometheus-scrape-secret/
+
+cd .tmp-repos/observability-stack
+git config user.name "auto-sync"
+git config user.email "sync@dvirlabs.com"
+
+if git diff --quiet; then
+  echo "✅ No changes to commit."
+else
+  git add manifests/prometheus-scrape-secret/additional-scrape-configs.yaml
+  git commit -m "auto: update Prometheus scrape config"
+  git push origin master
+fi