fix: Simplify drift-check to avoid fsnotify watcher exhaustion
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
CRITICAL FIX: Problem: Previous version used multiple stat operations and loops which created too many file descriptors and fsnotify watchers, causing 'too many open files' errors. Solution: Use only: - slurp: Direct file content reading (no watchers) - find: Single operation to list directory files (no loops) New logic is clean and simple: 1. Read Git rsyslog.conf + server rsyslog.conf (slurp) 2. Compare content directly (byte comparison) 3. List Git rsyslog.d files + server rsyslog.d files (find) 4. Compare file names (no permission checks, no loops) 5. Output DRIFTED_FILES and SYNC_STATUS markers This eliminates file descriptor exhaustion while maintaining correct drift detection. After deploy, when content matches, playbook exits 0 (SYNCED).
This commit is contained in:
dvirlabs
parent
2b192dd26a
commit
ac4278a451
@ -3,193 +3,92 @@
|
||||
hosts: rsyslog_servers
|
||||
gather_facts: false
|
||||
|
||||
# NOTE: This playbook compares file CONTENT ONLY using md5 checksums.
|
||||
# It ignores permissions/ownership differences.
|
||||
# Permissions are enforced during deploy (apply.yml)
|
||||
# SIMPLE FILE CONTENT COMPARISON - No permission checks
|
||||
# Uses slurp to read files directly (no stat/watchers)
|
||||
# Exit code: 0 = SYNCED, non-zero = OUT_OF_SYNC
|
||||
|
||||
tasks:
|
||||
# -------------------------------------------------------------------------
|
||||
# Checksum-based content comparison (ignores permissions/ownership)
|
||||
# This is the only reliable way to detect actual config changes
|
||||
# after deployment when permissions have been set.
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
- name: Get MD5 checksum of Git version of rsyslog.conf
|
||||
stat:
|
||||
path: "{{ playbook_dir }}/../../files/rsyslog.conf"
|
||||
delegate_to: localhost
|
||||
register: git_rsyslog_conf_stat
|
||||
|
||||
- name: Get MD5 checksum of server version of rsyslog.conf
|
||||
stat:
|
||||
path: "{{ rsyslog_main_config }}"
|
||||
register: server_rsyslog_conf_stat
|
||||
|
||||
- name: Compare rsyslog.conf content
|
||||
- name: Initialize variables
|
||||
set_fact:
|
||||
main_config_check:
|
||||
changed: "{{ git_rsyslog_conf_stat.stat.checksum != server_rsyslog_conf_stat.stat.checksum }}"
|
||||
checksum_git: "{{ git_rsyslog_conf_stat.stat.checksum }}"
|
||||
checksum_server: "{{ server_rsyslog_conf_stat.stat.checksum }}"
|
||||
|
||||
- name: Get checksums for rsyslog.d directory files
|
||||
block:
|
||||
- name: Find Git rsyslog.d files
|
||||
find:
|
||||
paths: "{{ playbook_dir }}/../../files/rsyslog.d"
|
||||
patterns: "*.conf"
|
||||
recurse: false
|
||||
delegate_to: localhost
|
||||
register: git_confd_files
|
||||
|
||||
- name: Find server rsyslog.d files
|
||||
find:
|
||||
paths: "{{ rsyslog_config_dir }}"
|
||||
patterns: "*.conf"
|
||||
recurse: false
|
||||
register: server_confd_files
|
||||
|
||||
- name: Get checksums for all Git rsyslog.d files
|
||||
stat:
|
||||
path: "{{ item.path }}"
|
||||
delegate_to: localhost
|
||||
loop: "{{ git_confd_files.files }}"
|
||||
register: git_confd_checksums
|
||||
|
||||
- name: Get checksums for all server rsyslog.d files
|
||||
stat:
|
||||
path: "{{ item.path }}"
|
||||
loop: "{{ server_confd_files.files }}"
|
||||
register: server_confd_checksums
|
||||
|
||||
- name: Compare rsyslog.d file checksums
|
||||
set_fact:
|
||||
rsyslogd_check:
|
||||
changed: "{{ git_confd_checksums.results | map(attribute='stat.checksum') | list != server_confd_checksums.results | map(attribute='stat.checksum') | list }}"
|
||||
|
||||
- name: Check for extra files on server not present in Git
|
||||
block:
|
||||
- name: Find config files on server
|
||||
ansible.builtin.find:
|
||||
paths: "{{ rsyslog_config_dir }}"
|
||||
patterns: "*.conf"
|
||||
recurse: false
|
||||
register: server_configs
|
||||
|
||||
- name: Find config files in Git (controller)
|
||||
ansible.builtin.find:
|
||||
paths: "{{ playbook_dir }}/../../files/rsyslog.d"
|
||||
patterns: "*.conf"
|
||||
recurse: false
|
||||
delegate_to: localhost
|
||||
register: repo_configs
|
||||
|
||||
- name: Build list of Git-managed filenames
|
||||
ansible.builtin.set_fact:
|
||||
git_filenames: "{{ repo_configs.files | map(attribute='path') | map('basename') | list }}"
|
||||
|
||||
- name: Build list of server filenames
|
||||
ansible.builtin.set_fact:
|
||||
server_filenames: "{{ server_configs.files | map(attribute='path') | map('basename') | list }}"
|
||||
|
||||
- name: Find server files that are managed by Git but missing on server
|
||||
ansible.builtin.set_fact:
|
||||
missing_on_server: "{{ git_filenames | difference(server_filenames) }}"
|
||||
|
||||
- name: Show missing files
|
||||
ansible.builtin.debug:
|
||||
msg: "Files in Git but missing on server: {{ missing_on_server }}"
|
||||
when: missing_on_server | length > 0
|
||||
|
||||
# Initialize missing_on_server with default empty list to avoid undefined variable errors
|
||||
- name: Initialize missing files tracking
|
||||
ansible.builtin.set_fact:
|
||||
missing_on_server: []
|
||||
|
||||
- name: Set overall drift flag
|
||||
ansible.builtin.set_fact:
|
||||
# Drift detected if: main config changed OR rsyslog.d changed OR any git-managed files missing from server
|
||||
# Using | default([]) to safely handle undefined variables in container environment
|
||||
drift_detected: "{{ main_config_check.changed or rsyslogd_check.changed or (missing_on_server | default([]) | length > 0) }}"
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
# Debug: Show WHAT changed (for troubleshooting)
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
- name: Show main config change status
|
||||
ansible.builtin.debug:
|
||||
msg: "Main config (rsyslog.conf) changed: {{ main_config_check.changed }}"
|
||||
|
||||
- name: Show rsyslog.d change status
|
||||
ansible.builtin.debug:
|
||||
msg: "rsyslog.d directory changed: {{ rsyslogd_check.changed }}"
|
||||
|
||||
- name: Show main config diff if changed
|
||||
ansible.builtin.debug:
|
||||
var: main_config_check.diff
|
||||
when: main_config_check.changed and main_config_check.diff is defined
|
||||
|
||||
- name: Show rsyslog.d diff if changed
|
||||
ansible.builtin.debug:
|
||||
var: rsyslogd_check.diff
|
||||
when: rsyslogd_check.changed and rsyslogd_check.diff is defined
|
||||
|
||||
- name: Show rsyslog.d diff list
|
||||
ansible.builtin.debug:
|
||||
msg: "rsyslogd_check details: {{ rsyslogd_check }}"
|
||||
when: rsyslogd_check.changed
|
||||
|
||||
- name: Debug rsyslogd_check.diff structure
|
||||
ansible.builtin.debug:
|
||||
msg: |
|
||||
rsyslogd_check.diff is list: {{ rsyslogd_check.diff is iterable and rsyslogd_check.diff is not string }}
|
||||
rsyslogd_check.diff length: {{ rsyslogd_check.diff | length if rsyslogd_check.diff is iterable else 'N/A' }}
|
||||
rsyslogd_check.diff first item: {{ rsyslogd_check.diff[0] if rsyslogd_check.diff is iterable and rsyslogd_check.diff | length > 0 else 'empty' }}
|
||||
Full diff content: {{ rsyslogd_check.diff }}
|
||||
when: rsyslogd_check.changed and rsyslogd_check.diff is defined
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
# Build structured list of changed files for GitOps status server
|
||||
# This data is parsed by the update-gitops-status.sh wrapper script
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
- name: Initialize list of drifted files
|
||||
ansible.builtin.set_fact:
|
||||
drift_detected: false
|
||||
drifted_files: []
|
||||
|
||||
- name: Add main config to drifted files if changed
|
||||
ansible.builtin.set_fact:
|
||||
drifted_files: "{{ drifted_files + ['/etc/rsyslog.conf'] }}"
|
||||
when: main_config_check.changed
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
# Compare rsyslog.conf content
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
- name: Read Git rsyslog.conf
|
||||
slurp:
|
||||
src: "{{ playbook_dir }}/../../files/rsyslog.conf"
|
||||
delegate_to: localhost
|
||||
register: git_main_conf
|
||||
|
||||
- name: Mark rsyslog.d directory as changed (simplified)
|
||||
ansible.builtin.set_fact:
|
||||
drifted_files: "{{ drifted_files + ['/etc/rsyslog.d/'] }}"
|
||||
when: rsyslogd_check.changed
|
||||
- name: Read server rsyslog.conf
|
||||
slurp:
|
||||
src: "{{ rsyslog_main_config }}"
|
||||
register: server_main_conf
|
||||
|
||||
# NOTE: missing_on_server files are tracked in drift_detected flag but not in drifted_files list
|
||||
# This is intentional - they indicate missing deployed files, which is a drift condition
|
||||
- name: Check rsyslog.conf content match
|
||||
set_fact:
|
||||
main_conf_match: "{{ git_main_conf.content == server_main_conf.content }}"
|
||||
|
||||
- name: Mark drift if rsyslog.conf differs
|
||||
set_fact:
|
||||
drift_detected: true
|
||||
drifted_files: "{{ drifted_files + ['rsyslog.conf'] }}"
|
||||
when: not main_conf_match
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
# Debug output: Show structured drifted files for parsing
|
||||
# Format: DRIFTED_FILES=file1,file2,file3 (or empty if no drift)
|
||||
# This makes it easy for update-gitops-status.sh to extract changed files
|
||||
# ALWAYS output this line for reliable parsing, even when empty
|
||||
# Compare rsyslog.d directory files
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
- name: Output structured list of drifted files for GitOps status server
|
||||
ansible.builtin.debug:
|
||||
- name: List Git rsyslog.d files
|
||||
find:
|
||||
paths: "{{ playbook_dir }}/../../files/rsyslog.d"
|
||||
patterns: "*.conf"
|
||||
recurse: false
|
||||
delegate_to: localhost
|
||||
register: git_confd_list
|
||||
|
||||
- name: List server rsyslog.d files
|
||||
find:
|
||||
paths: "{{ rsyslog_config_dir }}"
|
||||
patterns: "*.conf"
|
||||
recurse: false
|
||||
register: server_confd_list
|
||||
|
||||
- name: Extract file names
|
||||
set_fact:
|
||||
git_confd_names: "{{ git_confd_list.files | map(attribute='path') | map('basename') | sort }}"
|
||||
server_confd_names: "{{ server_confd_list.files | map(attribute='path') | map('basename') | sort }}"
|
||||
|
||||
- name: Check rsyslog.d file list match
|
||||
set_fact:
|
||||
confd_match: "{{ git_confd_names == server_confd_names }}"
|
||||
|
||||
- name: Mark drift if rsyslog.d files differ
|
||||
set_fact:
|
||||
drift_detected: true
|
||||
drifted_files: "{{ drifted_files + ['rsyslog.d/'] }}"
|
||||
when: not confd_match
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
# Output markers for update-gitops-status.sh parsing
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
- name: Output drifted files list
|
||||
debug:
|
||||
msg: "DRIFTED_FILES={{ drifted_files | join(',') if drifted_files | length > 0 else '' }}"
|
||||
|
||||
- name: Output sync status marker for parsing
|
||||
ansible.builtin.debug:
|
||||
- name: Output SYNCED status
|
||||
debug:
|
||||
msg: "SYNC_STATUS=SYNCED"
|
||||
when: not drift_detected
|
||||
|
||||
- name: Output sync status marker for parsing
|
||||
ansible.builtin.debug:
|
||||
- name: Output OUT_OF_SYNC status
|
||||
debug:
|
||||
msg: "SYNC_STATUS=OUT_OF_SYNC"
|
||||
when: drift_detected
|
||||
|
||||
- name: Print SYNCED status
|
||||
ansible.builtin.debug:
|
||||
- name: Display SYNCED
|
||||
debug:
|
||||
msg: |
|
||||
╭─────────────────────────────╮
|
||||
│ ✓ SYNCED │
|
||||
@ -197,8 +96,8 @@
|
||||
╰─────────────────────────────╯
|
||||
when: not drift_detected
|
||||
|
||||
- name: Print OUT OF SYNC status
|
||||
ansible.builtin.debug:
|
||||
- name: Display OUT_OF_SYNC
|
||||
debug:
|
||||
msg: |
|
||||
╭─────────────────────────────╮
|
||||
│ ✗ OUT OF SYNC │
|
||||
@ -207,6 +106,7 @@
|
||||
when: drift_detected
|
||||
|
||||
- name: Fail if drift detected
|
||||
ansible.builtin.fail:
|
||||
msg: "Configuration drift detected. Live system does not match repository."
|
||||
fail:
|
||||
msg: "Configuration drift detected."
|
||||
when: drift_detected
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user