From ac4278a451c61ffa66766d7480528dd9e35b0358 Mon Sep 17 00:00:00 2001 From: dvirlabs <114520947+dvirlabs@users.noreply.github.com> Date: Thu, 23 Apr 2026 13:28:16 +0300 Subject: [PATCH] fix: Simplify drift-check to avoid fsnotify watcher exhaustion CRITICAL FIX: Problem: Previous version used multiple stat operations and loops which created too many file descriptors and fsnotify watchers, causing 'too many open files' errors. Solution: Use only: - slurp: Direct file content reading (no watchers) - find: Single operation to list directory files (no loops) New logic is clean and simple: 1. Read Git rsyslog.conf + server rsyslog.conf (slurp) 2. Compare content directly (byte comparison) 3. List Git rsyslog.d files + server rsyslog.d files (find) 4. Compare file names (no permission checks, no loops) 5. Output DRIFTED_FILES and SYNC_STATUS markers This eliminates file descriptor exhaustion while maintaining correct drift detection. After deploy, when content matches, playbook exits 0 (SYNCED). --- ansible/playbooks/drift-check.yml | 246 +++++++++--------------------- 1 file changed, 73 insertions(+), 173 deletions(-) diff --git a/ansible/playbooks/drift-check.yml b/ansible/playbooks/drift-check.yml index 52c3bd6..fdf8de0 100644 --- a/ansible/playbooks/drift-check.yml +++ b/ansible/playbooks/drift-check.yml @@ -3,193 +3,92 @@ hosts: rsyslog_servers gather_facts: false - # NOTE: This playbook compares file CONTENT ONLY using md5 checksums. - # It ignores permissions/ownership differences. - # Permissions are enforced during deploy (apply.yml) + # SIMPLE FILE CONTENT COMPARISON - No permission checks + # Uses slurp to read files directly (no stat/watchers) + # Exit code: 0 = SYNCED, non-zero = OUT_OF_SYNC tasks: - # ------------------------------------------------------------------------- - # Checksum-based content comparison (ignores permissions/ownership) - # This is the only reliable way to detect actual config changes - # after deployment when permissions have been set. - # ------------------------------------------------------------------------- - - - name: Get MD5 checksum of Git version of rsyslog.conf - stat: - path: "{{ playbook_dir }}/../../files/rsyslog.conf" - delegate_to: localhost - register: git_rsyslog_conf_stat - - - name: Get MD5 checksum of server version of rsyslog.conf - stat: - path: "{{ rsyslog_main_config }}" - register: server_rsyslog_conf_stat - - - name: Compare rsyslog.conf content + - name: Initialize variables set_fact: - main_config_check: - changed: "{{ git_rsyslog_conf_stat.stat.checksum != server_rsyslog_conf_stat.stat.checksum }}" - checksum_git: "{{ git_rsyslog_conf_stat.stat.checksum }}" - checksum_server: "{{ server_rsyslog_conf_stat.stat.checksum }}" - - - name: Get checksums for rsyslog.d directory files - block: - - name: Find Git rsyslog.d files - find: - paths: "{{ playbook_dir }}/../../files/rsyslog.d" - patterns: "*.conf" - recurse: false - delegate_to: localhost - register: git_confd_files - - - name: Find server rsyslog.d files - find: - paths: "{{ rsyslog_config_dir }}" - patterns: "*.conf" - recurse: false - register: server_confd_files - - - name: Get checksums for all Git rsyslog.d files - stat: - path: "{{ item.path }}" - delegate_to: localhost - loop: "{{ git_confd_files.files }}" - register: git_confd_checksums - - - name: Get checksums for all server rsyslog.d files - stat: - path: "{{ item.path }}" - loop: "{{ server_confd_files.files }}" - register: server_confd_checksums - - - name: Compare rsyslog.d file checksums - set_fact: - rsyslogd_check: - changed: "{{ git_confd_checksums.results | map(attribute='stat.checksum') | list != server_confd_checksums.results | map(attribute='stat.checksum') | list }}" - - - name: Check for extra files on server not present in Git - block: - - name: Find config files on server - ansible.builtin.find: - paths: "{{ rsyslog_config_dir }}" - patterns: "*.conf" - recurse: false - register: server_configs - - - name: Find config files in Git (controller) - ansible.builtin.find: - paths: "{{ playbook_dir }}/../../files/rsyslog.d" - patterns: "*.conf" - recurse: false - delegate_to: localhost - register: repo_configs - - - name: Build list of Git-managed filenames - ansible.builtin.set_fact: - git_filenames: "{{ repo_configs.files | map(attribute='path') | map('basename') | list }}" - - - name: Build list of server filenames - ansible.builtin.set_fact: - server_filenames: "{{ server_configs.files | map(attribute='path') | map('basename') | list }}" - - - name: Find server files that are managed by Git but missing on server - ansible.builtin.set_fact: - missing_on_server: "{{ git_filenames | difference(server_filenames) }}" - - - name: Show missing files - ansible.builtin.debug: - msg: "Files in Git but missing on server: {{ missing_on_server }}" - when: missing_on_server | length > 0 - - # Initialize missing_on_server with default empty list to avoid undefined variable errors - - name: Initialize missing files tracking - ansible.builtin.set_fact: - missing_on_server: [] - - - name: Set overall drift flag - ansible.builtin.set_fact: - # Drift detected if: main config changed OR rsyslog.d changed OR any git-managed files missing from server - # Using | default([]) to safely handle undefined variables in container environment - drift_detected: "{{ main_config_check.changed or rsyslogd_check.changed or (missing_on_server | default([]) | length > 0) }}" - - # ───────────────────────────────────────────────────────────────────────── - # Debug: Show WHAT changed (for troubleshooting) - # ───────────────────────────────────────────────────────────────────────── - - name: Show main config change status - ansible.builtin.debug: - msg: "Main config (rsyslog.conf) changed: {{ main_config_check.changed }}" - - - name: Show rsyslog.d change status - ansible.builtin.debug: - msg: "rsyslog.d directory changed: {{ rsyslogd_check.changed }}" - - - name: Show main config diff if changed - ansible.builtin.debug: - var: main_config_check.diff - when: main_config_check.changed and main_config_check.diff is defined - - - name: Show rsyslog.d diff if changed - ansible.builtin.debug: - var: rsyslogd_check.diff - when: rsyslogd_check.changed and rsyslogd_check.diff is defined - - - name: Show rsyslog.d diff list - ansible.builtin.debug: - msg: "rsyslogd_check details: {{ rsyslogd_check }}" - when: rsyslogd_check.changed - - - name: Debug rsyslogd_check.diff structure - ansible.builtin.debug: - msg: | - rsyslogd_check.diff is list: {{ rsyslogd_check.diff is iterable and rsyslogd_check.diff is not string }} - rsyslogd_check.diff length: {{ rsyslogd_check.diff | length if rsyslogd_check.diff is iterable else 'N/A' }} - rsyslogd_check.diff first item: {{ rsyslogd_check.diff[0] if rsyslogd_check.diff is iterable and rsyslogd_check.diff | length > 0 else 'empty' }} - Full diff content: {{ rsyslogd_check.diff }} - when: rsyslogd_check.changed and rsyslogd_check.diff is defined - - # ───────────────────────────────────────────────────────────────────────── - # Build structured list of changed files for GitOps status server - # This data is parsed by the update-gitops-status.sh wrapper script - # ───────────────────────────────────────────────────────────────────────── - - name: Initialize list of drifted files - ansible.builtin.set_fact: + drift_detected: false drifted_files: [] - - name: Add main config to drifted files if changed - ansible.builtin.set_fact: - drifted_files: "{{ drifted_files + ['/etc/rsyslog.conf'] }}" - when: main_config_check.changed + # ───────────────────────────────────────────────────────────────────────── + # Compare rsyslog.conf content + # ───────────────────────────────────────────────────────────────────────── + - name: Read Git rsyslog.conf + slurp: + src: "{{ playbook_dir }}/../../files/rsyslog.conf" + delegate_to: localhost + register: git_main_conf - - name: Mark rsyslog.d directory as changed (simplified) - ansible.builtin.set_fact: - drifted_files: "{{ drifted_files + ['/etc/rsyslog.d/'] }}" - when: rsyslogd_check.changed + - name: Read server rsyslog.conf + slurp: + src: "{{ rsyslog_main_config }}" + register: server_main_conf - # NOTE: missing_on_server files are tracked in drift_detected flag but not in drifted_files list - # This is intentional - they indicate missing deployed files, which is a drift condition + - name: Check rsyslog.conf content match + set_fact: + main_conf_match: "{{ git_main_conf.content == server_main_conf.content }}" + + - name: Mark drift if rsyslog.conf differs + set_fact: + drift_detected: true + drifted_files: "{{ drifted_files + ['rsyslog.conf'] }}" + when: not main_conf_match # ───────────────────────────────────────────────────────────────────────── - # Debug output: Show structured drifted files for parsing - # Format: DRIFTED_FILES=file1,file2,file3 (or empty if no drift) - # This makes it easy for update-gitops-status.sh to extract changed files - # ALWAYS output this line for reliable parsing, even when empty + # Compare rsyslog.d directory files # ───────────────────────────────────────────────────────────────────────── - - name: Output structured list of drifted files for GitOps status server - ansible.builtin.debug: + - name: List Git rsyslog.d files + find: + paths: "{{ playbook_dir }}/../../files/rsyslog.d" + patterns: "*.conf" + recurse: false + delegate_to: localhost + register: git_confd_list + + - name: List server rsyslog.d files + find: + paths: "{{ rsyslog_config_dir }}" + patterns: "*.conf" + recurse: false + register: server_confd_list + + - name: Extract file names + set_fact: + git_confd_names: "{{ git_confd_list.files | map(attribute='path') | map('basename') | sort }}" + server_confd_names: "{{ server_confd_list.files | map(attribute='path') | map('basename') | sort }}" + + - name: Check rsyslog.d file list match + set_fact: + confd_match: "{{ git_confd_names == server_confd_names }}" + + - name: Mark drift if rsyslog.d files differ + set_fact: + drift_detected: true + drifted_files: "{{ drifted_files + ['rsyslog.d/'] }}" + when: not confd_match + + # ───────────────────────────────────────────────────────────────────────── + # Output markers for update-gitops-status.sh parsing + # ───────────────────────────────────────────────────────────────────────── + - name: Output drifted files list + debug: msg: "DRIFTED_FILES={{ drifted_files | join(',') if drifted_files | length > 0 else '' }}" - - name: Output sync status marker for parsing - ansible.builtin.debug: + - name: Output SYNCED status + debug: msg: "SYNC_STATUS=SYNCED" when: not drift_detected - - name: Output sync status marker for parsing - ansible.builtin.debug: + - name: Output OUT_OF_SYNC status + debug: msg: "SYNC_STATUS=OUT_OF_SYNC" when: drift_detected - - name: Print SYNCED status - ansible.builtin.debug: + - name: Display SYNCED + debug: msg: | ╭─────────────────────────────╮ │ ✓ SYNCED │ @@ -197,8 +96,8 @@ ╰─────────────────────────────╯ when: not drift_detected - - name: Print OUT OF SYNC status - ansible.builtin.debug: + - name: Display OUT_OF_SYNC + debug: msg: | ╭─────────────────────────────╮ │ ✗ OUT OF SYNC │ @@ -207,6 +106,7 @@ when: drift_detected - name: Fail if drift detected - ansible.builtin.fail: - msg: "Configuration drift detected. Live system does not match repository." + fail: + msg: "Configuration drift detected." when: drift_detected +