diff --git a/argocd-apps/air-flow.yaml b/argocd-apps/air-flow.yaml new file mode 100644 index 0000000..b462824 --- /dev/null +++ b/argocd-apps/air-flow.yaml @@ -0,0 +1,23 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: air-flow + namespace: argocd + labels: + env: dev-tools +spec: + project: dev-tools + source: + repoURL: ssh://git@gitea-ssh.dev-tools.svc.cluster.local:2222/dvirlabs/dev-tools.git + targetRevision: HEAD + path: charts/air-flow + helm: + valueFiles: + - ../../manifests/air-flow/values.yaml + destination: + server: https://kubernetes.default.svc + namespace: dev-tools + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/charts/airflow/.helmignore b/charts/airflow/.helmignore new file mode 100644 index 0000000..6d231e1 --- /dev/null +++ b/charts/airflow/.helmignore @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +bin + +# We do not want to include our Python Helm Chart Unit test files +tests diff --git a/charts/airflow/.pre-commit-config.yaml b/charts/airflow/.pre-commit-config.yaml new file mode 100644 index 0000000..f95650c --- /dev/null +++ b/charts/airflow/.pre-commit-config.yaml @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.2.0' +default_language_version: + python: python3 + node: 22.19.0 + golang: 1.24.0 +repos: + - repo: https://github.com/Lucas-C/pre-commit-hooks + rev: ad1b27d73581aa16cca06fc4a0761fc563ffe8e8 # frozen: v1.5.6 + hooks: + - id: insert-license + name: Add license for all Helm template files + files: ^templates/.* + args: + - --comment-style + - "{{/*||*/}}" + - --license-filepath + - ../scripts/ci/license-templates/LICENSE.txt + - --fuzzy-match-generates-todo + - repo: local + hooks: + - id: update-chart-dependencies + name: Update chart dependencies to latest (manual) + entry: ../scripts/ci/prek/update_chart_dependencies.py + stages: ['manual'] + language: python + files: ^\.pre-commit-config\.yaml$|^../scripts/ci/prek/update_build_dependencies\.py$ + pass_filenames: false + require_serial: true + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0 + hooks: + - id: pretty-format-json + name: Format JSON files + args: + - --autofix + - --no-sort-keys + - --indent + - "4" + files: + (?x) + ^values\.schema\.json$| + ^values_schema\.schema\.json$ + pass_filenames: true + - repo: local + hooks: + - id: lint-helm-chart + name: Lint Helm Chart + entry: ../scripts/ci/prek/lint_helm.py + language: python + pass_filenames: false + files: ^.* + require_serial: true + - id: validate-chart-annotations + name: Validate chart annotations + entry: ../scripts/ci/prek/validate_chart_annotations.py + language: python + pass_filenames: false + files: ^Chart\.yaml$ + - id: kubeconform + name: Kubeconform check on our helm chart + entry: ../scripts/ci/prek/check_kubeconform.py + language: python + pass_filenames: false + files: ^.* + require_serial: true + - id: lint-json-schema + name: Lint chart/values.schema.json + entry: ../scripts/ci/prek/lint_json_schema.py + args: + - --spec-file + - values_schema.schema.json + - values.schema.json + language: python + pass_filenames: false + files: ^values\.schema\.json$|^values_schema\.schema\.json$ + require_serial: true + - id: update-vendored-in-k8s-json-schema + name: Vendor k8s definitions into values.schema.json + entry: ../scripts/ci/prek/vendor_k8s_json_schema.py + language: python + files: ^values\.schema\.json$ + - id: lint-json-schema + name: Lint chart/values.yaml + entry: ../scripts/ci/prek/lint_json_schema.py + args: + - --enforce-defaults + - --spec-file + - values.schema.json + - values.yaml + language: python + pass_filenames: false + files: ^values\.yaml$|^values\.schema\.json$ + require_serial: true + - id: lint-chart-schema + name: Lint chart/values.schema.json file + entry: ../scripts/ci/prek/chart_schema.py + language: python + pass_filenames: false + files: ^values\.schema\.json$ + require_serial: true diff --git a/charts/airflow/Chart.lock b/charts/airflow/Chart.lock new file mode 100644 index 0000000..b214f55 --- /dev/null +++ b/charts/airflow/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 13.2.24 +digest: sha256:07f12ed410f106bf13eca69df16a1ef6690c4d4bfcb037943bbff6e71a22201d +generated: "2023-12-09T17:23:53.209725+01:00" diff --git a/charts/airflow/Chart.yaml b/charts/airflow/Chart.yaml new file mode 100644 index 0000000..bd3adff --- /dev/null +++ b/charts/airflow/Chart.yaml @@ -0,0 +1,155 @@ +annotations: + artifacthub.io/changes: | + - description: > + Workers config options have been moved under workers.celery.* and workers.kubernetes.*: + safeToEvict, hostAliases, priorityClassName, runtimeClassName, schedulerName, + serviceAccount, extraContainers, extraInitContainers, extraVolumes, affinity, + tolerations, topologySpreadConstraints, podAnnotations, labels, env, extraVolumeMounts + are moved to both; extraPorts, volumeClaimTemplates, waitForMigrations, hpa, + annotations, logGroomerSidecar are Celery-specific and moved under workers.celery.* + kind: changed + links: + - name: '#61915' + url: https://github.com/apache/airflow/pull/61915 + - name: '#61919' + url: https://github.com/apache/airflow/pull/61919 + - name: '#61960' + url: https://github.com/apache/airflow/pull/61960 + - name: '#61961' + url: https://github.com/apache/airflow/pull/61961 + - name: '#61962' + url: https://github.com/apache/airflow/pull/61962 + - name: '#62030' + url: https://github.com/apache/airflow/pull/62030 + - name: '#62048' + url: https://github.com/apache/airflow/pull/62048 + - name: '#62054' + url: https://github.com/apache/airflow/pull/62054 + - name: '#64730' + url: https://github.com/apache/airflow/pull/64730 + - name: '#64734' + url: https://github.com/apache/airflow/pull/64734 + - name: '#64739' + url: https://github.com/apache/airflow/pull/64739 + - name: '#64741' + url: https://github.com/apache/airflow/pull/64741 + - name: '#64746' + url: https://github.com/apache/airflow/pull/64746 + - name: '#64860' + url: https://github.com/apache/airflow/pull/64860 + - name: '#64976' + url: https://github.com/apache/airflow/pull/64976 + - name: '#64980' + url: https://github.com/apache/airflow/pull/64980 + - name: '#64982' + url: https://github.com/apache/airflow/pull/64982 + - name: '#65027' + url: https://github.com/apache/airflow/pull/65027 + - name: '#65030' + url: https://github.com/apache/airflow/pull/65030 + - name: '#65033' + url: https://github.com/apache/airflow/pull/65033 + - name: '#65056' + url: https://github.com/apache/airflow/pull/65056 + - name: '#65059' + url: https://github.com/apache/airflow/pull/65059 + - description: Default Airflow image updated to 3.2.0 + kind: changed + links: + - name: '#64841' + url: https://github.com/apache/airflow/pull/64841 + - description: Fix wrong broker URL secret ref + kind: fixed + links: + - name: '#65006' + url: https://github.com/apache/airflow/pull/65006 + - description: Add ttlSecondsAfterFinished to database cleanup job + kind: added + links: + - name: '#64164' + url: https://github.com/apache/airflow/pull/64164 + - description: > + Support tpl rendering in ServiceAccount annotations, metadataConnection, and config ConfigMap names + kind: added + links: + - name: '#64763' + url: https://github.com/apache/airflow/pull/64763 + - description: Generate JWT Secret of recommended length + kind: changed + links: + - name: '#65082' + url: https://github.com/apache/airflow/pull/65082 + - description: Fix Helm chart image volume schema validation + kind: fixed + links: + - name: '#65409' + url: https://github.com/apache/airflow/pull/65409 + - description: Remove duplicate fallback branch in airflowPodSecurityContextsIds helper + kind: fixed + links: + - name: '#65558' + url: https://github.com/apache/airflow/pull/65558 + - description: Render cleanup RBAC only for KubernetesExecutor + kind: fixed + links: + - name: '#65539' + url: https://github.com/apache/airflow/pull/65539 + - description: Fix default args/command for database cleanup + kind: fixed + links: + - name: '#63821' + url: https://github.com/apache/airflow/pull/63821 + - description: Fix invalid deprecation warning in NOTES.txt + kind: fixed + links: + - name: '#64296' + url: https://github.com/apache/airflow/pull/64296 + - description: Add missing fields in schema file + kind: fixed + links: + - name: '#64339' + url: https://github.com/apache/airflow/pull/64339 + artifacthub.io/links: | + - name: Documentation + url: https://airflow.apache.org/docs/helm-chart/1.21.0/ + artifacthub.io/screenshots: | + - title: Home Page + url: https://airflow.apache.org/docs/apache-airflow/3.2.0/_images/home_dark.png + - title: DAG Overview Dashboard + url: https://airflow.apache.org/docs/apache-airflow/3.2.0/_images/dag_overview_dashboard.png + - title: DAGs View + url: https://airflow.apache.org/docs/apache-airflow/3.2.0/_images/dags.png + - title: Assets View + url: https://airflow.apache.org/docs/apache-airflow/3.2.0/_images/asset_view.png + - title: Grid View + url: https://airflow.apache.org/docs/apache-airflow/3.2.0/_images/dag_overview_grid.png + - title: Graph View + url: https://airflow.apache.org/docs/apache-airflow/3.2.0/_images/dag_overview_graph.png + - title: Variable View + url: https://airflow.apache.org/docs/apache-airflow/3.2.0/_images/variable_hidden.png + - title: Code View + url: https://airflow.apache.org/docs/apache-airflow/3.2.0/_images/dag_overview_code.png +apiVersion: v2 +appVersion: 3.2.0 +dependencies: +- condition: postgresql.enabled + name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 13.2.24 +description: The official Helm chart to deploy Apache Airflow, a platform to programmatically + author, schedule, and monitor workflows +home: https://airflow.apache.org/ +icon: https://airflow.apache.org/images/airflow_dark_bg.png +keywords: +- apache +- airflow +- workflow +- scheduler +maintainers: +- email: dev@airflow.apache.org + name: Apache Airflow PMC +name: airflow +sources: +- https://github.com/apache/airflow +type: application +version: 1.21.0 diff --git a/charts/airflow/INSTALL b/charts/airflow/INSTALL new file mode 100644 index 0000000..86f4cdf --- /dev/null +++ b/charts/airflow/INSTALL @@ -0,0 +1,14 @@ +## INSTALL / BUILD instructions for Apache Airflow Chart + +# The Assumption here is that you have a running Kubernetes cluster +# and helm installed & configured to talk with the cluster + +# Run `helm install` Command +helm install airflow . + +# If you want to install in a particular namespace +## Create that namespace (example 'airflow' here, change it as needed) +kubectl create namespace airflow + +## Install the chart in that namespace +helm install airflow -n airflow . diff --git a/charts/airflow/LICENSE b/charts/airflow/LICENSE new file mode 100644 index 0000000..11069ed --- /dev/null +++ b/charts/airflow/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/charts/airflow/NOTICE b/charts/airflow/NOTICE new file mode 100644 index 0000000..c33a7b7 --- /dev/null +++ b/charts/airflow/NOTICE @@ -0,0 +1,17 @@ +Apache Airflow +Copyright 2016-2026 The Apache Software Foundation + +This product includes software developed at The Apache Software +Foundation (http://www.apache.org/). +======================================================================= + +postgresql: +----- +This product contains vendored-in postgresql Helm chart. + +Copyright © 2022 Bitnami + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/charts/airflow/README.md b/charts/airflow/README.md new file mode 100644 index 0000000..248e0b7 --- /dev/null +++ b/charts/airflow/README.md @@ -0,0 +1,75 @@ + + +# Helm Chart for Apache Airflow + +[![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/apache-airflow)](https://artifacthub.io/packages/search?repo=apache-airflow) + +[Apache Airflow](https://airflow.apache.org/) is a platform to programmatically author, schedule and monitor workflows. + +## Introduction + +This chart will bootstrap an [Airflow](https://airflow.apache.org) deployment on a [Kubernetes](http://kubernetes.io) +cluster using the [Helm](https://helm.sh) package manager. + +## Requirements + +- Kubernetes 1.30+ cluster +- Helm 3.0+ +- PV provisioner support in the underlying infrastructure (optionally) + +## Features + +* Supported executors (all Airflow versions): ``LocalExecutor``, ``CeleryExecutor``, ``KubernetesExecutor`` +* Supported executors (Airflow version ``2.11.X``): ``LocalKubernetesExecutor``, ``CeleryKubernetesExecutor`` +* Supported multiple Executors (``2.11+``) +* Supported AWS executors with AWS provider version ``8.21.0+``: + * ``airflow.providers.amazon.aws.executors.batch.AwsBatchExecutor`` + * ``airflow.providers.amazon.aws.executors.ecs.AwsEcsExecutor`` +* Supported Edge executor with edge3 provider version ``1.0.0+``: + * ``airflow.providers.edge3.executors.EdgeExecutor`` +* Supported Airflow version: ``2.11+``, ``3.0+`` +* Supported database backend: ``PostgreSQL``, ``MySQL`` +* Autoscaling for ``CeleryExecutor`` provided by KEDA +* ``PostgreSQL`` and ``PgBouncer`` with a battle-tested configuration +* **Security enhancements**: + * Container-specific Service Account Token Volume configuration implementing Principle of Least Privilege + * Only scheduler containers receive API access; init and sidecar containers operate without tokens + * Defense-in-depth security with both ServiceAccount and Pod-level controls + * Compatibility with security policies like Kyverno and compliance frameworks +* Monitoring: + * StatsD/Prometheus metrics for Airflow + * Prometheus metrics for PgBouncer + * Flower +* Automatic database migration after a new deployment +* Administrator account creation during deployment +* Kerberos secure configuration +* One-command deployment for any type of executor. You don't need to provide other services e.g. Redis/Database to test the Airflow. + +## Documentation + +Full documentation for Helm Chart (latest **stable** release) lives [on the website](https://airflow.apache.org/docs/helm-chart/). + +> Note: If you're looking for documentation for main branch (latest development branch): you can find it on [s.apache.org/airflow-docs/](http://apache-airflow-docs.s3-website.eu-central-1.amazonaws.com/docs/helm-chart/stable/index.html). +> Source code for documentation is in [../docs/helm-chart](https://github.com/apache/airflow/tree/main/docs/helm-chart) +> + +## Contributing + +Want to help build Apache Airflow? Check out our [contributing documentation](https://github.com/apache/airflow/blob/main/contributing-docs/README.rst). diff --git a/charts/airflow/RELEASE_NOTES.rst b/charts/airflow/RELEASE_NOTES.rst new file mode 100644 index 0000000..d6b4fff --- /dev/null +++ b/charts/airflow/RELEASE_NOTES.rst @@ -0,0 +1,1695 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. contents:: Apache Airflow Helm Chart Releases + :local: + :depth: 1 + +Run ``helm repo update`` before upgrading the chart to the latest version. + +.. towncrier release notes start + +Airflow Helm Chart 1.21.0 (2026-04-21) +-------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + + +Workers config options have been moved under ``workers.celery.*`` and ``workers.kubernetes.*`` +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Please update your configuration accordingly: + +* ``workers.safeToEvict`` is now deprecated in favor of ``workers.celery.safeToEvict``/``workers.kubernetes.safeToEvict`` (#61915). +* ``workers.hostAliases`` is now deprecated in favor of ``workers.celery.hostAliases``/``workers.kubernetes.hostAliases`` (#61960). +* ``workers.priorityClassName`` is now deprecated in favor of ``workers.celery.priorityClassName``/``workers.kubernetes.priorityClassName`` (#61961). +* ``workers.runtimeClassName`` is now deprecated in favor of ``workers.celery.runtimeClassName``/``workers.kubernetes.runtimeClassName`` (#61962). +* ``workers.schedulerName`` is now deprecated in favor of ``workers.celery.schedulerName``/``workers.kubernetes.schedulerName`` (#62030). +* ``workers.serviceAccount`` is now deprecated in favor of ``workers.celery.serviceAccount``/``workers.kubernetes.serviceAccount`` (#64730). +* ``workers.extraContainers`` is now deprecated in favor of ``workers.celery.extraContainers``/``workers.kubernetes.extraContainers`` (#64739). +* ``workers.extraInitContainers`` is now deprecated in favor of ``workers.celery.extraInitContainers``/``workers.kubernetes.extraInitContainers`` (#64741). +* ``workers.extraVolumes`` is now deprecated in favor of ``workers.celery.extraVolumes``/``workers.kubernetes.extraVolumes`` (#64746). +* ``workers.affinity`` is now deprecated in favor of ``workers.celery.affinity``/``workers.kubernetes.affinity`` (#64860). +* ``workers.tolerations`` is now deprecated in favor of ``workers.celery.tolerations``/``workers.kubernetes.tolerations`` (#64976). +* ``workers.topologySpreadConstraints`` is now deprecated in favor of ``workers.celery.topologySpreadConstraints``/``workers.kubernetes.topologySpreadConstraints`` (#64980). +* ``workers.podAnnotations`` is now deprecated in favor of ``workers.celery.podAnnotations``/``workers.kubernetes.podAnnotations`` (#65027). +* ``workers.labels`` is now deprecated in favor of ``workers.celery.labels``/``workers.kubernetes.labels`` (#65030). +* ``workers.env`` is now deprecated in favor of ``workers.celery.env``/``workers.kubernetes.env`` (#65056). +* ``workers.extraVolumeMounts`` is now deprecated in favor of ``workers.celery.extraVolumeMounts``/``workers.kubernetes.extraVolumeMounts`` (#65059). +* ``workers.extraPorts`` is now deprecated in favor of ``workers.celery.extraPorts`` (#61919). +* ``workers.volumeClaimTemplates`` is now deprecated in favor of ``workers.celery.volumeClaimTemplates`` (#62048). +* ``workers.waitForMigrations`` is now deprecated in favor of ``workers.celery.waitForMigrations`` (#62054). +* ``workers.hpa`` is now deprecated in favor of ``workers.celery.hpa`` (#64734). +* ``workers.annotations`` is now deprecated in favor of ``workers.celery.annotations`` (#64982). +* ``workers.logGroomerSidecar`` is now deprecated in favor of ``workers.celery.logGroomerSidecar`` (#65033). + +The previous configuration options are still working but are deprecated and will be removed in a future version. + +Default Airflow image is updated to ``3.2.0`` (#64841) +"""""""""""""""""""""""""""""""""""""""""""""""""""""" +The default Airflow image that is used with the Chart is now ``3.2.0``, previously it was ``3.1.8``. + +New Features +^^^^^^^^^^^^ + +- Add ``ttlSecondsAfterFinished`` to database cleanup job (#64164) +- Support ``tpl`` rendering in ServiceAccount annotations, ``metadataConnection``, and config ConfigMap names (#64763) + +Improvements +^^^^^^^^^^^^ + +- Generate JWT Secret of recommended length (#65082) + +Bug Fixes +^^^^^^^^^ + +- Fix wrong broker URL secret ref (#65006) +- Fix Helm chart image volume schema validation (#65409) +- Remove duplicate fallback branch in ``airflowPodSecurityContextsIds`` helper (#65558) +- Render cleanup RBAC only for ``KubernetesExecutor`` (#65539) +- Fix default args/command for database cleanup (#63821) +- Fix invalid deprecation warning in NOTES.txt (#64296) +- Add missing fields in schema file (#64339) + +Doc only changes +^^^^^^^^^^^^^^^^ + +- Document secret key names for Helm chart ``secretName`` options (#64136) +- Update customizing-labels documentation (#64170) +- Fix documentation link (#64355) + +Misc +^^^^ + +- Simplify Helm Chart Logic & Misc (#63957) +- Improve consistency of ``values.yaml`` & misc (#64559) +- Align ``log_id_template`` with current default in Elasticsearch provider (#64332) +- Update alpine version in pgbouncer and pgbouncer-exporter (#65413) +- Add default ``GO_VERSION`` for pgbouncer-exporter Dockerfile (#65446) + + +Airflow Helm Chart 1.20.0 (2026-03-16) +-------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +Support for old versions of Apache Airflow <2.11 has been dropped (#61018) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Minimum supported version of Apache Airflow is now 2.11.0. If you want to deploy an +old version of Apache Airflow, please use the last released version of the chart 1.19.0. + +``workers`` specific sections have been moved to ``workers.celery`` / ``workers.kubernetes`` sections +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Please update your configuration accordingly: + +* ``workers.command`` command is now deprecated in favor of ``workers.celery.command``/``workers.kubernetes.command`` (#60067). +* ``workers.securityContexts`` command is now deprecated in favor of ``workers.celery.securityContexts``/``workers.kubernetes.securityContexts`` (#60396). +* ``workers.containerLifecycleHooks`` command is now deprecated in favor of ``workers.celery.containerLifecycleHooks``/``workers.kubernetes.containerLifecycleHooks`` (#61369). +* ``workers.kerberosSidecar`` section is now deprecated in favor of ``workers.celery.kerberosSidecar``/``workers.kubernetes.kerberosSidecar`` (#61881). +* ``workers.kerberosInitContainer`` section is now deprecated in favor of ``workers.celery.kerberosInitContainer``/``workers.kubernetes.kerberosInitContainer`` (#60751). +* ``workers.terminationGracePeriodSeconds`` command is now deprecated in favor of ``workers.celery.terminationGracePeriodSeconds``/``workers.kubernetes.terminationGracePeriodSeconds`` (#61892). +* ``workers.nodeSelector`` command is now deprecated in favor of ``workers.celery.nodeSelector``/``workers.kubernetes.nodeSelector`` (#61957). +* ``workers.podDisruptionBudget`` section is now deprecated in favor of ``workers.celery.podDisruptionBudget``. Please update your configuration accordingly. (#61414) +* ``workers.keda`` section is now deprecated in favor of ``workers.celery.keda``. Please update your configuration accordingly. (#61820) +* ``workers.resources`` section is now deprecated in favor of ``workers.celery.resources`` and ``workers.kubernetes.resources``. Please update your configuration accordingly. (#61890) + +The previous configuration options are still working, but are deprecated and will be removed in a future version. + + +As Git-Sync is not service-type object, the readiness probe will be removed. (#62334) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +To enable feature behaviour set ``dags.gitSync.recommendedProbeSetting`` to ``true``. Section itself will be removed in future release as to not break setups during upgrades. + +As Git-Sync has dedicated liveness service, the liveness probe behaviour will be changed. To enable feature behaviour set ``dags.gitSync.recommendedProbeSetting`` to ``true``. + +Please update your configuration accordingly. + + +Automatic env variables removed from ``container_extra_envs`` and ``custom_airflow_environment`` (#60750) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The automatic prefix addition for Kubernetes Executor environment variables and secrets has been removed from both the ``container_extra_envs`` and ``custom_airflow_environment`` helper functions. + +**What changed:** + +Previously, when you added environment variables to component-specific configurations (e.g., ``.Values.scheduler.env``), the chart automatically created an additional environment variable (to specified in the ``env`` section) with the ``AIRFLOW__KUBERNETES_ENVIRONMENT_VARIABLES__`` prefix for Kubernetes Executor worker pods. After this change, only the variable specified in ``env`` section will be created. +Furthermore, for values specified under ``.Values.secret`` section, the ``AIRFLOW__KUBERNETES_SECRETS__`` prefix is no longer automatically added. Secrets are now passed as-is via ``secretKeyRef`` without the prefixed copy for worker pods. + +**Why this change:** + +* Prevent unintended exposure of sensitive data like ``client_secret`` information. Previously, due to prefix, it was recognized as internal Airflow configuration leading to unintended exposure in Airflow UI (under Admin -> Configuration), even when ``AIRFLOW__API__EXPOSE_CONFIG`` is set to ``non-sensitive-only``. +* Avoid unintended environment propagation to workers: component-specific env configurations are intended strictly for specific components. Previous behaviour caused these variables to be passed to worker pods, which could result in configuration conflicts and unexpected side effects. + +**Migration Required:** + +If you need to pass environment variables specifically to Kubernetes Executor worker pods, use one of the following approaches: + +**Option 1: Use ``.Values.env``** + +.. code-block:: yaml + + env: + - name: my_var + value: "my_value" + +Environment variables specified under ``.Values.env`` are now passed as-is without the automatic prefix (same behaviour as component-specific env). + +**Option 2: Use ``.Values.config.kubernetes_environment_variables``** + +.. code-block:: yaml + + config: + kubernetes_environment_variables: + my_var: "my_value" + + +Default Airflow image is updated to ``3.1.8`` (#63392) +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``3.1.8``, previously it was ``3.1.7``. + + +Features +^^^^^^^^ + +- Support Helm template expressions in ``podAnnotations`` and ``airflowPodAnnotations`` values (#63019) +- Add minute-level log retention to clean-logs script (#61855) +- Add LOG_MAX_SIZE environment variables to log groomer (#61559) + +Improvements +^^^^^^^^^^^^ + +- Remove automatic ``KUBERNETES_ENVIRONMENT_VARIABLES`` and ``KUBERNETES_SECRETS`` prefixes from chart helpers (#60750) +- Remove JWT secrets from triggerer, worker and dag-processor (#63204) +- Add ``workers.celery.nodeSelector`` & ``workers.kubernetes.nodeSelector`` (#61957) +- Add ``workers.celery.terminationGracePeriodSeconds`` & ``workers.kubernetes.terminationGracePeriodSeconds`` (#61892) +- Add ``workers.celery.resources`` & ``workers.kubernetes.resources`` (#61890) +- Add ``workers.celery.keda`` section (#61820) +- Add ``workers.celery.podDisruptionBudget`` (#61414) +- Add ``workers.celery.containerLifecycleHooks`` & ``workers.kubernetes.containerLifecycleHooks`` (#61369) +- Refactor Git-Sync ``livenessProbe`` & deprecate ``readinessProbe`` & add ``startupProbe`` (#62334) +- Warn on deprecated per-component ``securityContext`` values (#62729) +- Add ingress deprecation warnings for ``apiServer``, ``statsd``, and ``pgbouncer`` (#62490) +- Add missing support for: ``securityContexts`` and ``containerLifecycleHook`` (#60677) + +Bug Fixes +^^^^^^^^^ + +- More restrictive chart rendering logic (#63464) +- Omit api-server ``spec.replicas`` when HPA is enabled (#63187) +- Add ``workers.celery.kerberosSidecar`` & ``workers.kubernetes.kerberosSidecar`` sections (#61881) +- Fix chart NOTES.txt showing deprecation warnings only without secret key (#62722) +- Fix ``tpl`` rendering for TLS hosts in ingress templates #62358 (#62548) +- Fix ``webserver.defaultUser.enabled=false`` not honored (#62143) + +Doc only changes +^^^^^^^^^^^^^^^^ + +- Cleanup Helm Chart documentation (#62544) +- Add missing deprecation warnings for workers section (#63659) + +Misc +^^^^ + +- Drop support for all Airflow versions below 2.11 in Helm Chart (#61018) +- Default airflow version to 3.1.8 (#63392) +- Add ``*.iml`` to .gitignore in all distributions (#63636) +- Upgrade important CI environment (#62792, #62610) +- Allow to use short SPDX license identifier for selected files (#62073) +- Fix all build-system/requires including transitive dependencies (#62570) + + +Airflow Helm Chart 1.19.0 (2026-02-17) +-------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +StatsD metrics aggregation now supports configurable TTL-enabled LRU cache to prevent memory growth in long-running daemons (#60933) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The Helm Chart now includes new configuration options for StatsD aggregation management: + +* ``statsd.cache.type`` - Enable TTL-enabled ``lru`` cache or ``random`` cache for metrics aggregation (default: ``lru``) +* ``statsd.cache.size`` - Maximum number of metrics to cache (default: 1000) +* ``statsd.cache.ttl`` - Time-to-live for cached metrics in seconds (``0s`` is TTL disabled) (default: ``0s``) + +This feature addresses uncontrolled memory growth in StatsD daemons by automatically cleaning up stale or unused metric entries. When enabled, the cache uses both LRU (Least Recently Used) eviction and TTL (Time To Live) expiration to manage memory usage effectively. + +To maintain backward compatibility, the default behaviour remains unchanged. Users experiencing memory growth issues with StatsD can enable this feature by setting ``statsd.cache.ttl`` to value higher than ``0`` in their Helm values. + +Support for Multiple Celery Worker Sets in the Helm Chart (#58547) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +This change introduces support for advanced Celery Workers topologies to Apache Airflow Helm Chart, enabling more flexible resource allocation and precise autoscaling configurations. + +**Flexible Worker Topologies**: The new ``workers.celery.enableDefault`` flag allows users to configure a deployment consisting only of specialized worker sets defined in ``workers.celery.sets`` section. + +**Multi-Queue Autoscaling Support**: Updates the KEDA ScaledObject generation to support comma-separated queue lists. By using the ``SQL IN (...)`` clause, we ensure that KEDA scales worker sets based on the precise aggregate workload of all their assigned queues. + +**Granular Configuration Overrides**: This change allows for overwrite of any currently available workers configuration per worker set. For example, a user can enable KEDA globally, but explicitly disable it for a specific worker set that requires a static number of replicas. + +Options to create a default user have been moved under the ``createUserJob`` section +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Please update your configuration accordingly: + +* ``webserver.defaultUser`` section is now deprecated in favor of ``createUserJob`` (#59767) + +The previous configuration options are still working but are deprecated and will be removed in a future version. + +Note that the previous documentation described also the option ``apiServer.defaultUser``, which was never implemented in the chart. The only supported option is now ``createUserJob``. Using ``apiServer.defaultUser`` will raise an error. + +Celery specific config options have been moved under the ``celery`` section in ``workers`` +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Please update your configuration accordingly: + +* ``workers.replicas`` command is now deprecated in favor of ``workers.celery.replicas`` (#59730) +* ``workers.revisionHistoryLimit`` command is now deprecated in favor of ``workers.celery.revisionHistoryLimit`` (#60056) +* ``workers.args`` command is now deprecated in favor of ``workers.celery.args`` (#60163) +* ``workers.livenessProbe`` section is now deprecated in favor of ``workers.celery.livenessProbe`` (#60186) +* ``workers.updateStrategy`` section is now deprecated in favor of ``workers.celery.updateStrategy`` (#60351) +* ``workers.strategy`` section is now deprecated in favor of ``workers.celery.strategy`` (#60354) +* ``workers.podManagementPolicy`` section is now deprecated in favor of ``workers.celery.podManagementPolicy`` (#60359) +* ``workers.persistence`` section is now deprecated in favor of ``workers.celery.persistence`` (#60238) + +The previous configuration options are still working but are deprecated and will be removed in a future version. + +Manual Service Account Token Volume configuration for pod-launching executors (#59156) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Added support for manual Service Account Token Volume configuration when using pod-launching executors +(``CeleryExecutor``, ``CeleryKubernetesExecutor``, ``KubernetesExecutor``, ``LocalKubernetesExecutor``). +This implements defense-in-depth security with both ServiceAccount and Pod-level controls, providing +compatibility with security policies like Kyverno and enabling container-specific privilege assignment +following the Principle of Least Privilege. + + +Add ``imagePullSecrets`` option (#58094) +"""""""""""""""""""""""""""""""""""""""" + +Add ``.Values.imagePullSecrets`` as the new mechanism for configuring registry credentials, +deprecating both ``.Values.registry.secretName`` and the automatic creation of the ``-registry`` secret from ``.Values.registry.connection``. + + +Default Airflow image is updated to ``3.1.7`` (#61447) +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``3.1.7``, previously it was ``3.0.2``. + +Default git-sync image is updated to ``4.4.2`` (#54085) +""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default git-sync image that is used with the Chart is now ``4.4.2``, previously it was ``4.3.0``. + +New Features +^^^^^^^^^^^^ + +- Add ``PodDisruptionBudget`` for Dag Processor (#60294) +- Add ``PodDisruptionBudget`` for Triggerer and Workers (#59068) +- Add ``HorizontalPodAutoscaler`` (HPA) for API Server (#52392) +- Add support for launching jobs with ``KubernetesJobOperator`` (#52024) +- Add ``CronJob`` to clean old records in the database (#58155) + + +Improvements +^^^^^^^^^^^^ + +- Improve dag_bundle_config_list Configuration (#60645) +- Add ``workers.celery.kerberosInitContainer`` & ``workers.kubernetes.kerberosInitContainer`` (#60751, #60427) +- Add ``workers.celery.securityContexts`` & ``workers.kubernetes.securityContexts`` (#60396) +- Add ``workers.celery.podManagementPolicy`` field (#60359) +- Add ``workers.celery.strategy`` field (#60354) +- Add ``workers.celery.updateStrategy`` field (#60351) +- Add ``workers.celery.persistence`` section (#60238) +- Add ``workers.celery.livenessProbe`` section (#60186) +- Add ``workers.celery.args`` field (#60163) +- Add ``workers.celery.command`` & ``workers.kubernetes.command`` (#60067) +- Allow custom ``volumeClaimTemplates`` when ``logs.persistence.enabled`` is true (#60118) +- Add checksum for JWT secret in API server and scheduler deployments (#60111) +- Add ``workers.celery.revisionHistoryLimit`` field (#60056) +- Add Redis StatefulSet ``persistentVolumeClaimRetentionPolicy`` support (#59955) +- Add ``workers.celery.replicas`` field (#59730) +- Add custom envs to database cleanup (#59804) +- Extend ``airflow_ti_running`` metrics by scheduled, queued and deferred (#58819) +- Create an explicit control for ``createUserJob`` (#56057) +- Make cleanup cronjob conditional on kubernetes executor (#58695) +- Add database cleanup options and remove deprecated ``securityContext`` field (#58663) +- Add ability to disable API Server (#56493) +- Add ``registry.secretNames`` and ``registry.connections`` options (#58094) +- Allow custom labels in StatsD, redis and Dag Processor (#55832) +- Allow setting ``restartPolicy`` for batch jobs in chart (#54354) +- Add readiness and liveliness support for git sync relay sidecars (#50218) +- Allow overriding ``schedulerName`` on worker/tasks pods (#53983) +- Allow additional ``PodDisruptionBudget`` config properties (#58864) +- Add EdgeExecutor to KEDA query (#55560) +- Allow ``revisionHistoryLimit`` to be set to 0 (#60340) +- Allow optional ``subPath`` for logs volume mount (#52350) +- Move triggerer from ``pod-log-reader-role`` to ``pod-launcher-role`` (#56872) + +Bug Fixes +^^^^^^^^^ + +- Remove ``kedaNetworkPolicySelector`` from helpers (#61564) +- Use the ``bitnamilegacy/postgresql`` image (#61156) +- Fix Compatibility of Celery Worker Sets with Workers Separation (#60420) +- Fix database cleanup cronjob ImagePullSecrets (#58626) +- Remove ``workers.celery`` breaking change (#61049) +- Fix missing templating in API Server ``extraInitContainers`` (#60812) +- Fix ``securityContext.containers``/``ingress.apiServer`` in values.schema.json (#60575) +- Remove unused ``containerLifecycleHooks`` field (#60239) +- Remove unneeded logic in api-server (#60147) +- Remove ``defaultUser`` from API Server in values.schema.json (#59762) +- Isolate ``defaultUser`` handling in ``createUserJob`` (#59767) +- Fix rendering condition of ``git_sync_ssh_key_volume`` (#59418) +- Add watch for events to the Pod launcher role (#59080) +- Ensure that git-sync actually runs when ``dags.gitSync.enabled=true`` and ``dags.persistence.enabled=true`` (#59123) +- Don't add labels to non-existent configuration options (#59213) +- Add log volume to init container for scheduler, triggerer and worker (#56418) +- Correctly derive celery sync_parallelism from scheduler CPU limits (#58733) +- Fix ingress notes (#59122) +- Fix Liveness / Readiness / Startup probe path for Airflow 3.x (#58734) +- Fix flower network policy condition when multiple executors (#58635) +- Missing SCC Role bindings for redis and api-server (#57985) +- Ensure graceful Redis shutdown(#58432) +- Start Redis directly, not via shell (#58790) +- Add missing ``airflow.fullname`` on kubernetes objects (#52953) +- StatsD deployment volume mount without subpath for live reloading (#54986) +- Fix KEDA query for Kubernetes Executor (#55559) +- Add API Server config in k8s pod template (#53533) +- Fix helm schema validation for executor value (#54682) +- Correct watch verb quoting in Airflow Job Launcher Role (#53822) +- Trim non-alphanumeric characters from the executor label (#53534) +- Fix KEDA Query to Use executor Field Instead of queue for Multiple Executors (#52840) + +Doc only changes +^^^^^^^^^^^^^^^^ + +- Document how to run the API server behind a reverse proxy (#61095) +- Clarify ingress settings for Airflow 2 vs 3 in values.yaml (#60434) +- Add database cleanup docs to Helm productions docs (#58707) +- KEDA best practices + better documentation (#58246) +- Update chart info about built-in secrets and environment variables (#58317) +- Fix typo in PgBouncer section of the Production Guide (#56754) +- Update webserver secret note in NOTES.txt and Production Guide (#55106) +- Make term Dag consistent in docs v2 (#55099) +- Add API Server to container resources docs (#54698) +- Fix YAML block scalar when providing SSH key for git-sync (#56716) + + +Airflow Helm Chart 1.18.0 (2025-07-12) +-------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +No significant changes. + +Improvements +^^^^^^^^^^^^ +- Allow ConfigMap and Secret references in ``apiServer.env`` (#51191) +- Add custom annotations to JWT Secret (#52166) +- Allow ``valuesFrom`` in ``gitSync.env`` (#50228) + +Bug Fixes +^^^^^^^^^ +- Fix JWT secret name (#52268) +- Use ``api-server`` instead of ``webserver`` in NOTES.txt for Airflow 3.0+ (#52194) +- Change default executor in pod template to support executor parameter in task (#49433) +- Use ``merged`` to render airflow.cfg and include computed defaults (#51828) +- Use ``[api] secret_key`` for Airflow 3.0+ instead of ``[webserver] secret_key`` (#52269) +- Fix for ``fernetkey`` and add test of its value (#52977) + +Doc only changes +^^^^^^^^^^^^^^^^ +- Update supported executors in docs (#52132) +- Update service name for port-forward of Airflow UI (#51945) + +Airflow Helm Chart 1.17.0 (2025-06-21) +-------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +Default Airflow image is updated to ``3.0.2`` (#51594) +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``3.0.2``, previously it was ``2.10.5``. + +New Features +^^^^^^^^^^^^ +- Add extra secret annotations to most secrets (#48890) +- Add support for EdgeExecutor (#50897) + +Improvements +^^^^^^^^^^^^ +- Unify k8s labels & add some missing k8s labels (#49522) + +Bug Fixes +^^^^^^^^^ +- Fix missing api server ingress (#49727) +- Replace break function in ``pod-launcher-rolebinding`` template (#49219) +- Add ``webserver_config.py`` file to api-server (#50108) +- Declare missing API server properties (#51012) +- Add missing api server replicas parameter (#50814) +- Fix FAB ``enable_proxy_fix`` default for Airflow 3 (#50056) +- Add the dag processor ServiceAccount to SecurityContextConstraints role binding (#51080) +- Generate JWT secret during HELM install (#49923) +- Always deploy JWT secret (#51799) +- Add missing ``workers.kerberosInitContainer`` configuration in values (#51405) +- Truncate the executor label length (#51817) +- Fix execution_api_server_url when base_url has a subpath (#51454) + +Doc only changes +^^^^^^^^^^^^^^^^ +- Bump minimum helm version in docs (#48700) +- Clarify which worker fields apply to Celery and Kubernetes worker pods (#50458) +- Capitalize the term airflow (#49450) +- Add EdgeExecutor to readme (#51017) +- Add 3.X/2.X clarification for CeleryKubernetesExecutor (#49916) + +Misc +^^^^ +- Default Airflow image is updated to ``3.0.2`` (#51594) +- Bump minimal Kubernetes version to 1.30 (#51515) +- Delete unneeded ``and`` operator (#51114) + +Airflow Helm Chart 1.16.0 (2025-04-01) +-------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +Default git-sync image is updated to ``4.3.0`` (#41411) +""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default git-sync image that is used with the Chart is now ``4.3.0``, previously it was ``4.1.0``. + + +Default Airflow image is updated to ``2.10.5`` (#46624) +""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``2.10.5``, previously it was ``2.9.3``. + +Default PgBouncer image is updated to ``1.23.1`` (#47416) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default PgBouncer image that is used with the chart is now ``airflow-pgbouncer-2025.03.05-1.23.1``, previously it was ``airflow-pgbouncer-2024.01.19-1.21.0``. + +Default PgBouncer Exporter image is updated to ``v0.18.0`` (#47416) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default PgBouncer Exporter image that is used with the chart is now ``airflow-pgbouncer-exporter-2025.03.05-0.18.0``, previously it was ``airflow-pgbouncer-exporter-2024.06.18-0.17.0``. + +Default StatsD exporter image is updated to ``v0.28.0`` (#43393) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default StatsD exporter image that is used with the chart is now ``v0.28.0``, previously it was ``v0.26.1``. + +New Features +^^^^^^^^^^^^ +- Allow passing custom env to log groomer sidecar containers (#46003) +- Allow using existing persistence claim in Redis StatefulSet (#41619) +- Add ``hostAliases`` support in Triggerer (#41725) +- Enable HPA for Airflow Webserver (#41955) +- Add env support for database migration job (#42345) +- Support NodePort on Redis Service (#41811) +- Add heartbeat metric for DAG processor (#42398) +- Option to enable ipv6 ipaddress resolve support for StatsD host (#42625) +- Allow customizing ``podManagementPolicy`` in worker (#42673) +- Support multiple executors in chart (#43606, #44424) +- Swap internal RPC server for API server in the helm chart (#44463) +- Add OpenSearch remote logging options (#45082) +- Add ``startupProbe`` to flower deployment (#45012) +- Add PgBouncer and StatsD ingress (#41759) +- Add environment variable controlling the log grooming frequency (#46237) + +Improvements +^^^^^^^^^^^^ +- Update metrics names to allow multiple executors to report metrics (#40778) +- Add a specific internal IP address for the ClusterIP service (#40912) +- Remove scheduler automate ServiceAccount token (#44173) +- More controls for PgBouncer secrets configuration (#45248) +- Add ``ti.running`` metric export (#47773) +- Add optional configuration for ``startupProbe`` ``initialDelaySeconds`` (#47094) +- Introduce ``worker.extraPorts`` to expose additional ports to worker container (#46679) + +Bug Fixes +^^^^^^^^^ +- Enable ``AIRFLOW__CELERY__BROKER_URL_CMD`` when ``passwordSecretName`` is true (#40270) +- Properly implement termination grace period seconds (#41374) +- Add kerberos env to base container env, add webserver-config volume (#41645) +- Fix ``volumeClaimTemplates`` missing ``apiVersion`` and ``kind`` (#41771) +- Render global volumes and volume mounts into cleanup job (#40191) (#42268) +- Fix flower ingress service reference (#41179) +- Fix ``volumeClaimTemplate`` for scheduler in local and persistent mode (#42946) +- Fix role binding for multiple executors (#44424) +- Set container name to ``envSourceContainerName`` in KEDA ScaledObject (#44963) +- Update scheduler-deployment to cope with multiple executors (#46039) +- Replace disallowed characters in metadata label (#46811) +- Grant Airflow API Server Permission to Read Pod Logs (#47212) +- Fix scheduler ServiceAccount auto-mount for multi-executor (#46486) + +Doc only changes +^^^^^^^^^^^^^^^^ +- Reflect in docs that ``extraInitContainers`` is supported for jobs (#41674) +- Add guide how to PgBouncer with Kubernetes Secret (#42460) +- Update descriptions private registry params (#43721) +- Change description for kerberos ``reinitFrequency`` (#45343) +- Update Helm eviction configuration guide to reflect ``workers.safeToEvict`` default value (#44852) +- Add info that ``storageClassName`` can be templated (#45176) +- Fix broker-url secret name in production guide (#45863) +- Replace DAGs with dags in docs (#47959) +- Enhance ``airflowLocalSettings`` value description (#47855) +- Be consistent with denoting templated params (#46481) + +Misc +^^^^ +- Support templated hostname in NOTES (#41423) +- Default airflow version to 2.10.5 (#46624) +- Changing triggerer config option ``default_capacity`` to ``capacity`` (#48032) +- AIP-84 Move public api under /api/v2 (#47760) +- Default to the FabAuthManager in the chart (#47976) +- Update PgBouncer to ``1.23.1`` and PgBouncer exporter to ``0.18.0`` (#47416) +- Move api-server to port 8080 (#47310) +- Start the api-server in Airflow 3, webserver in Airflow 2 (#47085) +- Move ``fastapi-api`` command to ``api-server`` (#47076) +- Move execution_api_server_url config to the core section (#46969) +- Use standalone dag processor for Airflow 3 (#45659) +- Update ``quay.io/prometheus/statsd-exporter`` from ``v0.26.1`` to ``v0.28.0`` (#43393) + +Airflow Helm Chart 1.15.0 (2024-07-24) +-------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +Default Airflow image is updated to ``2.9.3`` (#40816) +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``2.9.3``, previously it was ``2.9.2``. + +Default PgBouncer Exporter image has been updated (#40318) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The PgBouncer Exporter image has been updated to ``airflow-pgbouncer-exporter-2024.06.18-0.17.0``, which addresses CVE-2024-24786. + +New Features +^^^^^^^^^^^^ + +- Add git-sync container lifecycle hooks (#40369) +- Add init containers for jobs (#40454) +- Add persistent volume claim retention policy (#40271) +- Add annotations for Redis StatefulSet (#40281) +- Add ``dags.gitSync.sshKey``, which allows the git-sync private key to be configured in the values file directly (#39936) +- Add ``extraEnvFrom`` to git-sync containers (#39031) + +Improvements +^^^^^^^^^^^^ + +- Link in ``UIAlert`` to production guide when a dynamic webserver secret is used now opens in a new tab (#40635) +- Support disabling helm hooks on ``extraConfigMaps`` and ``extraSecrets`` (#40294) + +Bug Fixes +^^^^^^^^^ + +- Add git-sync ssh secret to DAG processor (#40691) +- Fix duplicated ``safeToEvict`` annotations (#40554) +- Add missing ``triggerer.keda.usePgbouncer`` to values.yaml (#40614) +- Trim leading ``//`` character using mysql backend (#40401) + +Doc only changes +^^^^^^^^^^^^^^^^ + +- Updating chart download link to use the Apache download CDN (#40618) + +Misc +^^^^ + +- Update PgBouncer exporter image to ``airflow-pgbouncer-exporter-2024.06.18-0.17.0`` (#40318) +- Default airflow version to 2.9.3 (#40816) +- Fix ``startupProbe`` timing comment (#40412) + +Airflow Helm Chart 1.14.0 (2024-06-18) +-------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +``ClusterRole`` and ``ClusterRoleBinding`` names have been updated to be unique (#37197) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +``ClusterRole``s and ``ClusterRoleBinding``s created when ``multiNamespaceMode`` is enabled have been renamed to ensure unique names: + + * ``{{ include "airflow.fullname" . }}-pod-launcher-role`` has been renamed to ``{{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-pod-launcher-role`` + * ``{{ include "airflow.fullname" . }}-pod-launcher-rolebinding`` has been renamed to ``{{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-pod-launcher-rolebinding`` + * ``{{ include "airflow.fullname" . }}-pod-log-reader-role`` has been renamed to ``{{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-pod-log-reader-role`` + * ``{{ include "airflow.fullname" . }}-pod-log-reader-rolebinding`` has been renamed to ``{{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-pod-log-reader-rolebinding`` + * ``{{ include "airflow.fullname" . }}-scc-rolebinding`` has been renamed to ``{{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-scc-rolebinding`` + +``workers.safeToEvict`` default changed to False (#40229) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default for ``workers.safeToEvict`` now defaults to False. This is a safer default +as it prevents the nodes workers are running on from being scaled down by the +`K8s Cluster Autoscaler `_. +If you would like to retain the previous behavior, you can set this config to True. + +Default Airflow image is updated to ``2.9.2`` (#40160) +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``2.9.2``, previously it was ``2.8.3``. + +Default StatsD image is updated to ``v0.26.1`` (#38416) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default StatsD image that is used with the Chart is now ``v0.26.1``, previously it was ``v0.26.0``. + +New Features +^^^^^^^^^^^^ + +- Enable MySQL KEDA support for triggerer (#37365) +- Allow AWS Executors (#38524) + +Improvements +^^^^^^^^^^^^ + +- Allow ``valueFrom`` in env config of components (#40135) +- Enable templating in ``extraContainers`` and ``extraInitContainers`` (#38507) +- Add safe-to-evict annotation to pod-template-file (#37352) +- Support ``workers.command`` for KubernetesExecutor (#39132) +- Add ``priorityClassName`` to Jobs (#39133) +- Add Kerberos sidecar to pod-template-file (#38815) +- Add templated field support for extra containers (#38510) + +Bug Fixes +^^^^^^^^^ + +- Set ``workers.safeToEvict`` default to False (#40229) + +Doc only changes +^^^^^^^^^^^^^^^^ + +- Document ``extraContainers`` and ``extraInitContainers`` that are templated (#40033) +- Fix typo in HorizontalPodAutoscaling documentation (#39307) +- Fix supported k8s versions in docs (#39172) +- Fix typo in YAML path for ``brokerUrlSecretName`` (#39115) + +Misc +^^^^ +- Default Airflow version to 2.9.2 (#40160) +- Limit Redis image to 7.2 (#38928) +- Build Helm values schemas with Kubernetes 1.29 resources (#38460) +- Add missing containers to resources docs (#38534) +- Upgrade StatsD Exporter image to 0.26.1 (#38416) +- Remove K8S 1.25 support (#38367) + +Airflow Helm Chart 1.13.1 (2024-03-25) +-------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +Default Airflow image is updated to ``2.8.3`` (#38036) +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``2.8.3``, previously it was ``2.8.2``. + +Bug Fixes +^^^^^^^^^ +- Don't overwrite ``.Values.airflowPodAnnotations`` (#37917) +- Fix cluster-wide RBAC naming clash when using multiple ``multiNamespace`` releases with the same name (#37197) + +Misc +^^^^ +- Chart: Default airflow version to 2.8.3 (#38036) + +Airflow Helm Chart 1.13.0 (2024-03-05) +-------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +Default Airflow image is updated to ``2.8.2`` (#37704) +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``2.8.2``, previously it was ``2.8.1``. + + +New Features +^^^^^^^^^^^^ + +- Support labels specific to the database migration objects and pods (#37490) + +Improvements +^^^^^^^^^^^^ + +- Flower K8s Probe config (#37528) + +Bug Fixes +^^^^^^^^^ +- Remove duplicate ports key in webserver service (#37356) +- Add ``AIRFLOW_HOME`` env var to log groomer sidecar (#37588) +- Skip ``.`` path when preparing reproducible packages (#37402) + +Misc +^^^^ +- Default airflow version to 2.8.2 (#37704) + +Airflow Helm Chart 1.12.0 (2024-02-11) +-------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +The helm chart is now using a newer version of ``bitnami/postgresql`` dependency (#34817) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The version of ``bitnami/postgresql`` subchart upgraded from ``12.10.0`` to ``13.2.24``. +The version of ``PostgreSQL`` binaries upgraded from ``11`` to ``16.1.0``. + +The change requires existing ``bitnami/postgresql`` subchart users to perform manual major version upgrade using ``pg_dumpall`` or ``pg_upgrade``. + +As a reminder, it is recommended to `set up an external database `_ in production. + +Default Airflow image is updated to ``2.8.1`` (#36907) +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``2.8.1``, previously it was ``2.7.1``. + +Default PgBouncer and PgBouncer Exporter images have been updated (#36898) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The PgBouncer and PgBouncer Exporter images are based on newer software/os. + + * ``pgbouncer``: 1.21.0 based on alpine 3.14 (``airflow-pgbouncer-2024.01.19-1.21.0``) + * ``pgbouncer-exporter``: 0.16.0 based on alpine 3.19 (``apache/airflow:airflow-pgbouncer-exporter-2024.01.19-0.16.0``) + +Default StatsD image is updated to ``v0.26.0`` (#37187) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default StatsD image that is used with the Chart is now ``v0.26.0``, previously it was ``v0.22.8``. + +Default Redis image is updated to ``7-bookworm`` (#37187) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default Redis image that is used with the Chart is now ``7-bookworm``, previously it was ``7-bullseye``. + +New Features +^^^^^^^^^^^^ + +- Enable native HPA for Airflow Workers (#36174) +- Add init container + sidecar support for Airflow Kerberos (#35548) +- Support MySQL backend as KEDA trigger (#36167) + +Improvements +^^^^^^^^^^^^ + +- Improve PriorityClass to improve debuggability (#36365) +- Add ``securityContexts`` in dag processors log groomer sidecar (#34499) +- Add support for ``securityContexts`` in dag processors wait-for-migrations container (#35593) +- Add templating for PVC ``storageClassName`` (#35581) +- Add ``volumeClaimTemplate`` for worker (#34986) +- Add support for ``priorityClassName`` on Redis pods (#34879) +- Configurable mount path for DAGs volume (#35083) +- Add support for custom ``emptyDir`` config (#34837) +- Added ability to enable/disable scheduler and webserver (#36991) + +Bug Fixes +^^^^^^^^^ + +- Fix StatsD host in Airflow config (#35679) +- Set ``AIRFLOW_HOME`` env var with ``airflowHome`` value (#34839) +- Safer worker pod annotations (#35309) +- Set worker ``safeToEvict`` properly (#35130) +- Fix Redis broker URL with ``useStandardNaming`` (#34825) +- Fix metadata DB & port in KEDA connection when ``usePgbouncer`` is false (#34741) +- Fix PgBouncer connection with ``useStandardNaming`` (#34787) + +Doc only changes +^^^^^^^^^^^^^^^^ + +- Add docs about extending the Airflow Helm chart (#36331) +- Add comment for Elasticsearch connection scheme (#35588) +- Add notes about Virtualenvs preventing the need for custom images (#35306) + +Misc +^^^^ + +- Default Airflow version to 2.8.1 (#36907) +- Support git-sync v4 (#34731) +- Upgrade ``bitnami/postgresql`` subchart to ``13.2.24`` (#36156) +- Change git sync container indent to 4 (#35824) +- Remove K8S 1.24 support (#35214) +- Rebuild ``pgbouncer`` and ``pgbouncer-exporter`` images with newer versions (#36898) +- Update ``statsd`` and ``redis`` chart images (#37187) + +Airflow Helm Chart 1.11.0 (2023-10-02) +-------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +Support naming customization on helm chart resources, some resources may be renamed during upgrade (#31066) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +This is a new opt-in switch ``useStandardNaming``, for backwards compatibility, to leverage the standard naming convention, which allows full use of ``fullnameOverride`` and ``nameOverride`` in all resources. + +The following resources will be renamed using default of ``useStandardNaming=false`` when upgrading to 1.11.0 or a higher version. + +- ConfigMap ``{release}-airflow-config`` to ``{release}-config`` +- Secret ``{release}-airflow-metadata`` to ``{release}-metadata`` +- Secret ``{release}-airflow-result-backend`` to ``{release}-result-backend`` +- Ingress ``{release}-airflow-ingress`` to ``{release}-ingress`` + +For existing installations, all your resources will be recreated with a new name and Helm will delete the previous resources. + +This won't delete existing PVCs for logs used by StatefulSet/Deployments, but it will recreate them with brand new PVCs. +If you do want to preserve logs history you'll need to manually copy the data of these volumes into the new volumes after +deployment. Depending on what storage backend/class you're using this procedure may vary. If you don't mind starting +with fresh logs/redis volumes, you can just delete the old PVCs that will be names, for example: + +.. code-block:: bash + + kubectl delete pvc -n airflow logs-gta-triggerer-0 + kubectl delete pvc -n airflow logs-gta-worker-0 + kubectl delete pvc -n airflow redis-db-gta-redis-0 + +If you do not change ``useStandardNaming`` or ``fullnameOverride`` after upgrade, you can proceed as usual and no unexpected behaviours will be presented. + +``bitnami/postgresql`` subchart updated to ``12.10.0`` (#33747) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The PostgreSQL subchart that is used with the Chart is now ``12.10.0``, previously it was ``12.1.9``. + +Default git-sync image is updated to ``3.6.9`` (#33748) +""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default git-sync image that is used with the Chart is now ``3.6.9``, previously it was ``3.6.3``. + +Default Airflow image is updated to ``2.7.1`` (#34186) +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``2.7.1``, previously it was ``2.6.2``. + +New Features +^^^^^^^^^^^^ + +- Add support for scheduler name to PODs templates (#33843) +- Support KEDA scaling for triggerer (#32302) +- Add support for container lifecycle hooks (#32349, #34677) +- Support naming customization on helm chart resources (#31066) +- Adding ``startupProbe`` to scheduler and webserver (#33107) +- Allow disabling token mounts using ``automountServiceAccountToken`` (#32808) +- Add support for defining custom priority classes (#31615) +- Add support for ``runtimeClassName`` (#31868) +- Add support for custom query in workers KEDA trigger (#32308) + +Improvements +^^^^^^^^^^^^ + +- Add ``containerSecurityContext`` for cleanup job (#34351) +- Add existing secret support for PGBouncer metrics exporter (#32724) +- Allow templating in webserver ingress hostnames (#33142) +- Allow templating in flower ingress hostnames (#33363) +- Add configmap annotations to StatsD and webserver (#33340) +- Add pod security context to PgBouncer (#32662) +- Add an option to use a direct DB connection in KEDA when PgBouncer is enabled (#32608) +- Allow templating in cleanup.schedule (#32570) +- Template dag processor ``waitformigration`` containers ``extraVolumeMounts`` (#32100) +- Ability to inject extra containers into PgBouncer (#33686) +- Allowing ability to add custom env into PgBouncer container (#33438) +- Add support for env variables in the StatsD container (#33175) + +Bug Fixes +^^^^^^^^^ + +- Add ``airflow db migrate`` command to database migration job (#34178) +- Pass ``workers.terminationGracePeriodSeconds`` into KubeExecutor pod template (#33514) +- CeleryExecutor namespace depends on Airflow version (#32753) +- Fix dag processor not including webserver config volume (#32644) +- Dag processor liveness probe include ``--local`` and ``--job-type`` args (#32426) +- Revising flower_url_prefix considering default value (#33134) + +Doc only changes +^^^^^^^^^^^^^^^^ + +- Add more explicit "embedded postgres" exclusion for production (#33034) +- Update git-sync description (#32181) + +Misc +^^^^ + +- Default Airflow version to 2.7.1 (#34186) +- Update PostgreSQL subchart to 12.10.0 (#33747) +- Update git-sync to 3.6.9 (#33748) +- Remove unnecessary loops to load env from helm values (#33506) +- Replace ``common.tplvalues.render`` with ``tpl`` in ingress template files (#33384) +- Remove K8S 1.23 support (#32899) +- Fix chart named template comments (#32681) +- Remove outdated comment from chart values in the workers KEDA conf section (#32300) +- Remove unnecessary ``or`` function in template files (#34415) + +Airflow Helm Chart 1.10.0 (2023-06-26) +-------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +Default Airflow image is updated to ``2.6.2`` (#31979) +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``2.6.2``, previously it was ``2.5.3``. + +New Features +^^^^^^^^^^^^ + +- Add support for container security context (#31043) + +Improvements +^^^^^^^^^^^^ + +- Validate ``executor`` and ``config.core.executor`` match (#30693) +- Support ``minAvailable`` property for PodDisruptionBudget (#30603) +- Add ``volumeMounts`` to dag processor ``waitForMigrations`` (#30990) +- Template extra volumes (#30773) + +Bug Fixes +^^^^^^^^^ + +- Fix webserver probes timeout and period (#30609) +- Add missing ``waitForMigrations`` for workers (#31625) +- Add missing ``priorityClassName`` to K8S worker pod template (#31328) +- Adding log groomer sidecar to dag processor (#30726) +- Do not propagate global security context to statsd and redis (#31865) + +Misc +^^^^ + +- Default Airflow version to 2.6.2 (#31979) +- Use template comments for the chart license header (#30569) +- Align ``apiVersion`` and ``kind`` order in chart templates (#31850) +- Cleanup Kubernetes < 1.23 support (#31847) + +Airflow Helm Chart 1.9.0 (2023-04-14) +------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +Default PgBouncer and PgBouncer Exporter images have been updated (#29919) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The PgBouncer and PgBouncer Exporter images are based on newer software/os. They are also multi-platform AMD/ARM images: + + * ``pgbouncer``: 1.16.1 based on alpine 3.14 (``airflow-pgbouncer-2023.02.24-1.16.1``) + * ``pgbouncer-exporter``: 0.14.0 based on alpine 3.17 (``apache/airflow:airflow-pgbouncer-exporter-2023.02.21-0.14.0``) + +Default Airflow image is updated to ``2.5.3`` (#30411) +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``2.5.3``, previously it was ``2.5.1``. + +New Features +^^^^^^^^^^^^ + +- Add support for ``hostAliases`` for Airflow webserver and scheduler (#30051) +- Add support for annotations on StatsD Deployment and cleanup CronJob (#30126) +- Add support for annotations in logs PVC (#29270) +- Add support for annotations in extra ConfigMap and Secrets (#30303) +- Add support for pod annotations to PgBouncer (#30168) +- Add support for ``ttlSecondsAfterFinished`` on ``migrateDatabaseJob`` and ``createUserJob`` (#29314) +- Add support for using SHA digest of Docker images (#30214) + +Improvements +^^^^^^^^^^^^ + +- Template extra volumes in Helm Chart (#29357) +- Make Liveness/Readiness Probe timeouts configurable for PgBouncer Exporter (#29752) +- Enable individual trigger logging (#29482) + +Bug Fixes +^^^^^^^^^ + +- Add ``config.kubernetes_executor`` to values (#29818) +- Block extra properties in image config (#30217) +- Remove replicas if KEDA is enabled (#29838) +- Mount ``kerberos.keytab`` to worker when enabled (#29526) +- Fix adding annotations for dag persistence PVC (#29622) +- Fix ``bitnami/postgresql`` default username and password (#29478) +- Add global volumes in pod template file (#29295) +- Add log groomer sidecar to triggerer service (#29392) +- Helm deployment fails when ``postgresql.nameOverride`` is used (#29214) + +Doc only changes +^^^^^^^^^^^^^^^^ + +- Add gitSync optional env description (#29378) +- Add webserver NodePort example (#29460) +- Include Rancher in Helm chart install instructions (#28416) +- Change RSA SSH host key to reflect update from Github (#30286) + +Misc +^^^^ + +- Update Airflow version to 2.5.3 (#30411) +- Switch to newer versions of PgBouncer and PgBouncer Exporter in chart (#29919) +- Reformat chart templates (#29917) +- Reformat chart templates part 2 (#29941) +- Reformat chart templates part 3 (#30312) +- Replace deprecated k8s registry references (#29938) +- Fix ``airflow_dags_mount`` formatting (#29296) +- Fix ``webserver.service.ports`` formatting (#29297) + +Airflow Helm Chart 1.8.0 (2023-02-06) +------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +``bitnami/postgresql`` subchart updated to ``12.1.9`` (#29071) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The version of postgresql installed is still version 11. + +If you are upgrading an existing helm release with the built-in postgres database, you will either need to delete your release and reinstall fresh, or manually delete these 2 objects: + +.. code-block:: + + kubectl delete secret {RELEASE_NAME}-postgresql + kubectl delete statefulset {RELEASE_NAME}-postgresql + +As a reminder, it is recommended to `set up an external database `_ in production. + +This version of the chart uses different variable names for setting usernames and passwords in the postgres database. + +- ``postgresql.auth.enablePostgresUser`` is used to determine if the "postgres" admin account will be created. +- ``postgresql.auth.postgresPassword`` sets the password for the "postgres" user. +- ``postgresql.auth.username`` and ``postrgesql.auth.password`` are used to set credentials for a non-admin account if desired. +- ``postgresql.postgresqlUsername`` and ``postgresql.postresqlPassword``, which were used in the previous version of the chart, are no longer used. + +Users will need to make those changes in their values files if they are changing the Postgres configuration. + +Previously the subchart version was ``10.5.3``. + +Default ``dags.gitSync.wait`` reduced to ``5`` seconds (#27625) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default for ``dags.gitSync.wait`` has been reduced from ``60`` seconds to ``5`` seconds to reduce the likelihood of DAGs +becoming inconsistent between Airflow components. This will, however, increase traffic to the remote git repository. + +Default Airflow image is updated to ``2.5.1`` (#29074) +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``2.5.1``, previously it was ``2.4.1``. + +Default git-sync image is updated to ``3.6.3`` (#27848) +""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default git-sync image that is used with the Chart is now ``3.6.3``, previously it was ``3.4.0``. + +Default redis image is updated to ``7-bullseye`` (#27443) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default redis image that is used with the Chart is now ``7-bullseye``, previously it was ``6-bullseye``. + +New Features +^^^^^^^^^^^^ + +- Add annotations on deployments (#28688) +- Add global volume & volumeMounts to the chart (#27781) + +Improvements +^^^^^^^^^^^^ + +- Add support for ``webserverConfigConfigMapName`` (#27419) +- Enhance chart to allow overriding command-line args to statsd exporter (#28041) +- Add support for NodePort in Services (#26945) +- Add worker log-groomer-sidecar enable option (#27178) +- Add HostAliases to Pod template file (#27544) +- Allow PgBouncer replicas to be configurable (#27439) + +Bug Fixes +^^^^^^^^^ + +- Create scheduler service to serve task logs for LocalKubernetesExecutor (#28828) +- Fix NOTES.txt to show correct URL (#28264) +- Add worker service account for LocalKubernetesExecutor (#28813) +- Remove checks for 1.19 api checks (#28461) +- Add airflow_local_settings to all airflow containers (#27779) +- Make custom env vars optional for job templates (#27148) +- Decrease default gitSync wait (#27625) +- Add ``extraVolumeMounts`` to sidecars too (#27420) +- Fix PgBouncer after PostgreSQL subchart upgrade (#29207) + +Doc only changes +^^^^^^^^^^^^^^^^ + +- Enhance production guide with a few Argo specific guidelines (#29078) +- Add doc note about Pod template images (#29032) +- Update production guide db section (#28610) +- Fix to LoadBalancer snippet (#28014) +- Fix gitSync example code (#28083) +- Correct repo example for cloning via ssh (#27671) + +Misc +^^^^ + +- Update Airflow version to 2.5.1 (#29074) +- Update git-sync to 3.6.3 (#27848) +- Upgrade ``bitnami/postgresql`` subchart to 12.1.9 (#29071) +- Update redis to 7 (#27443) +- Replace helm chart icon (#27704) + +Airflow Helm Chart 1.7.0 (2022-10-14) +------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +Default Airflow image is updated to ``2.4.1`` (#26485) +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``2.4.1``, previously it was ``2.3.2``. + +New Features +^^^^^^^^^^^^ + +- Make cleanup job history configurable (#26838) +- Added labels to specific Airflow components (#25031) +- Add StatsD ``overrideMappings`` in Helm chart values (#26598) +- Adding ``podAnnotations`` to StatsD deployment template (#25732) +- Container specific extra environment variables (#24784) +- Custom labels for extra Secrets and ConfigMaps (#25283) +- Add ``revisionHistoryLimit`` to all deployments (#25059) +- Adding ``podAnnotations`` to Redis StatefulSet (#23708) +- Provision Standalone Dag Processor (#23711) +- Add configurable scheme for webserver probes (#22815) +- Add support for KEDA HPA config to Helm chart (#24220) + +Improvements +^^^^^^^^^^^^ + +- Add 'executor' label to Airflow scheduler deployment (#25684) +- Add default ``flower_url_prefix`` in Helm chart values (#26415) +- Add liveness probe to Celery workers (#25561) +- Use ``sql_alchemy_conn`` for celery result backend when ``result_backend`` is not set (#24496) + +Bug Fixes +^^^^^^^^^ + +- Fix pod template ``imagePullPolicy`` (#26423) +- Do not declare a volume for ``sshKeySecret`` if dag persistence is enabled (#22913) +- Pass worker annotations to generated pod template (#24647) +- Fix semver compare number for ``jobs check`` command (#24480) +- Use ``--local`` flag for liveness probes in Airflow 2.5+ (#24999) + +Doc only changes +^^^^^^^^^^^^^^^^ + +- Improve documentation on helm hooks disabling (#26747) +- Remove ``ssh://`` prefix from git repo value (#26632) +- Fix ``defaultAirflowRepository`` comment (#26428) +- Baking DAGs into Docker image (#26401) +- Reload pods when using the same DAG tag (#24576) +- Minor clarifications about ``result_backend``, dag processor, and ``helm uninstall`` (#24929) +- Add hyperlinks to GitHub PRs for Release Notes (#24532) +- Terraform should not use Helm hooks for starting jobs (#26604) +- Flux should not use Helm hooks for starting jobs (#24288) +- Provide details on how to pull Airflow image from a private repository (#24394) +- Helm logo no longer a link (#23977) +- Document LocalKubernetesExecutor support in chart (#23876) +- Update Production Guide (#23836) + +Misc +^^^^ + +- Default Airflow version to 2.4.1 (#26485) +- Vendor in the Bitnami chart (#24395) +- Remove kubernetes 1.20 support (#25871) + + +Airflow Helm Chart 1.6.0 (2022-05-20) +------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +Default Airflow image is updated to ``2.3.0`` (#23386) +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``2.3.0``, previously it was ``2.2.4``. + +``ingress.enabled`` is deprecated +""""""""""""""""""""""""""""""""" + +Instead of having a single flag to control ingress resources for both the webserver and flower, there +are now separate flags to control them individually, ``ingress.web.enabled`` and ``ingress.flower.enabled``. +``ingress.enabled`` is now deprecated, but will still continue to control them both. + +Flower disabled by default +"""""""""""""""""""""""""" + +Flower is no longer enabled by default when using CeleryExecutor. If you'd like to deploy it, set +``flower.enabled`` to true in your values file. + +New Features +^^^^^^^^^^^^ + +- Support ``annotations`` on ``volumeClaimTemplates`` (#23433) +- Add support for ``topologySpreadConstraints`` to Helm Chart (#22712) +- Helm support for LocalKubernetesExecutor (#22388) +- Add ``securityContext`` config for Redis to Helm chart (#22182) +- Allow ``annotations`` on Helm DAG PVC (#22261) +- enable optional ``subPath`` for DAGs volume mount (#22323) +- Added support to override ``auth_type`` in ``auth_file`` in PgBouncer Helm configuration (#21999) +- Add ``extraVolumeMounts`` to Flower (#22414) +- Add webserver ``PodDisruptionBudget`` (#21735) + +Improvements +^^^^^^^^^^^^ + +- Ensure the messages from migration job show up early (#23479) +- Allow migration jobs and init containers to be optional (#22195) +- Use jobs check command for liveness probe check in Airflow 2 (#22143) + +Doc only changes +^^^^^^^^^^^^^^^^ + +- Adds ``resultBackendSecretName`` warning in Helm production docs (#23307) + +Misc +^^^^ + +- Update default Airflow version to ``2.3.0`` (#23386) +- Move the database configuration to a new section (#22284) +- Disable flower in chart by default (#23737) + + +Airflow Helm Chart 1.5.0, (2022-03-07) +-------------------------------------- + +Significant changes +^^^^^^^^^^^^^^^^^^^ + +Default Airflow image is updated to ``2.2.4`` +""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``2.2.4``, previously it was ``2.2.3``. + +Removed ``config.api`` +"""""""""""""""""""""" + +This section configured the authentication backend for the Airflow API but used the same values as the Airflow default setting, which made it unnecessary to +declare the same again. + +New Features +^^^^^^^^^^^^ + +- Add support for custom command and args in jobs (#20864) +- Support for ``priorityClassName`` (#20794) +- Add ``envFrom`` to the Flower deployment (#21401) +- Add annotations to cleanup pods (#21484) + +Improvements +^^^^^^^^^^^^ + +- Speedup liveness probe for scheduler and triggerer (#20833, #21108) +- Update git-sync to v3.4.0 (#21309) +- Remove default auth backend setting (#21640) + +Bug Fixes +^^^^^^^^^ + +- Fix elasticsearch URL when username/password are empty (#21222) +- Mount ``airflow.cfg`` in wait-for-airflow-migrations containers (#20609) +- Grant pod log reader to triggerer ServiceAccount (#21111) + +Doc only changes +^^^^^^^^^^^^^^^^ + +- Simplify chart docs for configuring Airflow (#21747) +- Add extra information about time synchronization needed (#21685) +- Fix extra containers docs (#20787) + +Misc +^^^^ + +- Use ``2.2.4`` as default Airflow version (#21745) +- Change Redis image to bullseye (#21875) + +Airflow Helm Chart 1.4.0, (2022-01-10) +-------------------------------------- + +Significant changes +^^^^^^^^^^^^^^^^^^^ + +Default Airflow image is updated to ``2.2.3`` +""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``2.2.3``, previously it was ``2.2.1``. + +``ingress.web.hosts`` and ``ingress.flower.hosts`` parameters data type has changed and ``ingress.web.tls`` and ``ingress.flower.tls`` have moved +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +``ingress.web.hosts`` and ``ingress.flower.hosts`` have had their types have been changed from an array of strings to an array of objects. ``ingress.web.tls`` and ``ingress.flower.tls`` can now be specified per host in ``ingress.web.hosts`` and ``ingress.flower.hosts`` respectively. + +The old parameter names will continue to work, however support for them will be removed in a future release so please update your values file. + +Fixed precedence of ``nodeSelector``, ``affinity`` and ``tolerations`` params +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +``nodeSelector``, ``affinity`` and ``tolerations`` params precedence has been fixed on all components. Now component-specific params +(e.g. ``webserver.affinity``) takes precedence over the global param (e.g. ``affinity``). + +Default ``KubernetesExecutor`` worker affinity removed +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Previously a default affinity was added to ``KubernetesExecutor`` workers to spread the workers out across nodes. This default affinity is no +longer set because, in general, there is no reason to spread task-specific workers across nodes. + +Changes in webserver and flower ``NetworkPolicy`` default ports +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The defaults for ``webserver.networkPolicy.ingress.ports`` and ``flower.networkPolicy.ingress.ports`` moved away from using named ports to numerical ports to avoid issues with OpenShift. + +Increase default ``livenessProbe`` ``timeoutSeconds`` for scheduler and triggerer +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The default timeout for the scheduler and triggerer ``livenessProbe`` has been increased from 10 seconds to 20 seconds. + +New Features +^^^^^^^^^^^^ + +- Add ``type`` to extra secrets param (#20599) +- Support elasticsearch connection ``scheme`` (#20564) +- Allows to disable built-in secret variables individually (#18974) +- Add support for ``securityContext`` (#18249) +- Add extra containers, volumes and volume mounts for jobs (#18808) +- Allow ingress multiple hostnames w/diff secrets (#18542) +- PgBouncer extra volumes, volume mounts, and ``sslmode`` (#19749) +- Allow specifying kerberos keytab (#19054) +- Allow disabling the Helm hooks (#18776, #20018) +- Add ``migration-wait-timeout`` (#20069) + +Improvements +^^^^^^^^^^^^ + +- Increase default ``livenessProbe`` timeout (#20698) +- Strict schema for k8s objects for values.yaml (#19181) +- Remove unnecessary ``pod_template_file`` defaults (#19690) +- Use built-in ``check-migrations`` command for Airflow>=2 (#19676) + +Bug Fixes +^^^^^^^^^ + +- Fix precedence of ``affinity``, ``nodeSelector``, and ``tolerations`` (#20641) +- Fix chart elasticsearch default port 80 to 9200. (#20616) +- Fix network policy issue for webserver and flower ui (#20199) +- Use local definitions for k8s schema validation (#20544) +- Add custom labels for ingresses/PVCs (#20535) +- Fix extra secrets/configmaps labels (#20464) +- Fix flower restarts on update (#20316) +- Properly quote namespace names (#20266) + +Doc only changes +^^^^^^^^^^^^^^^^ + +- Add ``helm dependency update`` step to chart INSTALL (#20702) +- Reword section covering the envvar secrets (#20566) +- Add "Customizing Workers" page (#20331) +- Include Datadog example in production guide (#17996) +- Update production Helm guide database section to use k8s secret (#19892) +- Fix ``multiNamespaceMode`` docs to also cover KPO (#19879) +- Clarify Helm behaviour when it comes to loading default connections (#19708) + +Misc +^^^^ + +- Use ``2.2.3`` as default Airflow version (#20450) +- Add ArtifactHUB annotations for docs and screenshots (#20558) +- Add kubernetes 1.21 support (#19557) + +Airflow Helm Chart 1.3.0 (2021-11-08) +------------------------------------- + +Significant changes +^^^^^^^^^^^^^^^^^^^ + +Default Airflow image is updated to ``2.2.1`` +""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow image that is used with the Chart is now ``2.2.1`` (which is Python ``3.7``), previously it was ``2.1.4`` (which is Python ``3.6``). + +The triggerer component requires Python ``3.7``. If you require Python ``3.6`` and Airflow ``2.2.0`` or later, use a ``3.6`` based image and set ``triggerer.enabled=False`` in your values. + +Resources made configurable for ``airflow-run-airflow-migrations`` job +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Now it's possible to set resources requests and limits for migration job through ``migrateDatabaseJob.resources`` value. + +New Features +^^^^^^^^^^^^ + +- Chart: Add resources for ``cleanup`` and ``createuser`` jobs (#19263) +- Chart: Add labels to jobs created by cleanup pods (#19225) +- Add migration job resources (#19175) +- Allow custom pod annotations to all components (#18481) +- Chart: Make PgBouncer cmd/args configurable (#18910) +- Chart: Use python 3.7 by default; support disabling triggerer (#18920) + +Improvements +^^^^^^^^^^^^ + +- Chart: Increase default liveness probe timeout (#19003) +- Chart: Mount DAGs in triggerer (#18753) + +Bug Fixes +^^^^^^^^^ + +- Allow Airflow UI to create worker pod via Clear > Run (#18272) +- Allow Airflow standard images to run in OpenShift utilizing the official Helm chart #18136 (#18147) + +Doc only changes +^^^^^^^^^^^^^^^^ + +- Chart: Fix ``extraEnvFrom`` examples (#19144) +- Chart docs: Update webserver secret key reference configuration (#18595) +- Fix helm chart links in source install guide (#18588) + +Misc +^^^^ + +- Chart: Update default Airflow version to ``2.2.1`` (#19326) +- Modernize dockerfiles builds (#19327) +- Chart: Use strict k8s schemas for template validation (#19379) + +Airflow Helm Chart 1.2.0 (2021-09-28) +------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +``ingress.web.host`` and ``ingress.flower.host`` parameters have been renamed and data type changed +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +``ingress.web.host`` and ``ingress.flower.host`` parameters have been renamed to ``ingress.web.hosts`` and ``ingress.flower.hosts``, respectively. Their types have been changed from a string to an array of strings. + +The old parameter names will continue to work, however support for them will be removed in a future release so please update your values file. + +Default Airflow version is updated to ``2.1.4`` +""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow version that is installed with the Chart is now ``2.1.4``, previously it was ``2.1.2``. + +Removed ``ingress.flower.precedingPaths`` and ``ingress.flower.succeedingPaths`` parameters +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +``ingress.flower.precedingPaths`` and ``ingress.flower.succeedingPaths`` parameters have been removed as they had previously had no effect on rendered YAML output. + +Change of default ``path`` on Ingress +""""""""""""""""""""""""""""""""""""" + +With the move to support the stable Kubernetes Ingress API the default path has been changed from being unset to ``/``. For most Ingress controllers this should not change the behavior of the resulting Ingress resource. + +New Features +^^^^^^^^^^^^ + +- Add Triggerer to Helm Chart (#17743) +- Chart: warn when webserver secret key isn't set (#18306) +- add ``extraContainers`` for ``migrateDatabaseJob`` (#18379) +- Labels on job templates (#18403) +- Chart: Allow running and waiting for DB Migrations using default image (#18218) +- Chart: Make cleanup cronjob cmd/args configurable (#17970) +- Chart: configurable number of retention days for log groomers (#17764) +- Chart: Add ``loadBalancerSourceRanges`` in webserver and flower services (#17666) +- Chart: Support ``extraContainers`` in k8s workers (#17562) + + +Improvements +^^^^^^^^^^^^ + +- Switch to latest version of PGBouncer-Exporter (#18429) +- Chart: Ability to access http k8s via multiple hostnames (#18257) +- Chart: Use stable API versions where available (#17211) +- Chart: Allow ``podTemplate`` to be templated (#17560) + +Bug Fixes +^^^^^^^^^ + +- Chart: Fix applying ``labels`` on Triggerer (#18299) +- Fixes warm shutdown for celery worker. (#18068) +- Chart: Fix minor Triggerer issues (#18105) +- Chart: fix webserver secret key update (#18079) +- Chart: fix running with ``uid`` ``0`` (#17688) +- Chart: use ServiceAccount template for log reader RoleBinding (#17645) +- Chart: Fix elasticsearch-secret template port default function (#17428) +- KEDA task count query should ignore k8s queue (#17433) + +Doc only changes +^^^^^^^^^^^^^^^^ + +- Chart Doc: Delete extra space in adding connections doc (#18424) +- Improves installing from sources pages for all components (#18251) +- Chart docs: Format ``loadBalancerSourceRanges`` using code-block (#17763) +- Doc: Fix a broken link in an ssh-related warning message (#17294) +- Chart: Add instructions to Update Helm Repo before upgrade (#17282) +- Chart docs: better note for logs existing PVC permissions (#17177) + +Misc +^^^^ + +- Chart: Update the default Airflow version to ``2.1.4`` (#18354) + +Airflow Helm Chart 1.1.0 (2021-07-26) +------------------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +Run ``helm repo update`` before upgrading the chart to the latest version. + +Default Airflow version is updated to ``2.1.2`` +""""""""""""""""""""""""""""""""""""""""""""""" + +The default Airflow version that is installed with the Chart is now ``2.1.2``, previously it was ``2.0.2``. + +Helm 2 no longer supported +"""""""""""""""""""""""""" + +This chart has dropped support for `Helm 2 as it has been deprecated `__ and no longer receiving security updates since November 2020. + +``webserver.extraNetworkPolicies`` and ``flower.extraNetworkPolicies`` parameters have been renamed +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +``webserver.extraNetworkPolicies`` and ``flower.extraNetworkPolicies`` have been renamed to ``webserver.networkPolicy.ingress.from`` and ``flower.networkPolicy.ingress.from``, respectively. Their values and behavior are the same. + +The old parameter names will continue to work, however support for them will be removed in a future release so please update your values file. + +Removed ``dags.gitSync.root``, ``dags.gitSync.dest``, and ``dags.gitSync.excludeWebserver`` parameters +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The ``dags.gitSync.root`` and ``dags.gitSync.dest`` parameters did not provide any useful behaviors to chart users so they have been removed. +If you have them set in your values file you can safely remove them. + +The ``dags.gitSync.excludeWebserver`` parameter was mistakenly included in the charts ``values.schema.json``. If you have it set in your values file, +you can safely remove it. + +``nodeSelector``, ``affinity`` and ``tolerations`` on ``migrateDatabaseJob`` and ``createUserJob`` jobs +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The ``migrateDatabaseJob`` and ``createUserJob`` jobs were incorrectly using the ``webserver``'s ``nodeSelector``, ``affinity`` +and ``tolerations`` (if set). Each job is now configured separately. + +New Features +^^^^^^^^^^^^ + +- Chart: Allow using ``krb5.conf`` with ``CeleryExecutor`` (#16822) +- Chart: Refactor webserver and flower NetworkPolicy (#16619) +- Chart: Apply worker's node assigning settings to Pod Template File (#16663) +- Chart: Support for overriding webserver and flower service ports (#16572) +- Chart: Support ``extraContainers`` and ``extraVolumes`` in flower (#16515) +- Chart: Allow configuration of pod resources in helm chart (#16425) +- Chart: Support job level annotations; fix jobs scheduling config (#16331) +- feat: Helm chart adding ``minReplicaCount`` to the KEDA ``worker-kedaautoscaler.yaml`` (#16262) +- Chart: Adds support for custom command and args (#16153) +- Chart: Add extra ini config to ``pgbouncer`` (#16120) +- Chart: Add ``extraInitContainers`` to scheduler/webserver/workers (#16098) +- Configurable resources for git-sync sidecar (#16080) +- Chart: Template ``airflowLocalSettings`` and ``webserver.webserverConfig`` (#16074) +- Support ``strategy``/``updateStrategy`` on scheduler (#16069) +- Chart: Add both airflow and extra annotations to jobs (#16058) +- ``loadBalancerIP`` and ``annotations`` for both Flower and Webserver (#15972) + +Improvements +^^^^^^^^^^^^ + +- Chart: Update Postgres subchart to 10.5.3 (#17041) +- Chart: Update the default Airflow version to ``2.1.2`` (#17013) +- Update default image as ``2.1.1`` for Helm Chart (#16785) +- Chart: warn when using default logging with ``KubernetesExecutor`` (#16784) +- Drop support for Helm 2 (#16575) +- Chart: ``podAntiAffinity`` for scheduler, webserver, and workers (#16315) +- Chart: Update the default Airflow Version to ``2.1.0`` (#16273) +- Chart: Only mount DAGs in webserver when required (#16229) +- Chart: Remove ``git-sync``: ``root`` and ``dest`` params (#15955) +- Chart: Add warning about missing ``knownHosts`` (#15950) + +Bug Fixes +^^^^^^^^^ + +- Chart: Create a random secret for Webserver's flask secret key (#17142) +- Chart: fix labels on cleanup ServiceAccount (#16722) +- Chart: Fix overriding node assigning settings on Worker Deployment (#16670) +- Chart: Always deploy a ``gitsync`` init container (#16339) +- Chart: Fix updating from ``KubernetesExecutor`` to ``CeleryExecutor`` (#16242) +- Chart: Adds labels to Kubernetes worker pods (#16203) +- Chart: Allow ``webserver.base_url`` to be templated (#16126) +- Chart: Fix ``PgBouncer`` exporter sidecar (#16099) +- Remove ``dags.gitSync.excludeWebserver`` from chart ``values.schema.json`` (#16070) +- Chart: Fix Elasticsearch secret created without Elasticsearch enabled (#16015) +- Handle special characters in passwords for Helm Chart (#16004) +- Fix flower ServiceAccount created without flower enable (#16011) +- Chart: ``gitsync`` Clean Up for ``KubernetesExecutor`` (#15925) +- Mount DAGs read only when using ``gitsync`` (#15953) + +Doc only changes +^^^^^^^^^^^^^^^^ + +- Chart docs: note uid write permissions for existing PVC (#17170) +- Chart Docs: Add single-line description for ``multiNamespaceMode`` (#17147) +- Chart: Update description for Helm chart to include 'official' (#17040) +- Chart: Better comment and example for ``podTemplate`` (#16859) +- Chart: Add more clear docs for setting ``pod_template_file.yaml`` (#16632) +- Fix description on ``scheduler.livenessprobe.periodSeconds`` (#16486) +- Chart docs: Fix ``extrasecrets`` example (#16305) +- Small improvements for ``README.md`` files (#16244) + +Misc +^^^^ + +- Removes pylint from our toolchain (#16682) +- Update link to match what is in pre-commit (#16408) +- Chart: Update the ``appVersion`` to 2.1.0 in ``Chart.yaml`` (#16337) +- Rename the main branch of the Airflow repo to be ``main`` (#16149) +- Update Chart version to ``1.1.0-rc1`` (#16124) diff --git a/charts/airflow/charts/postgresql/.helmignore b/charts/airflow/charts/postgresql/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/charts/airflow/charts/postgresql/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/airflow/charts/postgresql/Chart.lock b/charts/airflow/charts/postgresql/Chart.lock new file mode 100644 index 0000000..35f80ca --- /dev/null +++ b/charts/airflow/charts/postgresql/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: oci://registry-1.docker.io/bitnamicharts + version: 2.13.3 +digest: sha256:9a971689db0c66ea95ac2e911c05014c2b96c6077c991131ff84f2982f88fb83 +generated: "2023-11-03T20:45:06.276989379Z" diff --git a/charts/airflow/charts/postgresql/Chart.yaml b/charts/airflow/charts/postgresql/Chart.yaml new file mode 100644 index 0000000..c569db8 --- /dev/null +++ b/charts/airflow/charts/postgresql/Chart.yaml @@ -0,0 +1,37 @@ +annotations: + category: Database + images: | + - name: os-shell + image: docker.io/bitnami/os-shell:11-debian-11-r91 + - name: postgres-exporter + image: docker.io/bitnami/postgres-exporter:0.15.0-debian-11-r2 + - name: postgresql + image: docker.io/bitnami/postgresql:16.1.0-debian-11-r15 + licenses: Apache-2.0 +apiVersion: v2 +appVersion: 16.1.0 +dependencies: +- name: common + repository: oci://registry-1.docker.io/bitnamicharts + tags: + - bitnami-common + version: 2.x.x +description: PostgreSQL (Postgres) is an open source object-relational database known + for reliability and data integrity. ACID-compliant, it supports foreign keys, joins, + views, triggers and stored procedures. +home: https://bitnami.com +icon: https://bitnami.com/assets/stacks/postgresql/img/postgresql-stack-220x234.png +keywords: +- postgresql +- postgres +- database +- sql +- replication +- cluster +maintainers: +- name: VMware, Inc. + url: https://github.com/bitnami/charts +name: postgresql +sources: +- https://github.com/bitnami/charts/tree/main/bitnami/postgresql +version: 13.2.24 diff --git a/charts/airflow/charts/postgresql/README.md b/charts/airflow/charts/postgresql/README.md new file mode 100644 index 0000000..5348b1e --- /dev/null +++ b/charts/airflow/charts/postgresql/README.md @@ -0,0 +1,755 @@ + + +# Bitnami package for PostgreSQL + +PostgreSQL (Postgres) is an open source object-relational database known for reliability and data integrity. ACID-compliant, it supports foreign keys, joins, views, triggers and stored procedures. + +[Overview of PostgreSQL](http://www.postgresql.org) + +Trademarks: This software listing is packaged by Bitnami. The respective trademarks mentioned in the offering are owned by the respective companies, and use of them does not imply any affiliation or endorsement. + +## TL;DR + +```console +helm install my-release oci://registry-1.docker.io/bitnamicharts/postgresql +``` + +Looking to use PostgreSQL in production? Try [VMware Tanzu Application Catalog](https://bitnami.com/enterprise), the enterprise edition of Bitnami Application Catalog. + +## Introduction + +This chart bootstraps a [PostgreSQL](https://github.com/bitnami/containers/tree/main/bitnami/postgresql) deployment on a [Kubernetes](https://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +For HA, please see [this repo](https://github.com/bitnami/charts/tree/main/bitnami/postgresql-ha) + +Bitnami charts can be used with [Kubeapps](https://kubeapps.dev/) for deployment and management of Helm Charts in clusters. + +## Prerequisites + +- Kubernetes 1.23+ +- Helm 3.8.0+ +- PV provisioner support in the underlying infrastructure + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm install my-release oci://REGISTRY_NAME/REPOSITORY_NAME/postgresql +``` + +> Note: You need to substitute the placeholders `REGISTRY_NAME` and `REPOSITORY_NAME` with a reference to your Helm chart registry and repository. For example, in the case of Bitnami, you need to use `REGISTRY_NAME=registry-1.docker.io` and `REPOSITORY_NAME=bitnamicharts`. + +The command deploys PostgreSQL on the Kubernetes cluster in the default configuration. The [Parameters](#parameters) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components but PVC's associated with the chart and deletes the release. + +To delete the PVC's associated with `my-release`: + +```console +kubectl delete pvc -l release=my-release +``` + +> **Note**: Deleting the PVC's will delete postgresql data as well. Please be cautious before doing it. + +## Parameters + +### Global parameters + +| Name | Description | Value | +| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | +| `global.imageRegistry` | Global Docker image registry | `""` | +| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` | +| `global.storageClass` | Global StorageClass for Persistent Volume(s) | `""` | +| `global.postgresql.auth.postgresPassword` | Password for the "postgres" admin user (overrides `auth.postgresPassword`) | `""` | +| `global.postgresql.auth.username` | Name for a custom user to create (overrides `auth.username`) | `""` | +| `global.postgresql.auth.password` | Password for the custom user to create (overrides `auth.password`) | `""` | +| `global.postgresql.auth.database` | Name for a custom database to create (overrides `auth.database`) | `""` | +| `global.postgresql.auth.existingSecret` | Name of existing secret to use for PostgreSQL credentials (overrides `auth.existingSecret`). | `""` | +| `global.postgresql.auth.secretKeys.adminPasswordKey` | Name of key in existing secret to use for PostgreSQL credentials (overrides `auth.secretKeys.adminPasswordKey`). Only used when `global.postgresql.auth.existingSecret` is set. | `""` | +| `global.postgresql.auth.secretKeys.userPasswordKey` | Name of key in existing secret to use for PostgreSQL credentials (overrides `auth.secretKeys.userPasswordKey`). Only used when `global.postgresql.auth.existingSecret` is set. | `""` | +| `global.postgresql.auth.secretKeys.replicationPasswordKey` | Name of key in existing secret to use for PostgreSQL credentials (overrides `auth.secretKeys.replicationPasswordKey`). Only used when `global.postgresql.auth.existingSecret` is set. | `""` | +| `global.postgresql.service.ports.postgresql` | PostgreSQL service port (overrides `service.ports.postgresql`) | `""` | + +### Common parameters + +| Name | Description | Value | +| ------------------------ | -------------------------------------------------------------------------------------------- | --------------- | +| `kubeVersion` | Override Kubernetes version | `""` | +| `nameOverride` | String to partially override common.names.fullname template (will maintain the release name) | `""` | +| `fullnameOverride` | String to fully override common.names.fullname template | `""` | +| `clusterDomain` | Kubernetes Cluster Domain | `cluster.local` | +| `extraDeploy` | Array of extra objects to deploy with the release (evaluated as a template) | `[]` | +| `commonLabels` | Add labels to all the deployed resources | `{}` | +| `commonAnnotations` | Add annotations to all the deployed resources | `{}` | +| `diagnosticMode.enabled` | Enable diagnostic mode (all probes will be disabled and the command will be overridden) | `false` | +| `diagnosticMode.command` | Command to override all containers in the statefulset | `["sleep"]` | +| `diagnosticMode.args` | Args to override all containers in the statefulset | `["infinity"]` | + +### PostgreSQL common parameters + +| Name | Description | Value | +| ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | +| `image.registry` | PostgreSQL image registry | `REGISTRY_NAME` | +| `image.repository` | PostgreSQL image repository | `REPOSITORY_NAME/postgresql` | +| `image.digest` | PostgreSQL image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag | `""` | +| `image.pullPolicy` | PostgreSQL image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Specify image pull secrets | `[]` | +| `image.debug` | Specify if debug values should be set | `false` | +| `auth.enablePostgresUser` | Assign a password to the "postgres" admin user. Otherwise, remote access will be blocked for this user | `true` | +| `auth.postgresPassword` | Password for the "postgres" admin user. Ignored if `auth.existingSecret` is provided | `""` | +| `auth.username` | Name for a custom user to create | `""` | +| `auth.password` | Password for the custom user to create. Ignored if `auth.existingSecret` is provided | `""` | +| `auth.database` | Name for a custom database to create | `""` | +| `auth.replicationUsername` | Name of the replication user | `repl_user` | +| `auth.replicationPassword` | Password for the replication user. Ignored if `auth.existingSecret` is provided | `""` | +| `auth.existingSecret` | Name of existing secret to use for PostgreSQL credentials. `auth.postgresPassword`, `auth.password`, and `auth.replicationPassword` will be ignored and picked up from this secret. The secret might also contains the key `ldap-password` if LDAP is enabled. `ldap.bind_password` will be ignored and picked from this secret in this case. | `""` | +| `auth.secretKeys.adminPasswordKey` | Name of key in existing secret to use for PostgreSQL credentials. Only used when `auth.existingSecret` is set. | `postgres-password` | +| `auth.secretKeys.userPasswordKey` | Name of key in existing secret to use for PostgreSQL credentials. Only used when `auth.existingSecret` is set. | `password` | +| `auth.secretKeys.replicationPasswordKey` | Name of key in existing secret to use for PostgreSQL credentials. Only used when `auth.existingSecret` is set. | `replication-password` | +| `auth.usePasswordFiles` | Mount credentials as a files instead of using an environment variable | `false` | +| `architecture` | PostgreSQL architecture (`standalone` or `replication`) | `standalone` | +| `replication.synchronousCommit` | Set synchronous commit mode. Allowed values: `on`, `remote_apply`, `remote_write`, `local` and `off` | `off` | +| `replication.numSynchronousReplicas` | Number of replicas that will have synchronous replication. Note: Cannot be greater than `readReplicas.replicaCount`. | `0` | +| `replication.applicationName` | Cluster application name. Useful for advanced replication settings | `my_application` | +| `containerPorts.postgresql` | PostgreSQL container port | `5432` | +| `audit.logHostname` | Log client hostnames | `false` | +| `audit.logConnections` | Add client log-in operations to the log file | `false` | +| `audit.logDisconnections` | Add client log-outs operations to the log file | `false` | +| `audit.pgAuditLog` | Add operations to log using the pgAudit extension | `""` | +| `audit.pgAuditLogCatalog` | Log catalog using pgAudit | `off` | +| `audit.clientMinMessages` | Message log level to share with the user | `error` | +| `audit.logLinePrefix` | Template for log line prefix (default if not set) | `""` | +| `audit.logTimezone` | Timezone for the log timestamps | `""` | +| `ldap.enabled` | Enable LDAP support | `false` | +| `ldap.server` | IP address or name of the LDAP server. | `""` | +| `ldap.port` | Port number on the LDAP server to connect to | `""` | +| `ldap.prefix` | String to prepend to the user name when forming the DN to bind | `""` | +| `ldap.suffix` | String to append to the user name when forming the DN to bind | `""` | +| `ldap.basedn` | Root DN to begin the search for the user in | `""` | +| `ldap.binddn` | DN of user to bind to LDAP | `""` | +| `ldap.bindpw` | Password for the user to bind to LDAP | `""` | +| `ldap.searchAttribute` | Attribute to match against the user name in the search | `""` | +| `ldap.searchFilter` | The search filter to use when doing search+bind authentication | `""` | +| `ldap.scheme` | Set to `ldaps` to use LDAPS | `""` | +| `ldap.tls.enabled` | Se to true to enable TLS encryption | `false` | +| `ldap.uri` | LDAP URL beginning in the form `ldap[s]://host[:port]/basedn`. If provided, all the other LDAP parameters will be ignored. | `""` | +| `postgresqlDataDir` | PostgreSQL data dir folder | `/bitnami/postgresql/data` | +| `postgresqlSharedPreloadLibraries` | Shared preload libraries (comma-separated list) | `pgaudit` | +| `shmVolume.enabled` | Enable emptyDir volume for /dev/shm for PostgreSQL pod(s) | `true` | +| `shmVolume.sizeLimit` | Set this to enable a size limit on the shm tmpfs | `""` | +| `tls.enabled` | Enable TLS traffic support | `false` | +| `tls.autoGenerated` | Generate automatically self-signed TLS certificates | `false` | +| `tls.preferServerCiphers` | Whether to use the server's TLS cipher preferences rather than the client's | `true` | +| `tls.certificatesSecret` | Name of an existing secret that contains the certificates | `""` | +| `tls.certFilename` | Certificate filename | `""` | +| `tls.certKeyFilename` | Certificate key filename | `""` | +| `tls.certCAFilename` | CA Certificate filename | `""` | +| `tls.crlFilename` | File containing a Certificate Revocation List | `""` | + +### PostgreSQL Primary parameters + +| Name | Description | Value | +| ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | --------------------- | +| `primary.name` | Name of the primary database (eg primary, master, leader, ...) | `primary` | +| `primary.configuration` | PostgreSQL Primary main configuration to be injected as ConfigMap | `""` | +| `primary.pgHbaConfiguration` | PostgreSQL Primary client authentication configuration | `""` | +| `primary.existingConfigmap` | Name of an existing ConfigMap with PostgreSQL Primary configuration | `""` | +| `primary.extendedConfiguration` | Extended PostgreSQL Primary configuration (appended to main or default configuration) | `""` | +| `primary.existingExtendedConfigmap` | Name of an existing ConfigMap with PostgreSQL Primary extended configuration | `""` | +| `primary.initdb.args` | PostgreSQL initdb extra arguments | `""` | +| `primary.initdb.postgresqlWalDir` | Specify a custom location for the PostgreSQL transaction log | `""` | +| `primary.initdb.scripts` | Dictionary of initdb scripts | `{}` | +| `primary.initdb.scriptsConfigMap` | ConfigMap with scripts to be run at first boot | `""` | +| `primary.initdb.scriptsSecret` | Secret with scripts to be run at first boot (in case it contains sensitive information) | `""` | +| `primary.initdb.user` | Specify the PostgreSQL username to execute the initdb scripts | `""` | +| `primary.initdb.password` | Specify the PostgreSQL password to execute the initdb scripts | `""` | +| `primary.standby.enabled` | Whether to enable current cluster's primary as standby server of another cluster or not | `false` | +| `primary.standby.primaryHost` | The Host of replication primary in the other cluster | `""` | +| `primary.standby.primaryPort` | The Port of replication primary in the other cluster | `""` | +| `primary.extraEnvVars` | Array with extra environment variables to add to PostgreSQL Primary nodes | `[]` | +| `primary.extraEnvVarsCM` | Name of existing ConfigMap containing extra env vars for PostgreSQL Primary nodes | `""` | +| `primary.extraEnvVarsSecret` | Name of existing Secret containing extra env vars for PostgreSQL Primary nodes | `""` | +| `primary.command` | Override default container command (useful when using custom images) | `[]` | +| `primary.args` | Override default container args (useful when using custom images) | `[]` | +| `primary.livenessProbe.enabled` | Enable livenessProbe on PostgreSQL Primary containers | `true` | +| `primary.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `30` | +| `primary.livenessProbe.periodSeconds` | Period seconds for livenessProbe | `10` | +| `primary.livenessProbe.timeoutSeconds` | Timeout seconds for livenessProbe | `5` | +| `primary.livenessProbe.failureThreshold` | Failure threshold for livenessProbe | `6` | +| `primary.livenessProbe.successThreshold` | Success threshold for livenessProbe | `1` | +| `primary.readinessProbe.enabled` | Enable readinessProbe on PostgreSQL Primary containers | `true` | +| `primary.readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | `5` | +| `primary.readinessProbe.periodSeconds` | Period seconds for readinessProbe | `10` | +| `primary.readinessProbe.timeoutSeconds` | Timeout seconds for readinessProbe | `5` | +| `primary.readinessProbe.failureThreshold` | Failure threshold for readinessProbe | `6` | +| `primary.readinessProbe.successThreshold` | Success threshold for readinessProbe | `1` | +| `primary.startupProbe.enabled` | Enable startupProbe on PostgreSQL Primary containers | `false` | +| `primary.startupProbe.initialDelaySeconds` | Initial delay seconds for startupProbe | `30` | +| `primary.startupProbe.periodSeconds` | Period seconds for startupProbe | `10` | +| `primary.startupProbe.timeoutSeconds` | Timeout seconds for startupProbe | `1` | +| `primary.startupProbe.failureThreshold` | Failure threshold for startupProbe | `15` | +| `primary.startupProbe.successThreshold` | Success threshold for startupProbe | `1` | +| `primary.customLivenessProbe` | Custom livenessProbe that overrides the default one | `{}` | +| `primary.customReadinessProbe` | Custom readinessProbe that overrides the default one | `{}` | +| `primary.customStartupProbe` | Custom startupProbe that overrides the default one | `{}` | +| `primary.lifecycleHooks` | for the PostgreSQL Primary container to automate configuration before or after startup | `{}` | +| `primary.resources.limits` | The resources limits for the PostgreSQL Primary containers | `{}` | +| `primary.resources.requests.memory` | The requested memory for the PostgreSQL Primary containers | `256Mi` | +| `primary.resources.requests.cpu` | The requested cpu for the PostgreSQL Primary containers | `250m` | +| `primary.podSecurityContext.enabled` | Enable security context | `true` | +| `primary.podSecurityContext.fsGroup` | Group ID for the pod | `1001` | +| `primary.containerSecurityContext.enabled` | Enabled containers' Security Context | `true` | +| `primary.containerSecurityContext.runAsUser` | Set containers' Security Context runAsUser | `1001` | +| `primary.containerSecurityContext.runAsNonRoot` | Set container's Security Context runAsNonRoot | `true` | +| `primary.containerSecurityContext.privileged` | Set container's Security Context privileged | `false` | +| `primary.containerSecurityContext.readOnlyRootFilesystem` | Set container's Security Context readOnlyRootFilesystem | `false` | +| `primary.containerSecurityContext.allowPrivilegeEscalation` | Set container's Security Context allowPrivilegeEscalation | `false` | +| `primary.containerSecurityContext.capabilities.drop` | List of capabilities to be dropped | `["ALL"]` | +| `primary.containerSecurityContext.seccompProfile.type` | Set container's Security Context seccomp profile | `RuntimeDefault` | +| `primary.hostAliases` | PostgreSQL primary pods host aliases | `[]` | +| `primary.hostNetwork` | Specify if host network should be enabled for PostgreSQL pod (postgresql primary) | `false` | +| `primary.hostIPC` | Specify if host IPC should be enabled for PostgreSQL pod (postgresql primary) | `false` | +| `primary.labels` | Map of labels to add to the statefulset (postgresql primary) | `{}` | +| `primary.annotations` | Annotations for PostgreSQL primary pods | `{}` | +| `primary.podLabels` | Map of labels to add to the pods (postgresql primary) | `{}` | +| `primary.podAnnotations` | Map of annotations to add to the pods (postgresql primary) | `{}` | +| `primary.podAffinityPreset` | PostgreSQL primary pod affinity preset. Ignored if `primary.affinity` is set. Allowed values: `soft` or `hard` | `""` | +| `primary.podAntiAffinityPreset` | PostgreSQL primary pod anti-affinity preset. Ignored if `primary.affinity` is set. Allowed values: `soft` or `hard` | `soft` | +| `primary.nodeAffinityPreset.type` | PostgreSQL primary node affinity preset type. Ignored if `primary.affinity` is set. Allowed values: `soft` or `hard` | `""` | +| `primary.nodeAffinityPreset.key` | PostgreSQL primary node label key to match Ignored if `primary.affinity` is set. | `""` | +| `primary.nodeAffinityPreset.values` | PostgreSQL primary node label values to match. Ignored if `primary.affinity` is set. | `[]` | +| `primary.affinity` | Affinity for PostgreSQL primary pods assignment | `{}` | +| `primary.nodeSelector` | Node labels for PostgreSQL primary pods assignment | `{}` | +| `primary.tolerations` | Tolerations for PostgreSQL primary pods assignment | `[]` | +| `primary.topologySpreadConstraints` | Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template | `[]` | +| `primary.priorityClassName` | Priority Class to use for each pod (postgresql primary) | `""` | +| `primary.schedulerName` | Use an alternate scheduler, e.g. "stork". | `""` | +| `primary.terminationGracePeriodSeconds` | Seconds PostgreSQL primary pod needs to terminate gracefully | `""` | +| `primary.updateStrategy.type` | PostgreSQL Primary statefulset strategy type | `RollingUpdate` | +| `primary.updateStrategy.rollingUpdate` | PostgreSQL Primary statefulset rolling update configuration parameters | `{}` | +| `primary.extraVolumeMounts` | Optionally specify extra list of additional volumeMounts for the PostgreSQL Primary container(s) | `[]` | +| `primary.extraVolumes` | Optionally specify extra list of additional volumes for the PostgreSQL Primary pod(s) | `[]` | +| `primary.sidecars` | Add additional sidecar containers to the PostgreSQL Primary pod(s) | `[]` | +| `primary.initContainers` | Add additional init containers to the PostgreSQL Primary pod(s) | `[]` | +| `primary.extraPodSpec` | Optionally specify extra PodSpec for the PostgreSQL Primary pod(s) | `{}` | +| `primary.service.type` | Kubernetes Service type | `ClusterIP` | +| `primary.service.ports.postgresql` | PostgreSQL service port | `5432` | +| `primary.service.nodePorts.postgresql` | Node port for PostgreSQL | `""` | +| `primary.service.clusterIP` | Static clusterIP or None for headless services | `""` | +| `primary.service.annotations` | Annotations for PostgreSQL primary service | `{}` | +| `primary.service.loadBalancerIP` | Load balancer IP if service type is `LoadBalancer` | `""` | +| `primary.service.externalTrafficPolicy` | Enable client source IP preservation | `Cluster` | +| `primary.service.loadBalancerSourceRanges` | Addresses that are allowed when service is LoadBalancer | `[]` | +| `primary.service.extraPorts` | Extra ports to expose in the PostgreSQL primary service | `[]` | +| `primary.service.sessionAffinity` | Session Affinity for Kubernetes service, can be "None" or "ClientIP" | `None` | +| `primary.service.sessionAffinityConfig` | Additional settings for the sessionAffinity | `{}` | +| `primary.service.headless.annotations` | Additional custom annotations for headless PostgreSQL primary service | `{}` | +| `primary.persistence.enabled` | Enable PostgreSQL Primary data persistence using PVC | `true` | +| `primary.persistence.existingClaim` | Name of an existing PVC to use | `""` | +| `primary.persistence.mountPath` | The path the volume will be mounted at | `/bitnami/postgresql` | +| `primary.persistence.subPath` | The subdirectory of the volume to mount to | `""` | +| `primary.persistence.storageClass` | PVC Storage Class for PostgreSQL Primary data volume | `""` | +| `primary.persistence.accessModes` | PVC Access Mode for PostgreSQL volume | `["ReadWriteOnce"]` | +| `primary.persistence.size` | PVC Storage Request for PostgreSQL volume | `8Gi` | +| `primary.persistence.annotations` | Annotations for the PVC | `{}` | +| `primary.persistence.labels` | Labels for the PVC | `{}` | +| `primary.persistence.selector` | Selector to match an existing Persistent Volume (this value is evaluated as a template) | `{}` | +| `primary.persistence.dataSource` | Custom PVC data source | `{}` | +| `primary.persistentVolumeClaimRetentionPolicy.enabled` | Enable Persistent volume retention policy for Primary Statefulset | `false` | +| `primary.persistentVolumeClaimRetentionPolicy.whenScaled` | Volume retention behavior when the replica count of the StatefulSet is reduced | `Retain` | +| `primary.persistentVolumeClaimRetentionPolicy.whenDeleted` | Volume retention behavior that applies when the StatefulSet is deleted | `Retain` | + +### PostgreSQL read only replica parameters (only used when `architecture` is set to `replication`) + +| Name | Description | Value | +| ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | --------------------- | +| `readReplicas.name` | Name of the read replicas database (eg secondary, slave, ...) | `read` | +| `readReplicas.replicaCount` | Number of PostgreSQL read only replicas | `1` | +| `readReplicas.extendedConfiguration` | Extended PostgreSQL read only replicas configuration (appended to main or default configuration) | `""` | +| `readReplicas.extraEnvVars` | Array with extra environment variables to add to PostgreSQL read only nodes | `[]` | +| `readReplicas.extraEnvVarsCM` | Name of existing ConfigMap containing extra env vars for PostgreSQL read only nodes | `""` | +| `readReplicas.extraEnvVarsSecret` | Name of existing Secret containing extra env vars for PostgreSQL read only nodes | `""` | +| `readReplicas.command` | Override default container command (useful when using custom images) | `[]` | +| `readReplicas.args` | Override default container args (useful when using custom images) | `[]` | +| `readReplicas.livenessProbe.enabled` | Enable livenessProbe on PostgreSQL read only containers | `true` | +| `readReplicas.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `30` | +| `readReplicas.livenessProbe.periodSeconds` | Period seconds for livenessProbe | `10` | +| `readReplicas.livenessProbe.timeoutSeconds` | Timeout seconds for livenessProbe | `5` | +| `readReplicas.livenessProbe.failureThreshold` | Failure threshold for livenessProbe | `6` | +| `readReplicas.livenessProbe.successThreshold` | Success threshold for livenessProbe | `1` | +| `readReplicas.readinessProbe.enabled` | Enable readinessProbe on PostgreSQL read only containers | `true` | +| `readReplicas.readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | `5` | +| `readReplicas.readinessProbe.periodSeconds` | Period seconds for readinessProbe | `10` | +| `readReplicas.readinessProbe.timeoutSeconds` | Timeout seconds for readinessProbe | `5` | +| `readReplicas.readinessProbe.failureThreshold` | Failure threshold for readinessProbe | `6` | +| `readReplicas.readinessProbe.successThreshold` | Success threshold for readinessProbe | `1` | +| `readReplicas.startupProbe.enabled` | Enable startupProbe on PostgreSQL read only containers | `false` | +| `readReplicas.startupProbe.initialDelaySeconds` | Initial delay seconds for startupProbe | `30` | +| `readReplicas.startupProbe.periodSeconds` | Period seconds for startupProbe | `10` | +| `readReplicas.startupProbe.timeoutSeconds` | Timeout seconds for startupProbe | `1` | +| `readReplicas.startupProbe.failureThreshold` | Failure threshold for startupProbe | `15` | +| `readReplicas.startupProbe.successThreshold` | Success threshold for startupProbe | `1` | +| `readReplicas.customLivenessProbe` | Custom livenessProbe that overrides the default one | `{}` | +| `readReplicas.customReadinessProbe` | Custom readinessProbe that overrides the default one | `{}` | +| `readReplicas.customStartupProbe` | Custom startupProbe that overrides the default one | `{}` | +| `readReplicas.lifecycleHooks` | for the PostgreSQL read only container to automate configuration before or after startup | `{}` | +| `readReplicas.resources.limits` | The resources limits for the PostgreSQL read only containers | `{}` | +| `readReplicas.resources.requests.memory` | The requested memory for the PostgreSQL read only containers | `256Mi` | +| `readReplicas.resources.requests.cpu` | The requested cpu for the PostgreSQL read only containers | `250m` | +| `readReplicas.podSecurityContext.enabled` | Enable security context | `true` | +| `readReplicas.podSecurityContext.fsGroup` | Group ID for the pod | `1001` | +| `readReplicas.containerSecurityContext.enabled` | Enabled containers' Security Context | `true` | +| `readReplicas.containerSecurityContext.runAsUser` | Set containers' Security Context runAsUser | `1001` | +| `readReplicas.containerSecurityContext.runAsNonRoot` | Set container's Security Context runAsNonRoot | `true` | +| `readReplicas.containerSecurityContext.privileged` | Set container's Security Context privileged | `false` | +| `readReplicas.containerSecurityContext.readOnlyRootFilesystem` | Set container's Security Context readOnlyRootFilesystem | `false` | +| `readReplicas.containerSecurityContext.allowPrivilegeEscalation` | Set container's Security Context allowPrivilegeEscalation | `false` | +| `readReplicas.containerSecurityContext.capabilities.drop` | List of capabilities to be dropped | `["ALL"]` | +| `readReplicas.containerSecurityContext.seccompProfile.type` | Set container's Security Context seccomp profile | `RuntimeDefault` | +| `readReplicas.hostAliases` | PostgreSQL read only pods host aliases | `[]` | +| `readReplicas.hostNetwork` | Specify if host network should be enabled for PostgreSQL pod (PostgreSQL read only) | `false` | +| `readReplicas.hostIPC` | Specify if host IPC should be enabled for PostgreSQL pod (postgresql primary) | `false` | +| `readReplicas.labels` | Map of labels to add to the statefulset (PostgreSQL read only) | `{}` | +| `readReplicas.annotations` | Annotations for PostgreSQL read only pods | `{}` | +| `readReplicas.podLabels` | Map of labels to add to the pods (PostgreSQL read only) | `{}` | +| `readReplicas.podAnnotations` | Map of annotations to add to the pods (PostgreSQL read only) | `{}` | +| `readReplicas.podAffinityPreset` | PostgreSQL read only pod affinity preset. Ignored if `primary.affinity` is set. Allowed values: `soft` or `hard` | `""` | +| `readReplicas.podAntiAffinityPreset` | PostgreSQL read only pod anti-affinity preset. Ignored if `primary.affinity` is set. Allowed values: `soft` or `hard` | `soft` | +| `readReplicas.nodeAffinityPreset.type` | PostgreSQL read only node affinity preset type. Ignored if `primary.affinity` is set. Allowed values: `soft` or `hard` | `""` | +| `readReplicas.nodeAffinityPreset.key` | PostgreSQL read only node label key to match Ignored if `primary.affinity` is set. | `""` | +| `readReplicas.nodeAffinityPreset.values` | PostgreSQL read only node label values to match. Ignored if `primary.affinity` is set. | `[]` | +| `readReplicas.affinity` | Affinity for PostgreSQL read only pods assignment | `{}` | +| `readReplicas.nodeSelector` | Node labels for PostgreSQL read only pods assignment | `{}` | +| `readReplicas.tolerations` | Tolerations for PostgreSQL read only pods assignment | `[]` | +| `readReplicas.topologySpreadConstraints` | Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template | `[]` | +| `readReplicas.priorityClassName` | Priority Class to use for each pod (PostgreSQL read only) | `""` | +| `readReplicas.schedulerName` | Use an alternate scheduler, e.g. "stork". | `""` | +| `readReplicas.terminationGracePeriodSeconds` | Seconds PostgreSQL read only pod needs to terminate gracefully | `""` | +| `readReplicas.updateStrategy.type` | PostgreSQL read only statefulset strategy type | `RollingUpdate` | +| `readReplicas.updateStrategy.rollingUpdate` | PostgreSQL read only statefulset rolling update configuration parameters | `{}` | +| `readReplicas.extraVolumeMounts` | Optionally specify extra list of additional volumeMounts for the PostgreSQL read only container(s) | `[]` | +| `readReplicas.extraVolumes` | Optionally specify extra list of additional volumes for the PostgreSQL read only pod(s) | `[]` | +| `readReplicas.sidecars` | Add additional sidecar containers to the PostgreSQL read only pod(s) | `[]` | +| `readReplicas.initContainers` | Add additional init containers to the PostgreSQL read only pod(s) | `[]` | +| `readReplicas.extraPodSpec` | Optionally specify extra PodSpec for the PostgreSQL read only pod(s) | `{}` | +| `readReplicas.service.type` | Kubernetes Service type | `ClusterIP` | +| `readReplicas.service.ports.postgresql` | PostgreSQL service port | `5432` | +| `readReplicas.service.nodePorts.postgresql` | Node port for PostgreSQL | `""` | +| `readReplicas.service.clusterIP` | Static clusterIP or None for headless services | `""` | +| `readReplicas.service.annotations` | Annotations for PostgreSQL read only service | `{}` | +| `readReplicas.service.loadBalancerIP` | Load balancer IP if service type is `LoadBalancer` | `""` | +| `readReplicas.service.externalTrafficPolicy` | Enable client source IP preservation | `Cluster` | +| `readReplicas.service.loadBalancerSourceRanges` | Addresses that are allowed when service is LoadBalancer | `[]` | +| `readReplicas.service.extraPorts` | Extra ports to expose in the PostgreSQL read only service | `[]` | +| `readReplicas.service.sessionAffinity` | Session Affinity for Kubernetes service, can be "None" or "ClientIP" | `None` | +| `readReplicas.service.sessionAffinityConfig` | Additional settings for the sessionAffinity | `{}` | +| `readReplicas.service.headless.annotations` | Additional custom annotations for headless PostgreSQL read only service | `{}` | +| `readReplicas.persistence.enabled` | Enable PostgreSQL read only data persistence using PVC | `true` | +| `readReplicas.persistence.existingClaim` | Name of an existing PVC to use | `""` | +| `readReplicas.persistence.mountPath` | The path the volume will be mounted at | `/bitnami/postgresql` | +| `readReplicas.persistence.subPath` | The subdirectory of the volume to mount to | `""` | +| `readReplicas.persistence.storageClass` | PVC Storage Class for PostgreSQL read only data volume | `""` | +| `readReplicas.persistence.accessModes` | PVC Access Mode for PostgreSQL volume | `["ReadWriteOnce"]` | +| `readReplicas.persistence.size` | PVC Storage Request for PostgreSQL volume | `8Gi` | +| `readReplicas.persistence.annotations` | Annotations for the PVC | `{}` | +| `readReplicas.persistence.labels` | Labels for the PVC | `{}` | +| `readReplicas.persistence.selector` | Selector to match an existing Persistent Volume (this value is evaluated as a template) | `{}` | +| `readReplicas.persistence.dataSource` | Custom PVC data source | `{}` | +| `readReplicas.persistentVolumeClaimRetentionPolicy.enabled` | Enable Persistent volume retention policy for read only Statefulset | `false` | +| `readReplicas.persistentVolumeClaimRetentionPolicy.whenScaled` | Volume retention behavior when the replica count of the StatefulSet is reduced | `Retain` | +| `readReplicas.persistentVolumeClaimRetentionPolicy.whenDeleted` | Volume retention behavior that applies when the StatefulSet is deleted | `Retain` | + +### Backup parameters + +| Name | Description | Value | +| ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `backup.enabled` | Enable the logical dump of the database "regularly" | `false` | +| `backup.cronjob.schedule` | Set the cronjob parameter schedule | `@daily` | +| `backup.cronjob.timeZone` | Set the cronjob parameter timeZone | `""` | +| `backup.cronjob.concurrencyPolicy` | Set the cronjob parameter concurrencyPolicy | `Allow` | +| `backup.cronjob.failedJobsHistoryLimit` | Set the cronjob parameter failedJobsHistoryLimit | `1` | +| `backup.cronjob.successfulJobsHistoryLimit` | Set the cronjob parameter successfulJobsHistoryLimit | `3` | +| `backup.cronjob.startingDeadlineSeconds` | Set the cronjob parameter startingDeadlineSeconds | `""` | +| `backup.cronjob.ttlSecondsAfterFinished` | Set the cronjob parameter ttlSecondsAfterFinished | `""` | +| `backup.cronjob.restartPolicy` | Set the cronjob parameter restartPolicy | `OnFailure` | +| `backup.cronjob.podSecurityContext.enabled` | Enable PodSecurityContext for CronJob/Backup | `true` | +| `backup.cronjob.podSecurityContext.fsGroup` | Group ID for the CronJob | `1001` | +| `backup.cronjob.containerSecurityContext.enabled` | Enabled containers' Security Context | `true` | +| `backup.cronjob.containerSecurityContext.runAsUser` | Set containers' Security Context runAsUser | `1001` | +| `backup.cronjob.containerSecurityContext.runAsNonRoot` | Set container's Security Context runAsNonRoot | `true` | +| `backup.cronjob.containerSecurityContext.privileged` | Set container's Security Context privileged | `false` | +| `backup.cronjob.containerSecurityContext.readOnlyRootFilesystem` | Set container's Security Context readOnlyRootFilesystem | `false` | +| `backup.cronjob.containerSecurityContext.allowPrivilegeEscalation` | Set container's Security Context allowPrivilegeEscalation | `false` | +| `backup.cronjob.containerSecurityContext.capabilities.drop` | List of capabilities to be dropped | `["ALL"]` | +| `backup.cronjob.containerSecurityContext.seccompProfile.type` | Set container's Security Context seccomp profile | `RuntimeDefault` | +| `backup.cronjob.command` | Set backup container's command to run | `["/bin/sh","-c","pg_dumpall --clean --if-exists --load-via-partition-root --quote-all-identifiers --no-password --file=${PGDUMP_DIR}/pg_dumpall-$(date '+%Y-%m-%d-%H-%M').pgdump"]` | +| `backup.cronjob.labels` | Set the cronjob labels | `{}` | +| `backup.cronjob.annotations` | Set the cronjob annotations | `{}` | +| `backup.cronjob.nodeSelector` | Node labels for PostgreSQL backup CronJob pod assignment | `{}` | +| `backup.cronjob.storage.existingClaim` | Provide an existing `PersistentVolumeClaim` (only when `architecture=standalone`) | `""` | +| `backup.cronjob.storage.resourcePolicy` | Setting it to "keep" to avoid removing PVCs during a helm delete operation. Leaving it empty will delete PVCs after the chart deleted | `""` | +| `backup.cronjob.storage.storageClass` | PVC Storage Class for the backup data volume | `""` | +| `backup.cronjob.storage.accessModes` | PV Access Mode | `["ReadWriteOnce"]` | +| `backup.cronjob.storage.size` | PVC Storage Request for the backup data volume | `8Gi` | +| `backup.cronjob.storage.annotations` | PVC annotations | `{}` | +| `backup.cronjob.storage.mountPath` | Path to mount the volume at | `/backup/pgdump` | +| `backup.cronjob.storage.subPath` | Subdirectory of the volume to mount at | `""` | +| `backup.cronjob.storage.volumeClaimTemplates.selector` | A label query over volumes to consider for binding (e.g. when using local volumes) | `{}` | + +### NetworkPolicy parameters + +| Name | Description | Value | +| ------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | +| `networkPolicy.enabled` | Enable network policies | `false` | +| `networkPolicy.metrics.enabled` | Enable network policies for metrics (prometheus) | `false` | +| `networkPolicy.metrics.namespaceSelector` | Monitoring namespace selector labels. These labels will be used to identify the prometheus' namespace. | `{}` | +| `networkPolicy.metrics.podSelector` | Monitoring pod selector labels. These labels will be used to identify the Prometheus pods. | `{}` | +| `networkPolicy.ingressRules.primaryAccessOnlyFrom.enabled` | Enable ingress rule that makes PostgreSQL primary node only accessible from a particular origin. | `false` | +| `networkPolicy.ingressRules.primaryAccessOnlyFrom.namespaceSelector` | Namespace selector label that is allowed to access the PostgreSQL primary node. This label will be used to identified the allowed namespace(s). | `{}` | +| `networkPolicy.ingressRules.primaryAccessOnlyFrom.podSelector` | Pods selector label that is allowed to access the PostgreSQL primary node. This label will be used to identified the allowed pod(s). | `{}` | +| `networkPolicy.ingressRules.primaryAccessOnlyFrom.customRules` | Custom network policy for the PostgreSQL primary node. | `[]` | +| `networkPolicy.ingressRules.readReplicasAccessOnlyFrom.enabled` | Enable ingress rule that makes PostgreSQL read-only nodes only accessible from a particular origin. | `false` | +| `networkPolicy.ingressRules.readReplicasAccessOnlyFrom.namespaceSelector` | Namespace selector label that is allowed to access the PostgreSQL read-only nodes. This label will be used to identified the allowed namespace(s). | `{}` | +| `networkPolicy.ingressRules.readReplicasAccessOnlyFrom.podSelector` | Pods selector label that is allowed to access the PostgreSQL read-only nodes. This label will be used to identified the allowed pod(s). | `{}` | +| `networkPolicy.ingressRules.readReplicasAccessOnlyFrom.customRules` | Custom network policy for the PostgreSQL read-only nodes. | `[]` | +| `networkPolicy.egressRules.denyConnectionsToExternal` | Enable egress rule that denies outgoing traffic outside the cluster, except for DNS (port 53). | `false` | +| `networkPolicy.egressRules.customRules` | Custom network policy rule | `[]` | + +### Volume Permissions parameters + +| Name | Description | Value | +| ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | +| `volumePermissions.enabled` | Enable init container that changes the owner and group of the persistent volume | `false` | +| `volumePermissions.image.registry` | Init container volume-permissions image registry | `REGISTRY_NAME` | +| `volumePermissions.image.repository` | Init container volume-permissions image repository | `REPOSITORY_NAME/os-shell` | +| `volumePermissions.image.digest` | Init container volume-permissions image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag | `""` | +| `volumePermissions.image.pullPolicy` | Init container volume-permissions image pull policy | `IfNotPresent` | +| `volumePermissions.image.pullSecrets` | Init container volume-permissions image pull secrets | `[]` | +| `volumePermissions.resources.limits` | Init container volume-permissions resource limits | `{}` | +| `volumePermissions.resources.requests` | Init container volume-permissions resource requests | `{}` | +| `volumePermissions.containerSecurityContext.runAsUser` | User ID for the init container | `0` | +| `volumePermissions.containerSecurityContext.runAsGroup` | Group ID for the init container | `0` | +| `volumePermissions.containerSecurityContext.runAsNonRoot` | runAsNonRoot for the init container | `false` | +| `volumePermissions.containerSecurityContext.seccompProfile.type` | seccompProfile.type for the init container | `RuntimeDefault` | + +### Other Parameters + +| Name | Description | Value | +| --------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ------- | +| `serviceBindings.enabled` | Create secret for service binding (Experimental) | `false` | +| `serviceAccount.create` | Enable creation of ServiceAccount for PostgreSQL pod | `false` | +| `serviceAccount.name` | The name of the ServiceAccount to use. | `""` | +| `serviceAccount.automountServiceAccountToken` | Allows auto mount of ServiceAccountToken on the serviceAccount created | `true` | +| `serviceAccount.annotations` | Additional custom annotations for the ServiceAccount | `{}` | +| `rbac.create` | Create Role and RoleBinding (required for PSP to work) | `false` | +| `rbac.rules` | Custom RBAC rules to set | `[]` | +| `psp.create` | Whether to create a PodSecurityPolicy. WARNING: PodSecurityPolicy is deprecated in Kubernetes v1.21 or later, unavailable in v1.25 or later | `false` | + +### Metrics Parameters + +| Name | Description | Value | +| ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | ----------------------------------- | +| `metrics.enabled` | Start a prometheus exporter | `false` | +| `metrics.image.registry` | PostgreSQL Prometheus Exporter image registry | `REGISTRY_NAME` | +| `metrics.image.repository` | PostgreSQL Prometheus Exporter image repository | `REPOSITORY_NAME/postgres-exporter` | +| `metrics.image.digest` | PostgreSQL image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag | `""` | +| `metrics.image.pullPolicy` | PostgreSQL Prometheus Exporter image pull policy | `IfNotPresent` | +| `metrics.image.pullSecrets` | Specify image pull secrets | `[]` | +| `metrics.collectors` | Control enabled collectors | `{}` | +| `metrics.customMetrics` | Define additional custom metrics | `{}` | +| `metrics.extraEnvVars` | Extra environment variables to add to PostgreSQL Prometheus exporter | `[]` | +| `metrics.containerSecurityContext.enabled` | Enabled containers' Security Context | `true` | +| `metrics.containerSecurityContext.runAsUser` | Set containers' Security Context runAsUser | `1001` | +| `metrics.containerSecurityContext.runAsNonRoot` | Set container's Security Context runAsNonRoot | `true` | +| `metrics.containerSecurityContext.privileged` | Set container's Security Context privileged | `false` | +| `metrics.containerSecurityContext.readOnlyRootFilesystem` | Set container's Security Context readOnlyRootFilesystem | `false` | +| `metrics.containerSecurityContext.allowPrivilegeEscalation` | Set container's Security Context allowPrivilegeEscalation | `false` | +| `metrics.containerSecurityContext.capabilities.drop` | List of capabilities to be dropped | `["ALL"]` | +| `metrics.containerSecurityContext.seccompProfile.type` | Set container's Security Context seccomp profile | `RuntimeDefault` | +| `metrics.livenessProbe.enabled` | Enable livenessProbe on PostgreSQL Prometheus exporter containers | `true` | +| `metrics.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `5` | +| `metrics.livenessProbe.periodSeconds` | Period seconds for livenessProbe | `10` | +| `metrics.livenessProbe.timeoutSeconds` | Timeout seconds for livenessProbe | `5` | +| `metrics.livenessProbe.failureThreshold` | Failure threshold for livenessProbe | `6` | +| `metrics.livenessProbe.successThreshold` | Success threshold for livenessProbe | `1` | +| `metrics.readinessProbe.enabled` | Enable readinessProbe on PostgreSQL Prometheus exporter containers | `true` | +| `metrics.readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | `5` | +| `metrics.readinessProbe.periodSeconds` | Period seconds for readinessProbe | `10` | +| `metrics.readinessProbe.timeoutSeconds` | Timeout seconds for readinessProbe | `5` | +| `metrics.readinessProbe.failureThreshold` | Failure threshold for readinessProbe | `6` | +| `metrics.readinessProbe.successThreshold` | Success threshold for readinessProbe | `1` | +| `metrics.startupProbe.enabled` | Enable startupProbe on PostgreSQL Prometheus exporter containers | `false` | +| `metrics.startupProbe.initialDelaySeconds` | Initial delay seconds for startupProbe | `10` | +| `metrics.startupProbe.periodSeconds` | Period seconds for startupProbe | `10` | +| `metrics.startupProbe.timeoutSeconds` | Timeout seconds for startupProbe | `1` | +| `metrics.startupProbe.failureThreshold` | Failure threshold for startupProbe | `15` | +| `metrics.startupProbe.successThreshold` | Success threshold for startupProbe | `1` | +| `metrics.customLivenessProbe` | Custom livenessProbe that overrides the default one | `{}` | +| `metrics.customReadinessProbe` | Custom readinessProbe that overrides the default one | `{}` | +| `metrics.customStartupProbe` | Custom startupProbe that overrides the default one | `{}` | +| `metrics.containerPorts.metrics` | PostgreSQL Prometheus exporter metrics container port | `9187` | +| `metrics.resources.limits` | The resources limits for the PostgreSQL Prometheus exporter container | `{}` | +| `metrics.resources.requests` | The requested resources for the PostgreSQL Prometheus exporter container | `{}` | +| `metrics.service.ports.metrics` | PostgreSQL Prometheus Exporter service port | `9187` | +| `metrics.service.clusterIP` | Static clusterIP or None for headless services | `""` | +| `metrics.service.sessionAffinity` | Control where client requests go, to the same pod or round-robin | `None` | +| `metrics.service.annotations` | Annotations for Prometheus to auto-discover the metrics endpoint | `{}` | +| `metrics.serviceMonitor.enabled` | Create ServiceMonitor Resource for scraping metrics using Prometheus Operator | `false` | +| `metrics.serviceMonitor.namespace` | Namespace for the ServiceMonitor Resource (defaults to the Release Namespace) | `""` | +| `metrics.serviceMonitor.interval` | Interval at which metrics should be scraped. | `""` | +| `metrics.serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `""` | +| `metrics.serviceMonitor.labels` | Additional labels that can be used so ServiceMonitor will be discovered by Prometheus | `{}` | +| `metrics.serviceMonitor.selector` | Prometheus instance selector labels | `{}` | +| `metrics.serviceMonitor.relabelings` | RelabelConfigs to apply to samples before scraping | `[]` | +| `metrics.serviceMonitor.metricRelabelings` | MetricRelabelConfigs to apply to samples before ingestion | `[]` | +| `metrics.serviceMonitor.honorLabels` | Specify honorLabels parameter to add the scrape endpoint | `false` | +| `metrics.serviceMonitor.jobLabel` | The name of the label on the target service to use as the job name in prometheus. | `""` | +| `metrics.prometheusRule.enabled` | Create a PrometheusRule for Prometheus Operator | `false` | +| `metrics.prometheusRule.namespace` | Namespace for the PrometheusRule Resource (defaults to the Release Namespace) | `""` | +| `metrics.prometheusRule.labels` | Additional labels that can be used so PrometheusRule will be discovered by Prometheus | `{}` | +| `metrics.prometheusRule.rules` | PrometheusRule definitions | `[]` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +helm install my-release \ + --set auth.postgresPassword=secretpassword + oci://REGISTRY_NAME/REPOSITORY_NAME/postgresql +``` + +> Note: You need to substitute the placeholders `REGISTRY_NAME` and `REPOSITORY_NAME` with a reference to your Helm chart registry and repository. For example, in the case of Bitnami, you need to use `REGISTRY_NAME=registry-1.docker.io` and `REPOSITORY_NAME=bitnamicharts`. + +The above command sets the PostgreSQL `postgres` account password to `secretpassword`. + +> NOTE: Once this chart is deployed, it is not possible to change the application's access credentials, such as usernames or passwords, using Helm. To change these application credentials after deployment, delete any persistent volumes (PVs) used by the chart and re-deploy it, or use the application's built-in administrative tools if available. +> **Warning** Setting a password will be ignored on new installation in case when previous PostgreSQL release was deleted through the helm command. In that case, old PVC will have an old password, and setting it through helm won't take effect. Deleting persistent volumes (PVs) will solve the issue. Refer to [issue 2061](https://github.com/bitnami/charts/issues/2061) for more details + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +```console +helm install my-release -f values.yaml oci://REGISTRY_NAME/REPOSITORY_NAME/postgresql +``` + +> Note: You need to substitute the placeholders `REGISTRY_NAME` and `REPOSITORY_NAME` with a reference to your Helm chart registry and repository. For example, in the case of Bitnami, you need to use `REGISTRY_NAME=registry-1.docker.io` and `REPOSITORY_NAME=bitnamicharts`. +> **Tip**: You can use the default [values.yaml](https://github.com/bitnami/charts/tree/main/bitnami/postgresql/values.yaml) + +## Configuration and installation details + +### [Rolling VS Immutable tags](https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/) + +It is strongly recommended to use immutable tags in a production environment. This ensures your deployment does not change automatically if the same tag is updated with a different image. + +Bitnami will release a new chart updating its containers if a new version of the main container, significant changes, or critical vulnerabilities exist. + +### Customizing primary and read replica services in a replicated configuration + +At the top level, there is a service object which defines the services for both primary and readReplicas. For deeper customization, there are service objects for both the primary and read types individually. This allows you to override the values in the top level service object so that the primary and read can be of different service types and with different clusterIPs / nodePorts. Also in the case you want the primary and read to be of type nodePort, you will need to set the nodePorts to different values to prevent a collision. The values that are deeper in the primary.service or readReplicas.service objects will take precedence over the top level service object. + +### Use a different PostgreSQL version + +To modify the application version used in this chart, specify a different version of the image using the `image.tag` parameter and/or a different repository using the `image.repository` parameter. Refer to the [chart documentation for more information on these parameters and how to use them with images from a private registry](https://docs.bitnami.com/kubernetes/infrastructure/postgresql/configuration/change-image-version/). + +### postgresql.conf / pg_hba.conf files as configMap + +This helm chart also supports to customize the PostgreSQL configuration file. You can add additional PostgreSQL configuration parameters using the `primary.extendedConfiguration`/`readReplicas.extendedConfiguration` parameters as a string. Alternatively, to replace the entire default configuration use `primary.configuration`. + +You can also add a custom pg_hba.conf using the `primary.pgHbaConfiguration` parameter. + +In addition to these options, you can also set an external ConfigMap with all the configuration files. This is done by setting the `primary.existingConfigmap` parameter. Note that this will override the two previous options. + +### Initialize a fresh instance + +The [Bitnami PostgreSQL](https://github.com/bitnami/containers/tree/main/bitnami/postgresql) image allows you to use your custom scripts to initialize a fresh instance. In order to execute the scripts, you can specify custom scripts using the `primary.initdb.scripts` parameter as a string. + +In addition, you can also set an external ConfigMap with all the initialization scripts. This is done by setting the `primary.initdb.scriptsConfigMap` parameter. Note that this will override the two previous options. If your initialization scripts contain sensitive information such as credentials or passwords, you can use the `primary.initdb.scriptsSecret` parameter. + +The allowed extensions are `.sh`, `.sql` and `.sql.gz`. + +### Securing traffic using TLS + +TLS support can be enabled in the chart by specifying the `tls.` parameters while creating a release. The following parameters should be configured to properly enable the TLS support in the chart: + +- `tls.enabled`: Enable TLS support. Defaults to `false` +- `tls.certificatesSecret`: Name of an existing secret that contains the certificates. No defaults. +- `tls.certFilename`: Certificate filename. No defaults. +- `tls.certKeyFilename`: Certificate key filename. No defaults. + +For example: + +- First, create the secret with the cetificates files: + + ```console + kubectl create secret generic certificates-tls-secret --from-file=./cert.crt --from-file=./cert.key --from-file=./ca.crt + ``` + +- Then, use the following parameters: + + ```console + volumePermissions.enabled=true + tls.enabled=true + tls.certificatesSecret="certificates-tls-secret" + tls.certFilename="cert.crt" + tls.certKeyFilename="cert.key" + ``` + + > Note TLS and VolumePermissions: PostgreSQL requires certain permissions on sensitive files (such as certificate keys) to start up. Due to an on-going [issue](https://github.com/kubernetes/kubernetes/issues/57923) regarding kubernetes permissions and the use of `containerSecurityContext.runAsUser`, you must enable `volumePermissions` to ensure everything works as expected. + +### Sidecars + +If you need additional containers to run within the same pod as PostgreSQL (e.g. an additional metrics or logging exporter), you can do so via the `sidecars` config parameter. Simply define your container according to the Kubernetes container spec. + +```yaml +# For the PostgreSQL primary +primary: + sidecars: + - name: your-image-name + image: your-image + imagePullPolicy: Always + ports: + - name: portname + containerPort: 1234 +# For the PostgreSQL replicas +readReplicas: + sidecars: + - name: your-image-name + image: your-image + imagePullPolicy: Always + ports: + - name: portname + containerPort: 1234 +``` + +### Metrics + +The chart optionally can start a metrics exporter for [prometheus](https://prometheus.io). The metrics endpoint (port 9187) is not exposed and it is expected that the metrics are collected from inside the k8s cluster using something similar as the described in the [example Prometheus scrape configuration](https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml). + +The exporter allows to create custom metrics from additional SQL queries. See the Chart's `values.yaml` for an example and consult the [exporters documentation](https://github.com/wrouesnel/postgres_exporter#adding-new-metrics-via-a-config-file) for more details. + +### Use of global variables + +In more complex scenarios, we may have the following tree of dependencies + +```text + +--------------+ + | | + +------------+ Chart 1 +-----------+ + | | | | + | --------+------+ | + | | | + | | | + | | | + | | | + v v v ++-------+------+ +--------+------+ +--------+------+ +| | | | | | +| PostgreSQL | | Sub-chart 1 | | Sub-chart 2 | +| | | | | | ++--------------+ +---------------+ +---------------+ +``` + +The three charts below depend on the parent chart Chart 1. However, subcharts 1 and 2 may need to connect to PostgreSQL as well. In order to do so, subcharts 1 and 2 need to know the PostgreSQL credentials, so one option for deploying could be deploy Chart 1 with the following parameters: + +```text +postgresql.auth.username=testuser +subchart1.postgresql.auth.username=testuser +subchart2.postgresql.auth.username=testuser +postgresql.auth.password=testpass +subchart1.postgresql.auth.password=testpass +subchart2.postgresql.auth.password=testpass +postgresql.auth.database=testdb +subchart1.postgresql.auth.database=testdb +subchart2.postgresql.auth.database=testdb +``` + +If the number of dependent sub-charts increases, installing the chart with parameters can become increasingly difficult. An alternative would be to set the credentials using global variables as follows: + +```text +global.postgresql.auth.username=testuser +global.postgresql.auth.password=testpass +global.postgresql.auth.database=testdb +``` + +This way, the credentials will be available in all of the subcharts. + +## Persistence + +The [Bitnami PostgreSQL](https://github.com/bitnami/containers/tree/main/bitnami/postgresql) image stores the PostgreSQL data and configurations at the `/bitnami/postgresql` path of the container. + +Persistent Volume Claims are used to keep the data across deployments. This is known to work in GCE, AWS, and minikube. +See the [Parameters](#parameters) section to configure the PVC or to disable persistence. + +If you already have data in it, you will fail to sync to standby nodes for all commits, details can refer to the [code present in the container repository](https://github.com/bitnami/containers/tree/main/bitnami/postgresql). If you need to use those data, please covert them to sql and import after `helm install` finished. + +## NetworkPolicy + +To enable network policy for PostgreSQL, install [a networking plugin that implements the Kubernetes NetworkPolicy spec](https://kubernetes.io/docs/tasks/administer-cluster/declare-network-policy#before-you-begin), and set `networkPolicy.enabled` to `true`. + +For Kubernetes v1.5 & v1.6, you must also turn on NetworkPolicy by setting the DefaultDeny namespace annotation. Note: this will enforce policy for _all_ pods in the namespace: + +```console +kubectl annotate namespace default "net.beta.kubernetes.io/network-policy={\"ingress\":{\"isolation\":\"DefaultDeny\"}}" +``` + +With NetworkPolicy enabled, traffic will be limited to just port 5432. + +For more precise policy, set `networkPolicy.allowExternal=false`. This will only allow pods with the generated client label to connect to PostgreSQL. +This label will be displayed in the output of a successful install. + +## Differences between Bitnami PostgreSQL image and [Docker Official](https://hub.docker.com/_/postgres) image + +- The Docker Official PostgreSQL image does not support replication. If you pass any replication environment variable, this would be ignored. The only environment variables supported by the Docker Official image are POSTGRES_USER, POSTGRES_DB, POSTGRES_PASSWORD, POSTGRES_INITDB_ARGS, POSTGRES_INITDB_WALDIR and PGDATA. All the remaining environment variables are specific to the Bitnami PostgreSQL image. +- The Bitnami PostgreSQL image is non-root by default. This requires that you run the pod with `securityContext` and updates the permissions of the volume with an `initContainer`. A key benefit of this configuration is that the pod follows security best practices and is prepared to run on Kubernetes distributions with hard security constraints like OpenShift. +- For OpenShift up to 4.10, let set the volume permissions, security context, runAsUser and fsGroup automatically by OpenShift and disable the predefined settings of the helm chart: primary.securityContext.enabled=false,primary.containerSecurityContext.enabled=false,volumePermissions.enabled=false,shmVolume.enabled=false +- For OpenShift 4.11 and higher, let set OpenShift the runAsUser and fsGroup automatically. Configure the pod and container security context to restrictive defaults and disable the volume permissions setup: primary. + podSecurityContext.fsGroup=null,primary.podSecurityContext.seccompProfile.type=RuntimeDefault,primary.containerSecurityContext.runAsUser=null,primary.containerSecurityContext.allowPrivilegeEscalation=false,primary.containerSecurityContext.runAsNonRoot=true,primary.containerSecurityContext.seccompProfile.type=RuntimeDefault,primary.containerSecurityContext.capabilities.drop=['ALL'],volumePermissions.enabled=false,shmVolume.enabled=false + +### Setting Pod's affinity + +This chart allows you to set your custom affinity using the `XXX.affinity` parameter(s). Find more information about Pod's affinity in the [kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). + +As an alternative, you can use of the preset configurations for pod affinity, pod anti-affinity, and node affinity available at the [bitnami/common](https://github.com/bitnami/charts/tree/main/bitnami/common#affinities) chart. To do so, set the `XXX.podAffinityPreset`, `XXX.podAntiAffinityPreset`, or `XXX.nodeAffinityPreset` parameters. + +## Troubleshooting + +Find more information about how to deal with common errors related to Bitnami's Helm charts in [this troubleshooting guide](https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues). + +## Upgrading + +### To 13.0.0 + +This major version changes the default PostgreSQL image from 15.x to 16.x. Follow the [official instructions](https://www.postgresql.org/docs/16/upgrading.html) to upgrade to 16.x. + +### To 12.0.0 + +This major version changes the default PostgreSQL image from 14.x to 15.x. Follow the [official instructions](https://www.postgresql.org/docs/15/upgrading.html) to upgrade to 15.x. + +### To any previous version + +Refer to the [chart documentation for more information about how to upgrade from previous releases](https://docs.bitnami.com/kubernetes/infrastructure/postgresql/administration/upgrade/). + +## License + +Copyright © 2023 VMware, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/charts/airflow/charts/postgresql/charts/common/.helmignore b/charts/airflow/charts/postgresql/charts/common/.helmignore new file mode 100644 index 0000000..50af031 --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/airflow/charts/postgresql/charts/common/Chart.yaml b/charts/airflow/charts/postgresql/charts/common/Chart.yaml new file mode 100644 index 0000000..40cd22d --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/Chart.yaml @@ -0,0 +1,23 @@ +annotations: + category: Infrastructure + licenses: Apache-2.0 +apiVersion: v2 +appVersion: 2.13.3 +description: A Library Helm Chart for grouping common logic between bitnami charts. + This chart is not deployable by itself. +home: https://bitnami.com +icon: https://bitnami.com/downloads/logos/bitnami-mark.png +keywords: +- common +- helper +- template +- function +- bitnami +maintainers: +- name: VMware, Inc. + url: https://github.com/bitnami/charts +name: common +sources: +- https://github.com/bitnami/charts +type: library +version: 2.13.3 diff --git a/charts/airflow/charts/postgresql/charts/common/README.md b/charts/airflow/charts/postgresql/charts/common/README.md new file mode 100644 index 0000000..80da4cc --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/README.md @@ -0,0 +1,235 @@ +# Bitnami Common Library Chart + +A [Helm Library Chart](https://helm.sh/docs/topics/library_charts/#helm) for grouping common logic between Bitnami charts. + +## TL;DR + +```yaml +dependencies: + - name: common + version: 2.x.x + repository: oci://registry-1.docker.io/bitnamicharts +``` + +```console +helm dependency update +``` + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.names.fullname" . }} +data: + myvalue: "Hello World" +``` + +## Introduction + +This chart provides a common template helpers which can be used to develop new charts using [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.dev/) for deployment and management of Helm Charts in clusters. + +Looking to use our applications in production? Try [VMware Application Catalog](https://bitnami.com/enterprise), the enterprise edition of Bitnami Application Catalog. + +## Prerequisites + +- Kubernetes 1.23+ +- Helm 3.8.0+ + +## Parameters + +## Special input schemas + +### ImageRoot + +```yaml +registry: + type: string + description: Docker registry where the image is located + example: docker.io + +repository: + type: string + description: Repository and image name + example: bitnami/nginx + +tag: + type: string + description: image tag + example: 1.16.1-debian-10-r63 + +pullPolicy: + type: string + description: Specify a imagePullPolicy. Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + +pullSecrets: + type: array + items: + type: string + description: Optionally specify an array of imagePullSecrets (evaluated as templates). + +debug: + type: boolean + description: Set to true if you would like to see extra information on logs + example: false + +## An instance would be: +# registry: docker.io +# repository: bitnami/nginx +# tag: 1.16.1-debian-10-r63 +# pullPolicy: IfNotPresent +# debug: false +``` + +### Persistence + +```yaml +enabled: + type: boolean + description: Whether enable persistence. + example: true + +storageClass: + type: string + description: Ghost data Persistent Volume Storage Class, If set to "-", storageClassName: "" which disables dynamic provisioning. + example: "-" + +accessMode: + type: string + description: Access mode for the Persistent Volume Storage. + example: ReadWriteOnce + +size: + type: string + description: Size the Persistent Volume Storage. + example: 8Gi + +path: + type: string + description: Path to be persisted. + example: /bitnami + +## An instance would be: +# enabled: true +# storageClass: "-" +# accessMode: ReadWriteOnce +# size: 8Gi +# path: /bitnami +``` + +### ExistingSecret + +```yaml +name: + type: string + description: Name of the existing secret. + example: mySecret +keyMapping: + description: Mapping between the expected key name and the name of the key in the existing secret. + type: object + +## An instance would be: +# name: mySecret +# keyMapping: +# password: myPasswordKey +``` + +#### Example of use + +When we store sensitive data for a deployment in a secret, some times we want to give to users the possibility of using theirs existing secrets. + +```yaml +# templates/secret.yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "common.names.fullname" . }} + labels: + app: {{ include "common.names.fullname" . }} +type: Opaque +data: + password: {{ .Values.password | b64enc | quote }} + +# templates/dpl.yaml +--- +... + env: + - name: PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "common.secrets.name" (dict "existingSecret" .Values.existingSecret "context" $) }} + key: {{ include "common.secrets.key" (dict "existingSecret" .Values.existingSecret "key" "password") }} +... + +# values.yaml +--- +name: mySecret +keyMapping: + password: myPasswordKey +``` + +### ValidateValue + +#### NOTES.txt + +```console +{{- $validateValueConf00 := (dict "valueKey" "path.to.value00" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value01" "secret" "secretName" "field" "password-01") -}} + +{{ include "common.validations.values.multiple.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} +``` + +If we force those values to be empty we will see some alerts + +```console +helm install test mychart --set path.to.value00="",path.to.value01="" + 'path.to.value00' must not be empty, please add '--set path.to.value00=$PASSWORD_00' to the command. To get the current value: + + export PASSWORD_00=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-00}" | base64 -d) + + 'path.to.value01' must not be empty, please add '--set path.to.value01=$PASSWORD_01' to the command. To get the current value: + + export PASSWORD_01=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-01}" | base64 -d) +``` + +## Upgrading + +### To 1.0.0 + +[On November 13, 2020, Helm v2 support was formally finished](https://github.com/helm/charts#status-of-the-project), this major version is the result of the required changes applied to the Helm Chart to be able to incorporate the different features added in Helm v3 and to be consistent with the Helm project itself regarding the Helm v2 EOL. + +#### What changes were introduced in this major version? + +- Previous versions of this Helm Chart use `apiVersion: v1` (installable by both Helm 2 and 3), this Helm Chart was updated to `apiVersion: v2` (installable by Helm 3 only). [Here](https://helm.sh/docs/topics/charts/#the-apiversion-field) you can find more information about the `apiVersion` field. +- Use `type: library`. [Here](https://v3.helm.sh/docs/faq/#library-chart-support) you can find more information. +- The different fields present in the *Chart.yaml* file has been ordered alphabetically in a homogeneous way for all the Bitnami Helm Charts + +#### Considerations when upgrading to this version + +- If you want to upgrade to this version from a previous one installed with Helm v3, you shouldn't face any issues +- If you want to upgrade to this version using Helm v2, this scenario is not supported as this version doesn't support Helm v2 anymore +- If you installed the previous version with Helm v2 and wants to upgrade to this version with Helm v3, please refer to the [official Helm documentation](https://helm.sh/docs/topics/v2_v3_migration/#migration-use-cases) about migrating from Helm v2 to v3 + +#### Useful links + +- +- +- + +## License + +Copyright © 2023 VMware, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/charts/airflow/charts/postgresql/charts/common/templates/_affinities.tpl b/charts/airflow/charts/postgresql/charts/common/templates/_affinities.tpl new file mode 100644 index 0000000..e85b1df --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/_affinities.tpl @@ -0,0 +1,139 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Return a soft nodeAffinity definition +{{ include "common.affinities.nodes.soft" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.soft" -}} +preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} + weight: 1 +{{- end -}} + +{{/* +Return a hard nodeAffinity definition +{{ include "common.affinities.nodes.hard" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.hard" -}} +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} +{{- end -}} + +{{/* +Return a nodeAffinity definition +{{ include "common.affinities.nodes" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.nodes.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.nodes.hard" . -}} + {{- end -}} +{{- end -}} + +{{/* +Return a topologyKey definition +{{ include "common.affinities.topologyKey" (dict "topologyKey" "BAR") -}} +*/}} +{{- define "common.affinities.topologyKey" -}} +{{ .topologyKey | default "kubernetes.io/hostname" -}} +{{- end -}} + +{{/* +Return a soft podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.soft" (dict "component" "FOO" "customLabels" .Values.podLabels "extraMatchLabels" .Values.extraMatchLabels "topologyKey" "BAR" "extraPodAffinityTerms" .Values.extraPodAffinityTerms "context" $) -}} +*/}} +{{- define "common.affinities.pods.soft" -}} +{{- $component := default "" .component -}} +{{- $customLabels := default (dict) .customLabels -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +{{- $extraPodAffinityTerms := default (list) .extraPodAffinityTerms -}} +preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" .context )) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + weight: 1 + {{- range $extraPodAffinityTerms }} + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" $.context )) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := .extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + weight: {{ .weight | default 1 -}} + {{- end -}} +{{- end -}} + +{{/* +Return a hard podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.hard" (dict "component" "FOO" "customLabels" .Values.podLabels "extraMatchLabels" .Values.extraMatchLabels "topologyKey" "BAR" "extraPodAffinityTerms" .Values.extraPodAffinityTerms "context" $) -}} +*/}} +{{- define "common.affinities.pods.hard" -}} +{{- $component := default "" .component -}} +{{- $customLabels := default (dict) .customLabels -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +{{- $extraPodAffinityTerms := default (list) .extraPodAffinityTerms -}} +requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" .context )) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + {{- range $extraPodAffinityTerms }} + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" $.context )) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := .extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + {{- end -}} +{{- end -}} + +{{/* +Return a podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.pods" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.pods.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.pods.hard" . -}} + {{- end -}} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/_capabilities.tpl b/charts/airflow/charts/postgresql/charts/common/templates/_capabilities.tpl new file mode 100644 index 0000000..115674a --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/_capabilities.tpl @@ -0,0 +1,229 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Return the target Kubernetes version +*/}} +{{- define "common.capabilities.kubeVersion" -}} +{{- if .Values.global }} + {{- if .Values.global.kubeVersion }} + {{- .Values.global.kubeVersion -}} + {{- else }} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} + {{- end -}} +{{- else }} +{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for poddisruptionbudget. +*/}} +{{- define "common.capabilities.policy.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "policy/v1beta1" -}} +{{- else -}} +{{- print "policy/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for networkpolicy. +*/}} +{{- define "common.capabilities.networkPolicy.apiVersion" -}} +{{- if semverCompare "<1.7-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for cronjob. +*/}} +{{- define "common.capabilities.cronjob.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "batch/v1beta1" -}} +{{- else -}} +{{- print "batch/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for daemonset. +*/}} +{{- define "common.capabilities.daemonset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "common.capabilities.deployment.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "common.capabilities.statefulset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apps/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "common.capabilities.ingress.apiVersion" -}} +{{- if .Values.ingress -}} +{{- if .Values.ingress.apiVersion -}} +{{- .Values.ingress.apiVersion -}} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for RBAC resources. +*/}} +{{- define "common.capabilities.rbac.apiVersion" -}} +{{- if semverCompare "<1.17-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "rbac.authorization.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "rbac.authorization.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for CRDs. +*/}} +{{- define "common.capabilities.crd.apiVersion" -}} +{{- if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiextensions.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiextensions.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for APIService. +*/}} +{{- define "common.capabilities.apiService.apiVersion" -}} +{{- if semverCompare "<1.10-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiregistration.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiregistration.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "common.capabilities.hpa.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .context) -}} +{{- if .beta2 -}} +{{- print "autoscaling/v2beta2" -}} +{{- else -}} +{{- print "autoscaling/v2beta1" -}} +{{- end -}} +{{- else -}} +{{- print "autoscaling/v2" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for Vertical Pod Autoscaler. +*/}} +{{- define "common.capabilities.vpa.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .context) -}} +{{- if .beta2 -}} +{{- print "autoscaling/v2beta2" -}} +{{- else -}} +{{- print "autoscaling/v2beta1" -}} +{{- end -}} +{{- else -}} +{{- print "autoscaling/v2" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if PodSecurityPolicy is supported +*/}} +{{- define "common.capabilities.psp.supported" -}} +{{- if semverCompare "<1.25-0" (include "common.capabilities.kubeVersion" .) -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if AdmissionConfiguration is supported +*/}} +{{- define "common.capabilities.admissionConfiguration.supported" -}} +{{- if semverCompare ">=1.23-0" (include "common.capabilities.kubeVersion" .) -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for AdmissionConfiguration. +*/}} +{{- define "common.capabilities.admissionConfiguration.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiserver.config.k8s.io/v1alpha1" -}} +{{- else if semverCompare "<1.25-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiserver.config.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiserver.config.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for PodSecurityConfiguration. +*/}} +{{- define "common.capabilities.podSecurityConfiguration.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "pod-security.admission.config.k8s.io/v1alpha1" -}} +{{- else if semverCompare "<1.25-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "pod-security.admission.config.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "pod-security.admission.config.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the used Helm version is 3.3+. +A way to check the used Helm version was not introduced until version 3.3.0 with .Capabilities.HelmVersion, which contains an additional "{}}" structure. +This check is introduced as a regexMatch instead of {{ if .Capabilities.HelmVersion }} because checking for the key HelmVersion in <3.3 results in a "interface not found" error. +**To be removed when the catalog's minimun Helm version is 3.3** +*/}} +{{- define "common.capabilities.supportsHelmVersion" -}} +{{- if regexMatch "{(v[0-9])*[^}]*}}$" (.Capabilities | toString ) }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/_errors.tpl b/charts/airflow/charts/postgresql/charts/common/templates/_errors.tpl new file mode 100644 index 0000000..07ded6f --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/_errors.tpl @@ -0,0 +1,28 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Through error when upgrading using empty passwords values that must not be empty. + +Usage: +{{- $validationError00 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password00" "secret" "secretName" "field" "password-00") -}} +{{- $validationError01 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password01" "secret" "secretName" "field" "password-01") -}} +{{ include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $validationError00 $validationError01) "context" $) }} + +Required password params: + - validationErrors - String - Required. List of validation strings to be return, if it is empty it won't throw error. + - context - Context - Required. Parent context. +*/}} +{{- define "common.errors.upgrade.passwords.empty" -}} + {{- $validationErrors := join "" .validationErrors -}} + {{- if and $validationErrors .context.Release.IsUpgrade -}} + {{- $errorString := "\nPASSWORDS ERROR: You must provide your current passwords when upgrading the release." -}} + {{- $errorString = print $errorString "\n Note that even after reinstallation, old credentials may be needed as they may be kept in persistent volume claims." -}} + {{- $errorString = print $errorString "\n Further information can be obtained at https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues/#credential-errors-while-upgrading-chart-releases" -}} + {{- $errorString = print $errorString "\n%s" -}} + {{- printf $errorString $validationErrors | fail -}} + {{- end -}} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/_images.tpl b/charts/airflow/charts/postgresql/charts/common/templates/_images.tpl new file mode 100644 index 0000000..1bcb779 --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/_images.tpl @@ -0,0 +1,117 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" .Values.global ) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $separator := ":" -}} +{{- $termination := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if .imageRoot.digest }} + {{- $separator = "@" -}} + {{- $termination = .imageRoot.digest | toString -}} +{{- end -}} +{{- if $registryName }} + {{- printf "%s/%s%s%s" $registryName $repositoryName $separator $termination -}} +{{- else -}} + {{- printf "%s%s%s" $repositoryName $separator $termination -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets .name -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end }} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets .name -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets | uniq }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names evaluating values as templates +{{ include "common.images.renderPullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $) }} +*/}} +{{- define "common.images.renderPullSecrets" -}} + {{- $pullSecrets := list }} + {{- $context := .context }} + + {{- if $context.Values.global }} + {{- range $context.Values.global.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" $context)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" $context)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets | uniq }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Return the proper image version (ingores image revision/prerelease info & fallbacks to chart appVersion) +{{ include "common.images.version" ( dict "imageRoot" .Values.path.to.the.image "chart" .Chart ) }} +*/}} +{{- define "common.images.version" -}} +{{- $imageTag := .imageRoot.tag | toString -}} +{{/* regexp from https://github.com/Masterminds/semver/blob/23f51de38a0866c5ef0bfc42b3f735c73107b700/version.go#L41-L44 */}} +{{- if regexMatch `^([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?$` $imageTag -}} + {{- $version := semver $imageTag -}} + {{- printf "%d.%d.%d" $version.Major $version.Minor $version.Patch -}} +{{- else -}} + {{- print .chart.AppVersion -}} +{{- end -}} +{{- end -}} + diff --git a/charts/airflow/charts/postgresql/charts/common/templates/_ingress.tpl b/charts/airflow/charts/postgresql/charts/common/templates/_ingress.tpl new file mode 100644 index 0000000..efa5b85 --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/_ingress.tpl @@ -0,0 +1,73 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Generate backend entry that is compatible with all Kubernetes API versions. + +Usage: +{{ include "common.ingress.backend" (dict "serviceName" "backendName" "servicePort" "backendPort" "context" $) }} + +Params: + - serviceName - String. Name of an existing service backend + - servicePort - String/Int. Port name (or number) of the service. It will be translated to different yaml depending if it is a string or an integer. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.ingress.backend" -}} +{{- $apiVersion := (include "common.capabilities.ingress.apiVersion" .context) -}} +{{- if or (eq $apiVersion "extensions/v1beta1") (eq $apiVersion "networking.k8s.io/v1beta1") -}} +serviceName: {{ .serviceName }} +servicePort: {{ .servicePort }} +{{- else -}} +service: + name: {{ .serviceName }} + port: + {{- if typeIs "string" .servicePort }} + name: {{ .servicePort }} + {{- else if or (typeIs "int" .servicePort) (typeIs "float64" .servicePort) }} + number: {{ .servicePort | int }} + {{- end }} +{{- end -}} +{{- end -}} + +{{/* +Print "true" if the API pathType field is supported +Usage: +{{ include "common.ingress.supportsPathType" . }} +*/}} +{{- define "common.ingress.supportsPathType" -}} +{{- if (semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .)) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the ingressClassname field is supported +Usage: +{{ include "common.ingress.supportsIngressClassname" . }} +*/}} +{{- define "common.ingress.supportsIngressClassname" -}} +{{- if semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if cert-manager required annotations for TLS signed +certificates are set in the Ingress annotations +Ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations +Usage: +{{ include "common.ingress.certManagerRequest" ( dict "annotations" .Values.path.to.the.ingress.annotations ) }} +*/}} +{{- define "common.ingress.certManagerRequest" -}} +{{ if or (hasKey .annotations "cert-manager.io/cluster-issuer") (hasKey .annotations "cert-manager.io/issuer") (hasKey .annotations "kubernetes.io/tls-acme") }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/_labels.tpl b/charts/airflow/charts/postgresql/charts/common/templates/_labels.tpl new file mode 100644 index 0000000..d90a6cd --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/_labels.tpl @@ -0,0 +1,46 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Kubernetes standard labels +{{ include "common.labels.standard" (dict "customLabels" .Values.commonLabels "context" $) -}} +*/}} +{{- define "common.labels.standard" -}} +{{- if and (hasKey . "customLabels") (hasKey . "context") -}} +{{- $default := dict "app.kubernetes.io/name" (include "common.names.name" .context) "helm.sh/chart" (include "common.names.chart" .context) "app.kubernetes.io/instance" .context.Release.Name "app.kubernetes.io/managed-by" .context.Release.Service -}} +{{- with .context.Chart.AppVersion -}} +{{- $_ := set $default "app.kubernetes.io/version" . -}} +{{- end -}} +{{ template "common.tplvalues.merge" (dict "values" (list .customLabels $default) "context" .context) }} +{{- else -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Labels used on immutable fields such as deploy.spec.selector.matchLabels or svc.spec.selector +{{ include "common.labels.matchLabels" (dict "customLabels" .Values.podLabels "context" $) -}} + +We don't want to loop over custom labels appending them to the selector +since it's very likely that it will break deployments, services, etc. +However, it's important to overwrite the standard labels if the user +overwrote them on metadata.labels fields. +*/}} +{{- define "common.labels.matchLabels" -}} +{{- if and (hasKey . "customLabels") (hasKey . "context") -}} +{{ merge (pick (include "common.tplvalues.render" (dict "value" .customLabels "context" .context) | fromYaml) "app.kubernetes.io/name" "app.kubernetes.io/instance") (dict "app.kubernetes.io/name" (include "common.names.name" .context) "app.kubernetes.io/instance" .context.Release.Name ) | toYaml }} +{{- else -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/_names.tpl b/charts/airflow/charts/postgresql/charts/common/templates/_names.tpl new file mode 100644 index 0000000..a222924 --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/_names.tpl @@ -0,0 +1,71 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "common.names.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "common.names.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "common.names.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified dependency name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +Usage: +{{ include "common.names.dependency.fullname" (dict "chartName" "dependency-chart-name" "chartValues" .Values.dependency-chart "context" $) }} +*/}} +{{- define "common.names.dependency.fullname" -}} +{{- if .chartValues.fullnameOverride -}} +{{- .chartValues.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .chartName .chartValues.nameOverride -}} +{{- if contains $name .context.Release.Name -}} +{{- .context.Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .context.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts. +*/}} +{{- define "common.names.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a fully qualified app name adding the installation's namespace. +*/}} +{{- define "common.names.fullname.namespace" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) (include "common.names.namespace" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/_secrets.tpl b/charts/airflow/charts/postgresql/charts/common/templates/_secrets.tpl new file mode 100644 index 0000000..a193c46 --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/_secrets.tpl @@ -0,0 +1,172 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Generate secret name. + +Usage: +{{ include "common.secrets.name" (dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $) }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/main/bitnami/common#existingsecret + - defaultNameSuffix - String - Optional. It is used only if we have several secrets in the same deployment. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.secrets.name" -}} +{{- $name := (include "common.names.fullname" .context) -}} + +{{- if .defaultNameSuffix -}} +{{- $name = printf "%s-%s" $name .defaultNameSuffix | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- with .existingSecret -}} +{{- if not (typeIs "string" .) -}} +{{- with .name -}} +{{- $name = . -}} +{{- end -}} +{{- else -}} +{{- $name = . -}} +{{- end -}} +{{- end -}} + +{{- printf "%s" $name -}} +{{- end -}} + +{{/* +Generate secret key. + +Usage: +{{ include "common.secrets.key" (dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName") }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/main/bitnami/common#existingsecret + - key - String - Required. Name of the key in the secret. +*/}} +{{- define "common.secrets.key" -}} +{{- $key := .key -}} + +{{- if .existingSecret -}} + {{- if not (typeIs "string" .existingSecret) -}} + {{- if .existingSecret.keyMapping -}} + {{- $key = index .existingSecret.keyMapping $.key -}} + {{- end -}} + {{- end }} +{{- end -}} + +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Generate secret password or retrieve one if already created. + +Usage: +{{ include "common.secrets.passwords.manage" (dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - providedValues - List - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - length - int - Optional - Length of the generated random password. + - strong - Boolean - Optional - Whether to add symbols to the generated random password. + - chartName - String - Optional - Name of the chart used when said chart is deployed as a subchart. + - context - Context - Required - Parent context. + - failOnNew - Boolean - Optional - Default to true. If set to false, skip errors adding new keys to existing secrets. +The order in which this function returns a secret password: + 1. Already existing 'Secret' resource + (If a 'Secret' resource is found under the name provided to the 'secret' parameter to this function and that 'Secret' resource contains a key with the name passed as the 'key' parameter to this function then the value of this existing secret password will be returned) + 2. Password provided via the values.yaml + (If one of the keys passed to the 'providedValues' parameter to this function is a valid path to a key in the values.yaml and has a value, the value of the first key with a value will be returned) + 3. Randomly generated secret password + (A new random secret password with the length specified in the 'length' parameter will be generated and returned) + +*/}} +{{- define "common.secrets.passwords.manage" -}} + +{{- $password := "" }} +{{- $subchart := "" }} +{{- $failOnNew := default true .failOnNew }} +{{- $chartName := default "" .chartName }} +{{- $passwordLength := default 10 .length }} +{{- $providedPasswordKey := include "common.utils.getKeyFromList" (dict "keys" .providedValues "context" $.context) }} +{{- $providedPasswordValue := include "common.utils.getValueFromKey" (dict "key" $providedPasswordKey "context" $.context) }} +{{- $secretData := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret).data }} +{{- if $secretData }} + {{- if hasKey $secretData .key }} + {{- $password = index $secretData .key | quote }} + {{- else if $failOnNew }} + {{- printf "\nPASSWORDS ERROR: The secret \"%s\" does not contain the key \"%s\"\n" .secret .key | fail -}} + {{- end -}} +{{- else if $providedPasswordValue }} + {{- $password = $providedPasswordValue | toString | b64enc | quote }} +{{- else }} + + {{- if .context.Values.enabled }} + {{- $subchart = $chartName }} + {{- end -}} + + {{- $requiredPassword := dict "valueKey" $providedPasswordKey "secret" .secret "field" .key "subchart" $subchart "context" $.context -}} + {{- $requiredPasswordError := include "common.validations.values.single.empty" $requiredPassword -}} + {{- $passwordValidationErrors := list $requiredPasswordError -}} + {{- include "common.errors.upgrade.passwords.empty" (dict "validationErrors" $passwordValidationErrors "context" $.context) -}} + + {{- if .strong }} + {{- $subStr := list (lower (randAlpha 1)) (randNumeric 1) (upper (randAlpha 1)) | join "_" }} + {{- $password = randAscii $passwordLength }} + {{- $password = regexReplaceAllLiteral "\\W" $password "@" | substr 5 $passwordLength }} + {{- $password = printf "%s%s" $subStr $password | toString | shuffle | b64enc | quote }} + {{- else }} + {{- $password = randAlphaNum $passwordLength | b64enc | quote }} + {{- end }} +{{- end -}} +{{- printf "%s" $password -}} +{{- end -}} + +{{/* +Reuses the value from an existing secret, otherwise sets its value to a default value. + +Usage: +{{ include "common.secrets.lookup" (dict "secret" "secret-name" "key" "keyName" "defaultValue" .Values.myValue "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - defaultValue - String - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - context - Context - Required - Parent context. + +*/}} +{{- define "common.secrets.lookup" -}} +{{- $value := "" -}} +{{- $secretData := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret).data -}} +{{- if and $secretData (hasKey $secretData .key) -}} + {{- $value = index $secretData .key -}} +{{- else if .defaultValue -}} + {{- $value = .defaultValue | toString | b64enc -}} +{{- end -}} +{{- if $value -}} +{{- printf "%s" $value -}} +{{- end -}} +{{- end -}} + +{{/* +Returns whether a previous generated secret already exists + +Usage: +{{ include "common.secrets.exists" (dict "secret" "secret-name" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - context - Context - Required - Parent context. +*/}} +{{- define "common.secrets.exists" -}} +{{- $secret := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret) }} +{{- if $secret }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/_storage.tpl b/charts/airflow/charts/postgresql/charts/common/templates/_storage.tpl new file mode 100644 index 0000000..16405a0 --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/_storage.tpl @@ -0,0 +1,28 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper Storage Class +{{ include "common.storage.class" ( dict "persistence" .Values.path.to.the.persistence "global" $) }} +*/}} +{{- define "common.storage.class" -}} + +{{- $storageClass := .persistence.storageClass -}} +{{- if .global -}} + {{- if .global.storageClass -}} + {{- $storageClass = .global.storageClass -}} + {{- end -}} +{{- end -}} + +{{- if $storageClass -}} + {{- if (eq "-" $storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" $storageClass -}} + {{- end -}} +{{- end -}} + +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/_tplvalues.tpl b/charts/airflow/charts/postgresql/charts/common/templates/_tplvalues.tpl new file mode 100644 index 0000000..a8ed763 --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/_tplvalues.tpl @@ -0,0 +1,38 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Renders a value that contains template perhaps with scope if the scope is present. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }} +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }} +*/}} +{{- define "common.tplvalues.render" -}} +{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }} +{{- if contains "{{" (toJson .value) }} + {{- if .scope }} + {{- tpl (cat "{{- with $.RelativeScope -}}" $value "{{- end }}") (merge (dict "RelativeScope" .scope) .context) }} + {{- else }} + {{- tpl $value .context }} + {{- end }} +{{- else }} + {{- $value }} +{{- end }} +{{- end -}} + +{{/* +Merge a list of values that contains template after rendering them. +Merge precedence is consistent with http://masterminds.github.io/sprig/dicts.html#merge-mustmerge +Usage: +{{ include "common.tplvalues.merge" ( dict "values" (list .Values.path.to.the.Value1 .Values.path.to.the.Value2) "context" $ ) }} +*/}} +{{- define "common.tplvalues.merge" -}} +{{- $dst := dict -}} +{{- range .values -}} +{{- $dst = include "common.tplvalues.render" (dict "value" . "context" $.context "scope" $.scope) | fromYaml | merge $dst -}} +{{- end -}} +{{ $dst | toYaml }} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/_utils.tpl b/charts/airflow/charts/postgresql/charts/common/templates/_utils.tpl new file mode 100644 index 0000000..bfbddf0 --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/_utils.tpl @@ -0,0 +1,77 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Print instructions to get a secret value. +Usage: +{{ include "common.utils.secret.getvalue" (dict "secret" "secret-name" "field" "secret-value-field" "context" $) }} +*/}} +{{- define "common.utils.secret.getvalue" -}} +{{- $varname := include "common.utils.fieldToEnvVar" . -}} +export {{ $varname }}=$(kubectl get secret --namespace {{ include "common.names.namespace" .context | quote }} {{ .secret }} -o jsonpath="{.data.{{ .field }}}" | base64 -d) +{{- end -}} + +{{/* +Build env var name given a field +Usage: +{{ include "common.utils.fieldToEnvVar" dict "field" "my-password" }} +*/}} +{{- define "common.utils.fieldToEnvVar" -}} + {{- $fieldNameSplit := splitList "-" .field -}} + {{- $upperCaseFieldNameSplit := list -}} + + {{- range $fieldNameSplit -}} + {{- $upperCaseFieldNameSplit = append $upperCaseFieldNameSplit ( upper . ) -}} + {{- end -}} + + {{ join "_" $upperCaseFieldNameSplit }} +{{- end -}} + +{{/* +Gets a value from .Values given +Usage: +{{ include "common.utils.getValueFromKey" (dict "key" "path.to.key" "context" $) }} +*/}} +{{- define "common.utils.getValueFromKey" -}} +{{- $splitKey := splitList "." .key -}} +{{- $value := "" -}} +{{- $latestObj := $.context.Values -}} +{{- range $splitKey -}} + {{- if not $latestObj -}} + {{- printf "please review the entire path of '%s' exists in values" $.key | fail -}} + {{- end -}} + {{- $value = ( index $latestObj . ) -}} + {{- $latestObj = $value -}} +{{- end -}} +{{- printf "%v" (default "" $value) -}} +{{- end -}} + +{{/* +Returns first .Values key with a defined value or first of the list if all non-defined +Usage: +{{ include "common.utils.getKeyFromList" (dict "keys" (list "path.to.key1" "path.to.key2") "context" $) }} +*/}} +{{- define "common.utils.getKeyFromList" -}} +{{- $key := first .keys -}} +{{- $reverseKeys := reverse .keys }} +{{- range $reverseKeys }} + {{- $value := include "common.utils.getValueFromKey" (dict "key" . "context" $.context ) }} + {{- if $value -}} + {{- $key = . }} + {{- end -}} +{{- end -}} +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Checksum a template at "path" containing a *single* resource (ConfigMap,Secret) for use in pod annotations, excluding the metadata (see #18376). +Usage: +{{ include "common.utils.checksumTemplate" (dict "path" "/configmap.yaml" "context" $) }} +*/}} +{{- define "common.utils.checksumTemplate" -}} +{{- $obj := include (print .context.Template.BasePath .path) .context | fromYaml -}} +{{ omit $obj "apiVersion" "kind" "metadata" | toYaml | sha256sum }} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/_warnings.tpl b/charts/airflow/charts/postgresql/charts/common/templates/_warnings.tpl new file mode 100644 index 0000000..66dffc1 --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/_warnings.tpl @@ -0,0 +1,19 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Warning about using rolling tag. +Usage: +{{ include "common.warnings.rollingTag" .Values.path.to.the.imageRoot }} +*/}} +{{- define "common.warnings.rollingTag" -}} + +{{- if and (contains "bitnami/" .repository) (not (.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .repository }}:{{ .tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} + +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/validations/_cassandra.tpl b/charts/airflow/charts/postgresql/charts/common/templates/validations/_cassandra.tpl new file mode 100644 index 0000000..eda9aad --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/validations/_cassandra.tpl @@ -0,0 +1,77 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Cassandra required passwords are not empty. + +Usage: +{{ include "common.validations.values.cassandra.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where Cassandra values are stored, e.g: "cassandra-passwords-secret" + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.cassandra.passwords" -}} + {{- $existingSecret := include "common.cassandra.values.existingSecret" . -}} + {{- $enabled := include "common.cassandra.values.enabled" . -}} + {{- $dbUserPrefix := include "common.cassandra.values.key.dbUser" . -}} + {{- $valueKeyPassword := printf "%s.password" $dbUserPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "cassandra-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.cassandra.values.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.cassandra.dbUser.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.dbUser.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled cassandra. + +Usage: +{{ include "common.cassandra.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.cassandra.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.cassandra.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key dbUser + +Usage: +{{ include "common.cassandra.values.key.dbUser" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.key.dbUser" -}} + {{- if .subchart -}} + cassandra.dbUser + {{- else -}} + dbUser + {{- end -}} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/validations/_mariadb.tpl b/charts/airflow/charts/postgresql/charts/common/templates/validations/_mariadb.tpl new file mode 100644 index 0000000..17d83a2 --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/validations/_mariadb.tpl @@ -0,0 +1,108 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MariaDB required passwords are not empty. + +Usage: +{{ include "common.validations.values.mariadb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MariaDB values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mariadb.passwords" -}} + {{- $existingSecret := include "common.mariadb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mariadb.values.enabled" . -}} + {{- $architecture := include "common.mariadb.values.architecture" . -}} + {{- $authPrefix := include "common.mariadb.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mariadb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mariadb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mariadb-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mariadb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mariadb. + +Usage: +{{ include "common.mariadb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mariadb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mariadb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mariadb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mariadb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.key.auth" -}} + {{- if .subchart -}} + mariadb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/validations/_mongodb.tpl b/charts/airflow/charts/postgresql/charts/common/templates/validations/_mongodb.tpl new file mode 100644 index 0000000..bbb445b --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/validations/_mongodb.tpl @@ -0,0 +1,113 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MongoDB® required passwords are not empty. + +Usage: +{{ include "common.validations.values.mongodb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MongoDB® values are stored, e.g: "mongodb-passwords-secret" + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mongodb.passwords" -}} + {{- $existingSecret := include "common.mongodb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mongodb.values.enabled" . -}} + {{- $authPrefix := include "common.mongodb.values.key.auth" . -}} + {{- $architecture := include "common.mongodb.values.architecture" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyDatabase := printf "%s.database" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicaSetKey := printf "%s.replicaSetKey" $authPrefix -}} + {{- $valueKeyAuthEnabled := printf "%s.enabled" $authPrefix -}} + + {{- $authEnabled := include "common.utils.getValueFromKey" (dict "key" $valueKeyAuthEnabled "context" .context) -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") (eq $authEnabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mongodb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- $valueDatabase := include "common.utils.getValueFromKey" (dict "key" $valueKeyDatabase "context" .context) }} + {{- if and $valueUsername $valueDatabase -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mongodb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replicaset") -}} + {{- $requiredReplicaSetKey := dict "valueKey" $valueKeyReplicaSetKey "secret" .secret "field" "mongodb-replica-set-key" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicaSetKey -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mongodb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDb is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mongodb. + +Usage: +{{ include "common.mongodb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mongodb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mongodb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mongodb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.key.auth" -}} + {{- if .subchart -}} + mongodb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mongodb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/validations/_mysql.tpl b/charts/airflow/charts/postgresql/charts/common/templates/validations/_mysql.tpl new file mode 100644 index 0000000..ca3953f --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/validations/_mysql.tpl @@ -0,0 +1,108 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MySQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.mysql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MySQL values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mysql.passwords" -}} + {{- $existingSecret := include "common.mysql.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mysql.values.enabled" . -}} + {{- $architecture := include "common.mysql.values.architecture" . -}} + {{- $authPrefix := include "common.mysql.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mysql-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mysql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mysql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mysql.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mysql. + +Usage: +{{ include "common.mysql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mysql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mysql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mysql.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mysql.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.key.auth" -}} + {{- if .subchart -}} + mysql.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/validations/_postgresql.tpl b/charts/airflow/charts/postgresql/charts/common/templates/validations/_postgresql.tpl new file mode 100644 index 0000000..8c9aa57 --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/validations/_postgresql.tpl @@ -0,0 +1,134 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate PostgreSQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.postgresql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where postgresql values are stored, e.g: "postgresql-passwords-secret" + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.postgresql.passwords" -}} + {{- $existingSecret := include "common.postgresql.values.existingSecret" . -}} + {{- $enabled := include "common.postgresql.values.enabled" . -}} + {{- $valueKeyPostgresqlPassword := include "common.postgresql.values.key.postgressPassword" . -}} + {{- $valueKeyPostgresqlReplicationEnabled := include "common.postgresql.values.key.replicationPassword" . -}} + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + {{- $requiredPostgresqlPassword := dict "valueKey" $valueKeyPostgresqlPassword "secret" .secret "field" "postgresql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlPassword -}} + + {{- $enabledReplication := include "common.postgresql.values.enabled.replication" . -}} + {{- if (eq $enabledReplication "true") -}} + {{- $requiredPostgresqlReplicationPassword := dict "valueKey" $valueKeyPostgresqlReplicationEnabled "secret" .secret "field" "postgresql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to decide whether evaluate global values. + +Usage: +{{ include "common.postgresql.values.use.global" (dict "key" "key-of-global" "context" $) }} +Params: + - key - String - Required. Field to be evaluated within global, e.g: "existingSecret" +*/}} +{{- define "common.postgresql.values.use.global" -}} + {{- if .context.Values.global -}} + {{- if .context.Values.global.postgresql -}} + {{- index .context.Values.global.postgresql .key | quote -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.postgresql.values.existingSecret" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.existingSecret" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "existingSecret" "context" .context) -}} + + {{- if .subchart -}} + {{- default (.context.Values.postgresql.existingSecret | quote) $globalValue -}} + {{- else -}} + {{- default (.context.Values.existingSecret | quote) $globalValue -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled postgresql. + +Usage: +{{ include "common.postgresql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key postgressPassword. + +Usage: +{{ include "common.postgresql.values.key.postgressPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.postgressPassword" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "postgresqlUsername" "context" .context) -}} + + {{- if not $globalValue -}} + {{- if .subchart -}} + postgresql.postgresqlPassword + {{- else -}} + postgresqlPassword + {{- end -}} + {{- else -}} + global.postgresql.postgresqlPassword + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled.replication. + +Usage: +{{ include "common.postgresql.values.enabled.replication" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.enabled.replication" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.replication.enabled -}} + {{- else -}} + {{- printf "%v" .context.Values.replication.enabled -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key replication.password. + +Usage: +{{ include "common.postgresql.values.key.replicationPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.replicationPassword" -}} + {{- if .subchart -}} + postgresql.replication.password + {{- else -}} + replication.password + {{- end -}} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/validations/_redis.tpl b/charts/airflow/charts/postgresql/charts/common/templates/validations/_redis.tpl new file mode 100644 index 0000000..fc0d208 --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/validations/_redis.tpl @@ -0,0 +1,81 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Redis® required passwords are not empty. + +Usage: +{{ include "common.validations.values.redis.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where redis values are stored, e.g: "redis-passwords-secret" + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.redis.passwords" -}} + {{- $enabled := include "common.redis.values.enabled" . -}} + {{- $valueKeyPrefix := include "common.redis.values.keys.prefix" . -}} + {{- $standarizedVersion := include "common.redis.values.standarized.version" . }} + + {{- $existingSecret := ternary (printf "%s%s" $valueKeyPrefix "auth.existingSecret") (printf "%s%s" $valueKeyPrefix "existingSecret") (eq $standarizedVersion "true") }} + {{- $existingSecretValue := include "common.utils.getValueFromKey" (dict "key" $existingSecret "context" .context) }} + + {{- $valueKeyRedisPassword := ternary (printf "%s%s" $valueKeyPrefix "auth.password") (printf "%s%s" $valueKeyPrefix "password") (eq $standarizedVersion "true") }} + {{- $valueKeyRedisUseAuth := ternary (printf "%s%s" $valueKeyPrefix "auth.enabled") (printf "%s%s" $valueKeyPrefix "usePassword") (eq $standarizedVersion "true") }} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $useAuth := include "common.utils.getValueFromKey" (dict "key" $valueKeyRedisUseAuth "context" .context) -}} + {{- if eq $useAuth "true" -}} + {{- $requiredRedisPassword := dict "valueKey" $valueKeyRedisPassword "secret" .secret "field" "redis-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRedisPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled redis. + +Usage: +{{ include "common.redis.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.redis.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.redis.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right prefix path for the values + +Usage: +{{ include "common.redis.values.key.prefix" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.redis.values.keys.prefix" -}} + {{- if .subchart -}}redis.{{- else -}}{{- end -}} +{{- end -}} + +{{/* +Checks whether the redis chart's includes the standarizations (version >= 14) + +Usage: +{{ include "common.redis.values.standarized.version" (dict "context" $) }} +*/}} +{{- define "common.redis.values.standarized.version" -}} + + {{- $standarizedAuth := printf "%s%s" (include "common.redis.values.keys.prefix" .) "auth" -}} + {{- $standarizedAuthValues := include "common.utils.getValueFromKey" (dict "key" $standarizedAuth "context" .context) }} + + {{- if $standarizedAuthValues -}} + {{- true -}} + {{- end -}} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/templates/validations/_validations.tpl b/charts/airflow/charts/postgresql/charts/common/templates/validations/_validations.tpl new file mode 100644 index 0000000..31ceda8 --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/templates/validations/_validations.tpl @@ -0,0 +1,51 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate values must not be empty. + +Usage: +{{- $validateValueConf00 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-01") -}} +{{ include "common.validations.values.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" +*/}} +{{- define "common.validations.values.multiple.empty" -}} + {{- range .required -}} + {{- include "common.validations.values.single.empty" (dict "valueKey" .valueKey "secret" .secret "field" .field "context" $.context) -}} + {{- end -}} +{{- end -}} + +{{/* +Validate a value must not be empty. + +Usage: +{{ include "common.validations.value.empty" (dict "valueKey" "mariadb.password" "secret" "secretName" "field" "my-password" "subchart" "subchart" "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" + - subchart - String - Optional - Name of the subchart that the validated password is part of. +*/}} +{{- define "common.validations.values.single.empty" -}} + {{- $value := include "common.utils.getValueFromKey" (dict "key" .valueKey "context" .context) }} + {{- $subchart := ternary "" (printf "%s." .subchart) (empty .subchart) }} + + {{- if not $value -}} + {{- $varname := "my-value" -}} + {{- $getCurrentValue := "" -}} + {{- if and .secret .field -}} + {{- $varname = include "common.utils.fieldToEnvVar" . -}} + {{- $getCurrentValue = printf " To get the current value:\n\n %s\n" (include "common.utils.secret.getvalue" .) -}} + {{- end -}} + {{- printf "\n '%s' must not be empty, please add '--set %s%s=$%s' to the command.%s" .valueKey $subchart .valueKey $varname $getCurrentValue -}} + {{- end -}} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/charts/common/values.yaml b/charts/airflow/charts/postgresql/charts/common/values.yaml new file mode 100644 index 0000000..9abe0e1 --- /dev/null +++ b/charts/airflow/charts/postgresql/charts/common/values.yaml @@ -0,0 +1,8 @@ +# Copyright VMware, Inc. +# SPDX-License-Identifier: APACHE-2.0 + +## bitnami/common +## It is required by CI/CD tools and processes. +## @skip exampleValue +## +exampleValue: common-chart diff --git a/charts/airflow/charts/postgresql/templates/NOTES.txt b/charts/airflow/charts/postgresql/templates/NOTES.txt new file mode 100644 index 0000000..73c4a34 --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/NOTES.txt @@ -0,0 +1,115 @@ +CHART NAME: {{ .Chart.Name }} +CHART VERSION: {{ .Chart.Version }} +APP VERSION: {{ .Chart.AppVersion }} + +** Please be patient while the chart is being deployed ** + +{{- if .Values.diagnosticMode.enabled }} +The chart has been deployed in diagnostic mode. All probes have been disabled and the command has been overwritten with: + + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 4 }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 4 }} + +Get the list of pods by executing: + + kubectl get pods --namespace {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} + +Access the pod you want to debug by executing + + kubectl exec --namespace {{ .Release.Namespace }} -ti -- /opt/bitnami/scripts/postgresql/entrypoint.sh /bin/bash + +In order to replicate the container startup scripts execute this command: + + /opt/bitnami/scripts/postgresql/entrypoint.sh /opt/bitnami/scripts/postgresql/run.sh + +{{- else }} + +{{- $customUser := include "postgresql.v1.username" . }} +{{- $postgresPassword := include "common.secrets.lookup" (dict "secret" (include "common.names.fullname" .) "key" .Values.auth.secretKeys.adminPasswordKey "defaultValue" (ternary .Values.auth.postgresPassword .Values.auth.password (eq $customUser "postgres")) "context" $) -}} +{{- $authEnabled := and (not (or .Values.global.postgresql.auth.existingSecret .Values.auth.existingSecret)) (or $postgresPassword .Values.auth.enablePostgresUser (and (not (empty $customUser)) (ne $customUser "postgres"))) }} +{{- if not $authEnabled }} + +WARNING: PostgreSQL has been configured without authentication, this is not recommended for production environments. +{{- end }} + +PostgreSQL can be accessed via port {{ include "postgresql.v1.service.port" . }} on the following DNS names from within your cluster: + + {{ include "postgresql.v1.primary.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local - Read/Write connection + +{{- if eq .Values.architecture "replication" }} + + {{ include "postgresql.v1.readReplica.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local - Read only connection + +{{- end }} + +{{- if and (not (empty $customUser)) (ne $customUser "postgres") }} +{{- if .Values.auth.enablePostgresUser }} + +To get the password for "postgres" run: + + export POSTGRES_ADMIN_PASSWORD=$(kubectl get secret --namespace {{ .Release.Namespace }} {{ include "postgresql.v1.secretName" . }} -o jsonpath="{.data.{{include "postgresql.v1.adminPasswordKey" .}}}" | base64 -d) +{{- end }} + +To get the password for "{{ $customUser }}" run: + + export POSTGRES_PASSWORD=$(kubectl get secret --namespace {{ .Release.Namespace }} {{ include "postgresql.v1.secretName" . }} -o jsonpath="{.data.{{include "postgresql.v1.userPasswordKey" .}}}" | base64 -d) +{{- else }} +{{- if .Values.auth.enablePostgresUser }} + +To get the password for "{{ default "postgres" $customUser }}" run: + + export POSTGRES_PASSWORD=$(kubectl get secret --namespace {{ .Release.Namespace }} {{ include "postgresql.v1.secretName" . }} -o jsonpath="{.data.{{ ternary "password" (include "postgresql.v1.adminPasswordKey" .) (and (not (empty $customUser)) (ne $customUser "postgres")) }}}" | base64 -d) +{{- end }} +{{- end }} + +To connect to your database run the following command: + {{- if $authEnabled }} + + kubectl run {{ include "common.names.fullname" . }}-client --rm --tty -i --restart='Never' --namespace {{ .Release.Namespace }} --image {{ include "postgresql.v1.image" . }} --env="PGPASSWORD=$POSTGRES_PASSWORD" \ + --command -- psql --host {{ include "postgresql.v1.primary.fullname" . }} -U {{ default "postgres" $customUser }} -d {{- if include "postgresql.v1.database" . }} {{ include "postgresql.v1.database" . }}{{- else }} postgres{{- end }} -p {{ include "postgresql.v1.service.port" . }} + {{- else }} + + kubectl run {{ include "common.names.fullname" . }}-client --rm --tty -i --restart='Never' --namespace {{ .Release.Namespace }} --image {{ include "postgresql.v1.image" . }} \ + --command -- psql --host {{ include "postgresql.v1.primary.fullname" . }} -d {{- if include "postgresql.v1.database" . }} {{ include "postgresql.v1.database" . }}{{- else }} postgres{{- end }} -p {{ include "postgresql.v1.service.port" . }} + {{- end }} + + > NOTE: If you access the container using bash, make sure that you execute "/opt/bitnami/scripts/postgresql/entrypoint.sh /bin/bash" in order to avoid the error "psql: local user with ID {{ .Values.primary.containerSecurityContext.runAsUser }}} does not exist" + +To connect to your database from outside the cluster execute the following commands: + +{{- if contains "NodePort" .Values.primary.service.type }} + + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "postgresql.v1.primary.fullname" . }}) + {{- if $authEnabled }} + PGPASSWORD="$POSTGRES_PASSWORD" psql --host $NODE_IP --port $NODE_PORT -U {{ default "postgres" $customUser }} -d {{- if include "postgresql.v1.database" . }} {{ include "postgresql.v1.database" . }}{{- else }} postgres{{- end }} + {{- else }} + psql --host $NODE_IP --port $NODE_PORT -d {{- if include "postgresql.v1.database" . }} {{ include "postgresql.v1.database" . }}{{- else }} postgres{{- end }} + {{- end }} +{{- else if contains "LoadBalancer" .Values.primary.service.type }} + + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + Watch the status with: 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ include "postgresql.v1.primary.fullname" . }}' + + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "postgresql.v1.primary.fullname" . }} --template "{{ "{{ range (index .status.loadBalancer.ingress 0) }}{{ . }}{{ end }}" }}") + {{- if $authEnabled }} + PGPASSWORD="$POSTGRES_PASSWORD" psql --host $SERVICE_IP --port {{ include "postgresql.v1.service.port" . }} -U {{ default "postgres" $customUser }} -d {{- if include "postgresql.v1.database" . }} {{ include "postgresql.v1.database" . }}{{- else }} postgres{{- end }} + {{- else }} + psql --host $SERVICE_IP --port {{ include "postgresql.v1.service.port" . }} -d {{- if include "postgresql.v1.database" . }} {{ include "postgresql.v1.database" . }}{{- else }} postgres{{- end }} + {{- end }} +{{- else if contains "ClusterIP" .Values.primary.service.type }} + + kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ include "postgresql.v1.primary.fullname" . }} {{ include "postgresql.v1.service.port" . }}:{{ include "postgresql.v1.service.port" . }} & + {{- if $authEnabled }} + PGPASSWORD="$POSTGRES_PASSWORD" psql --host 127.0.0.1 -U {{ default "postgres" $customUser }} -d {{- if include "postgresql.v1.database" . }} {{ include "postgresql.v1.database" . }}{{- else }} postgres{{- end }} -p {{ include "postgresql.v1.service.port" . }} + {{- else }} + psql --host 127.0.0.1 -d {{- if include "postgresql.v1.database" . }} {{ include "postgresql.v1.database" . }}{{- else }} postgres{{- end }} -p {{ include "postgresql.v1.service.port" . }} + {{- end }} +{{- end }} +{{- end }} + +WARNING: The configured password will be ignored on new installation in case when previous PostgreSQL release was deleted through the helm command. In that case, old PVC will have an old password, and setting it through helm won't take effect. Deleting persistent volumes (PVs) will solve the issue. + +{{- include "postgresql.v1.validateValues" . -}} +{{- include "common.warnings.rollingTag" .Values.image -}} +{{- include "common.warnings.rollingTag" .Values.volumePermissions.image }} diff --git a/charts/airflow/charts/postgresql/templates/_helpers.tpl b/charts/airflow/charts/postgresql/templates/_helpers.tpl new file mode 100644 index 0000000..0ab9fd0 --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/_helpers.tpl @@ -0,0 +1,406 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Create a default fully qualified app name for PostgreSQL Primary objects +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "postgresql.v1.primary.fullname" -}} +{{- if eq .Values.architecture "replication" -}} + {{- printf "%s-%s" (include "common.names.fullname" .) .Values.primary.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} + {{- include "common.names.fullname" . -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified app name for PostgreSQL read-only replicas objects +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "postgresql.v1.readReplica.fullname" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) .Values.readReplicas.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the default FQDN for PostgreSQL primary headless service +We truncate at 63 chars because of the DNS naming spec. +*/}} +{{- define "postgresql.v1.primary.svc.headless" -}} +{{- printf "%s-hl" (include "postgresql.v1.primary.fullname" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the default FQDN for PostgreSQL read-only replicas headless service +We truncate at 63 chars because of the DNS naming spec. +*/}} +{{- define "postgresql.v1.readReplica.svc.headless" -}} +{{- printf "%s-hl" (include "postgresql.v1.readReplica.fullname" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Return the proper PostgreSQL image name +*/}} +{{- define "postgresql.v1.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global) }} +{{- end -}} + +{{/* +Return the proper PostgreSQL metrics image name +*/}} +{{- define "postgresql.v1.metrics.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.metrics.image "global" .Values.global) }} +{{- end -}} + +{{/* +Return the proper image name (for the init container volume-permissions image) +*/}} +{{- define "postgresql.v1.volumePermissions.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.volumePermissions.image "global" .Values.global) }} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names +*/}} +{{- define "postgresql.v1.imagePullSecrets" -}} +{{ include "common.images.renderPullSecrets" (dict "images" (list .Values.image .Values.metrics.image .Values.volumePermissions.image) "context" $) }} +{{- end -}} + +{{/* +Return the name for a custom user to create +*/}} +{{- define "postgresql.v1.username" -}} +{{- if .Values.global.postgresql.auth.username -}} + {{- .Values.global.postgresql.auth.username -}} +{{- else -}} + {{- .Values.auth.username -}} +{{- end -}} +{{- end -}} + +{{/* +Return the name for a custom database to create +*/}} +{{- define "postgresql.v1.database" -}} +{{- if .Values.global.postgresql.auth.database -}} + {{- printf "%s" (tpl .Values.global.postgresql.auth.database $) -}} +{{- else if .Values.auth.database -}} + {{- printf "%s" (tpl .Values.auth.database $) -}} +{{- end -}} +{{- end -}} + +{{/* +Get the password secret. +*/}} +{{- define "postgresql.v1.secretName" -}} +{{- if .Values.global.postgresql.auth.existingSecret -}} + {{- printf "%s" (tpl .Values.global.postgresql.auth.existingSecret $) -}} +{{- else if .Values.auth.existingSecret -}} + {{- printf "%s" (tpl .Values.auth.existingSecret $) -}} +{{- else -}} + {{- printf "%s" (include "common.names.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +Get the replication-password key. +*/}} +{{- define "postgresql.v1.replicationPasswordKey" -}} +{{- if or .Values.global.postgresql.auth.existingSecret .Values.auth.existingSecret -}} + {{- if .Values.global.postgresql.auth.secretKeys.replicationPasswordKey -}} + {{- printf "%s" (tpl .Values.global.postgresql.auth.secretKeys.replicationPasswordKey $) -}} + {{- else if .Values.auth.secretKeys.replicationPasswordKey -}} + {{- printf "%s" (tpl .Values.auth.secretKeys.replicationPasswordKey $) -}} + {{- else -}} + {{- "replication-password" -}} + {{- end -}} +{{- else -}} + {{- "replication-password" -}} +{{- end -}} +{{- end -}} + +{{/* +Get the admin-password key. +*/}} +{{- define "postgresql.v1.adminPasswordKey" -}} +{{- if or .Values.global.postgresql.auth.existingSecret .Values.auth.existingSecret -}} + {{- if .Values.global.postgresql.auth.secretKeys.adminPasswordKey -}} + {{- printf "%s" (tpl .Values.global.postgresql.auth.secretKeys.adminPasswordKey $) -}} + {{- else if .Values.auth.secretKeys.adminPasswordKey -}} + {{- printf "%s" (tpl .Values.auth.secretKeys.adminPasswordKey $) -}} + {{- end -}} +{{- else -}} + {{- "postgres-password" -}} +{{- end -}} +{{- end -}} + +{{/* +Get the user-password key. +*/}} +{{- define "postgresql.v1.userPasswordKey" -}} +{{- if or .Values.global.postgresql.auth.existingSecret .Values.auth.existingSecret -}} + {{- if or (empty (include "postgresql.v1.username" .)) (eq (include "postgresql.v1.username" .) "postgres") -}} + {{- printf "%s" (include "postgresql.v1.adminPasswordKey" .) -}} + {{- else -}} + {{- if .Values.global.postgresql.auth.secretKeys.userPasswordKey -}} + {{- printf "%s" (tpl .Values.global.postgresql.auth.secretKeys.userPasswordKey $) -}} + {{- else if .Values.auth.secretKeys.userPasswordKey -}} + {{- printf "%s" (tpl .Values.auth.secretKeys.userPasswordKey $) -}} + {{- end -}} + {{- end -}} +{{- else -}} + {{- "password" -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if a secret object should be created +*/}} +{{- define "postgresql.v1.createSecret" -}} +{{- $customUser := include "postgresql.v1.username" . -}} +{{- $postgresPassword := include "common.secrets.lookup" (dict "secret" (include "common.names.fullname" .) "key" .Values.auth.secretKeys.adminPasswordKey "defaultValue" (ternary (coalesce .Values.global.postgresql.auth.postgresPassword .Values.auth.postgresPassword .Values.global.postgresql.auth.password .Values.auth.password) (coalesce .Values.global.postgresql.auth.postgresPassword .Values.auth.postgresPassword) (or (empty $customUser) (eq $customUser "postgres"))) "context" $) -}} +{{- if and (not (or .Values.global.postgresql.auth.existingSecret .Values.auth.existingSecret)) (or $postgresPassword .Values.auth.enablePostgresUser (and (not (empty $customUser)) (ne $customUser "postgres")) (eq .Values.architecture "replication") (and .Values.ldap.enabled (or .Values.ldap.bind_password .Values.ldap.bindpw))) -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Return PostgreSQL service port +*/}} +{{- define "postgresql.v1.service.port" -}} +{{- if .Values.global.postgresql.service.ports.postgresql -}} + {{- .Values.global.postgresql.service.ports.postgresql -}} +{{- else -}} + {{- .Values.primary.service.ports.postgresql -}} +{{- end -}} +{{- end -}} + +{{/* +Return PostgreSQL service port +*/}} +{{- define "postgresql.v1.readReplica.service.port" -}} +{{- if .Values.global.postgresql.service.ports.postgresql -}} + {{- .Values.global.postgresql.service.ports.postgresql -}} +{{- else -}} + {{- .Values.readReplicas.service.ports.postgresql -}} +{{- end -}} +{{- end -}} + +{{/* +Get the PostgreSQL primary configuration ConfigMap name. +*/}} +{{- define "postgresql.v1.primary.configmapName" -}} +{{- if .Values.primary.existingConfigmap -}} + {{- printf "%s" (tpl .Values.primary.existingConfigmap $) -}} +{{- else -}} + {{- printf "%s-configuration" (include "postgresql.v1.primary.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if a configmap object should be created for PostgreSQL primary with the configuration +*/}} +{{- define "postgresql.v1.primary.createConfigmap" -}} +{{- if and (or .Values.primary.configuration .Values.primary.pgHbaConfiguration) (not .Values.primary.existingConfigmap) -}} + {{- true -}} +{{- else -}} +{{- end -}} +{{- end -}} + +{{/* +Get the PostgreSQL primary extended configuration ConfigMap name. +*/}} +{{- define "postgresql.v1.primary.extendedConfigmapName" -}} +{{- if .Values.primary.existingExtendedConfigmap -}} + {{- printf "%s" (tpl .Values.primary.existingExtendedConfigmap $) -}} +{{- else -}} + {{- printf "%s-extended-configuration" (include "postgresql.v1.primary.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +Get the PostgreSQL read replica extended configuration ConfigMap name. +*/}} +{{- define "postgresql.v1.readReplicas.extendedConfigmapName" -}} + {{- printf "%s-extended-configuration" (include "postgresql.v1.readReplica.fullname" .) -}} +{{- end -}} + +{{/* +Return true if a configmap object should be created for PostgreSQL primary with the extended configuration +*/}} +{{- define "postgresql.v1.primary.createExtendedConfigmap" -}} +{{- if and .Values.primary.extendedConfiguration (not .Values.primary.existingExtendedConfigmap) -}} + {{- true -}} +{{- else -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if a configmap object should be created for PostgreSQL read replica with the extended configuration +*/}} +{{- define "postgresql.v1.readReplicas.createExtendedConfigmap" -}} +{{- if .Values.readReplicas.extendedConfiguration -}} + {{- true -}} +{{- else -}} +{{- end -}} +{{- end -}} + +{{/* + Create the name of the service account to use + */}} +{{- define "postgresql.v1.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "common.names.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Return true if a configmap should be mounted with PostgreSQL configuration +*/}} +{{- define "postgresql.v1.mountConfigurationCM" -}} +{{- if or .Values.primary.configuration .Values.primary.pgHbaConfiguration .Values.primary.existingConfigmap -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Get the initialization scripts ConfigMap name. +*/}} +{{- define "postgresql.v1.initdb.scriptsCM" -}} +{{- if .Values.primary.initdb.scriptsConfigMap -}} + {{- printf "%s" (tpl .Values.primary.initdb.scriptsConfigMap $) -}} +{{- else -}} + {{- printf "%s-init-scripts" (include "postgresql.v1.primary.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if TLS is enabled for LDAP connection +*/}} +{{- define "postgresql.v1.ldap.tls.enabled" -}} +{{- if and (kindIs "string" .Values.ldap.tls) (not (empty .Values.ldap.tls)) -}} + {{- true -}} +{{- else if and (kindIs "map" .Values.ldap.tls) .Values.ldap.tls.enabled -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Get the readiness probe command +*/}} +{{- define "postgresql.v1.readinessProbeCommand" -}} +{{- $customUser := include "postgresql.v1.username" . -}} +- | +{{- if (include "postgresql.v1.database" .) }} + exec pg_isready -U {{ default "postgres" $customUser | quote }} -d "dbname={{ include "postgresql.v1.database" . }} {{- if .Values.tls.enabled }} sslcert={{ include "postgresql.v1.tlsCert" . }} sslkey={{ include "postgresql.v1.tlsCertKey" . }}{{- end }}" -h 127.0.0.1 -p {{ .Values.containerPorts.postgresql }} +{{- else }} + exec pg_isready -U {{ default "postgres" $customUser | quote }} {{- if .Values.tls.enabled }} -d "sslcert={{ include "postgresql.v1.tlsCert" . }} sslkey={{ include "postgresql.v1.tlsCertKey" . }}"{{- end }} -h 127.0.0.1 -p {{ .Values.containerPorts.postgresql }} +{{- end }} +{{- if contains "bitnami/" .Values.image.repository }} + [ -f /opt/bitnami/postgresql/tmp/.initialized ] || [ -f /bitnami/postgresql/.initialized ] +{{- end }} +{{- end -}} + +{{/* +Compile all warnings into a single message, and call fail. +*/}} +{{- define "postgresql.v1.validateValues" -}} +{{- $messages := list -}} +{{- $messages := append $messages (include "postgresql.v1.validateValues.ldapConfigurationMethod" .) -}} +{{- $messages := append $messages (include "postgresql.v1.validateValues.psp" .) -}} +{{- $messages := without $messages "" -}} +{{- $message := join "\n" $messages -}} + +{{- if $message -}} +{{- printf "\nVALUES VALIDATION:\n%s" $message | fail -}} +{{- end -}} +{{- end -}} + +{{/* +Validate values of Postgresql - If ldap.url is used then you don't need the other settings for ldap +*/}} +{{- define "postgresql.v1.validateValues.ldapConfigurationMethod" -}} +{{- if and .Values.ldap.enabled (and (not (empty .Values.ldap.url)) (not (empty .Values.ldap.server))) -}} +postgresql: ldap.url, ldap.server + You cannot set both `ldap.url` and `ldap.server` at the same time. + Please provide a unique way to configure LDAP. + More info at https://www.postgresql.org/docs/current/auth-ldap.html +{{- end -}} +{{- end -}} + +{{/* +Validate values of Postgresql - If PSP is enabled RBAC should be enabled too +*/}} +{{- define "postgresql.v1.validateValues.psp" -}} +{{- if and .Values.psp.create (not .Values.rbac.create) -}} +postgresql: psp.create, rbac.create + RBAC should be enabled if PSP is enabled in order for PSP to work. + More info at https://kubernetes.io/docs/concepts/policy/pod-security-policy/#authorizing-policies +{{- end -}} +{{- end -}} + +{{/* +Return the path to the cert file. +*/}} +{{- define "postgresql.v1.tlsCert" -}} +{{- if .Values.tls.autoGenerated -}} + {{- printf "/opt/bitnami/postgresql/certs/tls.crt" -}} +{{- else -}} + {{- required "Certificate filename is required when TLS in enabled" .Values.tls.certFilename | printf "/opt/bitnami/postgresql/certs/%s" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the path to the cert key file. +*/}} +{{- define "postgresql.v1.tlsCertKey" -}} +{{- if .Values.tls.autoGenerated -}} + {{- printf "/opt/bitnami/postgresql/certs/tls.key" -}} +{{- else -}} +{{- required "Certificate Key filename is required when TLS in enabled" .Values.tls.certKeyFilename | printf "/opt/bitnami/postgresql/certs/%s" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the path to the CA cert file. +*/}} +{{- define "postgresql.v1.tlsCACert" -}} +{{- if .Values.tls.autoGenerated -}} + {{- printf "/opt/bitnami/postgresql/certs/ca.crt" -}} +{{- else -}} + {{- printf "/opt/bitnami/postgresql/certs/%s" .Values.tls.certCAFilename -}} +{{- end -}} +{{- end -}} + +{{/* +Return the path to the CRL file. +*/}} +{{- define "postgresql.v1.tlsCRL" -}} +{{- if .Values.tls.crlFilename -}} +{{- printf "/opt/bitnami/postgresql/certs/%s" .Values.tls.crlFilename -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if a TLS credentials secret object should be created +*/}} +{{- define "postgresql.v1.createTlsSecret" -}} +{{- if and .Values.tls.autoGenerated (not .Values.tls.certificatesSecret) -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Return the path to the CA cert file. +*/}} +{{- define "postgresql.v1.tlsSecretName" -}} +{{- if .Values.tls.autoGenerated -}} + {{- printf "%s-crt" (include "common.names.fullname" .) -}} +{{- else -}} + {{ required "A secret containing TLS certificates is required when TLS is enabled" .Values.tls.certificatesSecret }} +{{- end -}} +{{- end -}} diff --git a/charts/airflow/charts/postgresql/templates/backup/cronjob.yaml b/charts/airflow/charts/postgresql/templates/backup/cronjob.yaml new file mode 100644 index 0000000..812fd84 --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/backup/cronjob.yaml @@ -0,0 +1,114 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if .Values.backup.enabled }} +{{- $customUser := include "postgresql.v1.username" . }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "postgresql.v1.primary.fullname" . }}-pgdumpall + namespace: {{ .Release.Namespace | quote }} + {{- $labels := include "common.tplvalues.merge" ( dict "values" ( list .Values.backup.cronjob.labels .Values.commonLabels ) "context" . ) }} + labels: {{- include "common.labels.standard" ( dict "customLabels" $labels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: pg_dumpall + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.backup.cronjob.annotations .Values.commonAnnotations ) "context" . ) }} + {{- if $annotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +spec: + schedule: {{ quote .Values.backup.cronjob.schedule }} + {{- if .Values.backup.cronjob.timezone }} + timeZone: {{ .Values.backup.cronjob.timezone | quote }} + {{- end }} + concurrencyPolicy: {{ .Values.backup.cronjob.concurrencyPolicy }} + failedJobsHistoryLimit: {{ .Values.backup.cronjob.failedJobsHistoryLimit }} + successfulJobsHistoryLimit: {{ .Values.backup.cronjob.successfulJobsHistoryLimit }} + {{- if .Values.backup.cronjob.startingDeadlineSeconds }} + startingDeadlineSeconds: {{ .Values.backup.cronjob.startingDeadlineSeconds }} + {{- end }} + jobTemplate: + spec: + {{- if .Values.backup.cronjob.ttlSecondsAfterFinished }} + ttlSecondsAfterFinished: {{ .Values.backup.cronjob.ttlSecondsAfterFinished }} + {{- end }} + template: + metadata: + labels: {{- include "common.labels.standard" ( dict "customLabels" $labels "context" $ ) | nindent 12 }} + app.kubernetes.io/component: pg_dumpall + {{- if $annotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 12 }} + {{- end }} + spec: + {{- include "postgresql.v1.imagePullSecrets" . | nindent 10 }} + {{- if .Values.backup.cronjob.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.backup.cronjob.nodeSelector "context" $) | nindent 12 }} + {{- end }} + containers: + - name: {{ include "postgresql.v1.primary.fullname" . }}-pgdumpall + image: {{ include "postgresql.v1.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + env: + - name: PGUSER + {{- if .Values.auth.enablePostgresUser }} + value: postgres + {{- else }} + value: {{ $customUser | quote }} + {{- end }} + {{- if .Values.auth.usePasswordFiles }} + - name: PGPASSFILE + value: {{ printf "/opt/bitnami/postgresql/secrets/%s" (include "postgresql.v1.adminPasswordKey" .) }} + {{- else }} + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" . }} + key: {{ include "postgresql.v1.adminPasswordKey" . }} + {{- end }} + - name: PGHOST + value: {{ include "postgresql.v1.primary.fullname" . }} + - name: PGPORT + value: {{ include "postgresql.v1.service.port" . | quote }} + - name: PGDUMP_DIR + value: {{ .Values.backup.cronjob.storage.mountPath }} + {{- if .Values.tls.enabled }} + - name: PGSSLROOTCERT + {{- if .Values.tls.autoGenerated -}} + value: /tmp/certs/ca.crt + {{- else }} + value: {{- printf "/tmp/certs/%s" .Values.tls.certCAFilename -}} + {{- end }} + {{- end }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.backup.cronjob.command "context" $) | nindent 14 }} + volumeMounts: + {{- if .Values.tls.enabled }} + - name: certs + mountPath: /certs + {{- end }} + - name: datadir + mountPath: {{ .Values.backup.cronjob.storage.mountPath }} + subPath: {{ .Values.backup.cronjob.storage.subPath }} + {{- if .Values.backup.cronjob.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.backup.cronjob.containerSecurityContext "enabled" | toYaml | nindent 14 }} + {{- end }} + restartPolicy: {{ .Values.backup.cronjob.restartPolicy }} + {{- if .Values.backup.cronjob.podSecurityContext.enabled }} + securityContext: + fsGroup: {{ .Values.backup.cronjob.podSecurityContext.fsGroup }} + {{- end }} + volumes: + {{- if .Values.tls.enabled }} + - name: raw-certificates + emptyDir: /tmp/certs + {{- end }} + {{- if .Values.backup.cronjob.storage.existingClaim }} + - name: datadir + persistentVolumeClaim: + claimName: {{ printf "%s" (tpl .Values.backup.cronjob.storage.existingClaim .) }} + {{- else }} + - name: datadir + persistentVolumeClaim: + claimName: {{ include "postgresql.v1.primary.fullname" . }}-pgdumpall + {{- end }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/backup/pvc.yaml b/charts/airflow/charts/postgresql/templates/backup/pvc.yaml new file mode 100644 index 0000000..6fe9cbf --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/backup/pvc.yaml @@ -0,0 +1,34 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.backup.enabled (not .Values.backup.cronjob.storage.existingClaim) -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "postgresql.v1.primary.fullname" . }}-pgdumpall + namespace: {{ .Release.Namespace | quote }} + {{- $labels := include "common.tplvalues.merge" ( dict "values" ( list .Values.backup.cronjob.labels .Values.commonLabels ) "context" . ) }} + labels: {{- include "common.labels.standard" ( dict "customLabels" $labels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: pg_dumpall + {{- if or .Values.backup.cronjob.annotations .Values.commonAnnotations .Values.backup.cronjob.storage.resourcePolicy }} + annotations: + {{- if or .Values.backup.cronjob.annotations .Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.backup.cronjob.annotations .Values.commonAnnotations ) "context" . ) }} + {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.backup.cronjob.storage.resourcePolicy }} + helm.sh/resource-policy: {{ .Values.backup.cronjob.storage.resourcePolicy | quote }} + {{- end }} + {{- end }} +spec: + accessModes: + {{- range .Values.backup.cronjob.storage.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.backup.cronjob.storage.size | quote }} + {{ include "common.storage.class" (dict "persistence" .Values.backup.cronjob.storage "global" .Values.global) }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/extra-list.yaml b/charts/airflow/charts/postgresql/templates/extra-list.yaml new file mode 100644 index 0000000..2d35a58 --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/extra-list.yaml @@ -0,0 +1,9 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- range .Values.extraDeploy }} +--- +{{ include "common.tplvalues.render" (dict "value" . "context" $) }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/networkpolicy-egress.yaml b/charts/airflow/charts/postgresql/templates/networkpolicy-egress.yaml new file mode 100644 index 0000000..b67817c --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/networkpolicy-egress.yaml @@ -0,0 +1,34 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.networkPolicy.enabled (or .Values.networkPolicy.egressRules.denyConnectionsToExternal .Values.networkPolicy.egressRules.customRules) }} +apiVersion: {{ include "common.capabilities.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + name: {{ printf "%s-egress" (include "common.names.fullname" .) }} + namespace: {{ .Release.Namespace }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 6 }} + policyTypes: + - Egress + egress: + {{- if .Values.networkPolicy.egressRules.denyConnectionsToExternal }} + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + - to: + - namespaceSelector: {} + {{- end }} + {{- if .Values.networkPolicy.egressRules.customRules }} + {{- include "common.tplvalues.render" (dict "value" .Values.networkPolicy.egressRules.customRules "context" $) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/primary/configmap.yaml b/charts/airflow/charts/postgresql/templates/primary/configmap.yaml new file mode 100644 index 0000000..7a69891 --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/primary/configmap.yaml @@ -0,0 +1,26 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if (include "postgresql.v1.primary.createConfigmap" .) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-configuration" (include "postgresql.v1.primary.fullname" .) }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: primary + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + {{- if .Values.primary.configuration }} + postgresql.conf: | + {{- include "common.tplvalues.render" ( dict "value" .Values.primary.configuration "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.primary.pgHbaConfiguration }} + pg_hba.conf: | + {{- include "common.tplvalues.render" ( dict "value" .Values.primary.pgHbaConfiguration "context" $ ) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/primary/extended-configmap.yaml b/charts/airflow/charts/postgresql/templates/primary/extended-configmap.yaml new file mode 100644 index 0000000..456f8ee --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/primary/extended-configmap.yaml @@ -0,0 +1,20 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if (include "postgresql.v1.primary.createExtendedConfigmap" .) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-extended-configuration" (include "postgresql.v1.primary.fullname" .) }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: primary + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + override.conf: |- + {{- include "common.tplvalues.render" ( dict "value" .Values.primary.extendedConfiguration "context" $ ) | nindent 4 }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/primary/initialization-configmap.yaml b/charts/airflow/charts/postgresql/templates/primary/initialization-configmap.yaml new file mode 100644 index 0000000..80d804a --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/primary/initialization-configmap.yaml @@ -0,0 +1,17 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.primary.initdb.scripts (not .Values.primary.initdb.scriptsConfigMap) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-init-scripts" (include "postgresql.v1.primary.fullname" .) }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: {{- include "common.tplvalues.render" (dict "value" .Values.primary.initdb.scripts "context" .) | nindent 2 }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/primary/metrics-configmap.yaml b/charts/airflow/charts/postgresql/templates/primary/metrics-configmap.yaml new file mode 100644 index 0000000..7da2bcd --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/primary/metrics-configmap.yaml @@ -0,0 +1,18 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.metrics.enabled .Values.metrics.customMetrics }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-metrics" (include "postgresql.v1.primary.fullname" .) }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + custom-metrics.yaml: {{ toYaml .Values.metrics.customMetrics | quote }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/primary/metrics-svc.yaml b/charts/airflow/charts/postgresql/templates/primary/metrics-svc.yaml new file mode 100644 index 0000000..3d94510 --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/primary/metrics-svc.yaml @@ -0,0 +1,31 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ printf "%s-metrics" (include "postgresql.v1.primary.fullname" .) }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: metrics + {{- if or .Values.commonAnnotations .Values.metrics.service.annotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.metrics.service.annotations .Values.commonAnnotations ) "context" . ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + sessionAffinity: {{ .Values.metrics.service.sessionAffinity }} + {{- if .Values.metrics.service.clusterIP }} + clusterIP: {{ .Values.metrics.service.clusterIP }} + {{- end }} + ports: + - name: http-metrics + port: {{ .Values.metrics.service.ports.metrics }} + targetPort: http-metrics + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.primary.podLabels .Values.commonLabels ) "context" . ) }} + selector: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: primary +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/primary/networkpolicy.yaml b/charts/airflow/charts/postgresql/templates/primary/networkpolicy.yaml new file mode 100644 index 0000000..9da3fb4 --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/primary/networkpolicy.yaml @@ -0,0 +1,61 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.networkPolicy.enabled (or .Values.networkPolicy.metrics.enabled .Values.networkPolicy.ingressRules.primaryAccessOnlyFrom.enabled) }} +apiVersion: {{ include "common.capabilities.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + name: {{ printf "%s-ingress" (include "postgresql.v1.primary.fullname" .) }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: primary + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + {{- $primaryPodLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.primary.podLabels .Values.commonLabels ) "context" . ) }} + podSelector: + matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $primaryPodLabels "context" $ ) | nindent 6 }} + app.kubernetes.io/component: primary + ingress: + {{- if and .Values.metrics.enabled .Values.networkPolicy.metrics.enabled (or .Values.networkPolicy.metrics.namespaceSelector .Values.networkPolicy.metrics.podSelector) }} + - from: + {{- if .Values.networkPolicy.metrics.namespaceSelector }} + - namespaceSelector: + matchLabels: {{- include "common.tplvalues.render" (dict "value" .Values.networkPolicy.metrics.namespaceSelector "context" $) | nindent 14 }} + {{- end }} + {{- if .Values.networkPolicy.metrics.podSelector }} + - podSelector: + matchLabels: {{- include "common.tplvalues.render" (dict "value" .Values.networkPolicy.metrics.podSelector "context" $) | nindent 14 }} + {{- end }} + ports: + - port: {{ .Values.metrics.containerPorts.metrics }} + {{- end }} + {{- if and .Values.networkPolicy.ingressRules.primaryAccessOnlyFrom.enabled (or .Values.networkPolicy.ingressRules.primaryAccessOnlyFrom.namespaceSelector .Values.networkPolicy.ingressRules.primaryAccessOnlyFrom.podSelector) }} + - from: + {{- if .Values.networkPolicy.ingressRules.primaryAccessOnlyFrom.namespaceSelector }} + - namespaceSelector: + matchLabels: {{- include "common.tplvalues.render" (dict "value" .Values.networkPolicy.ingressRules.primaryAccessOnlyFrom.namespaceSelector "context" $) | nindent 14 }} + {{- end }} + {{- if .Values.networkPolicy.ingressRules.primaryAccessOnlyFrom.podSelector }} + - podSelector: + matchLabels: {{- include "common.tplvalues.render" (dict "value" .Values.networkPolicy.ingressRules.primaryAccessOnlyFrom.podSelector "context" $) | nindent 14 }} + {{- end }} + ports: + - port: {{ .Values.containerPorts.postgresql }} + {{- end }} + {{- if and .Values.networkPolicy.ingressRules.primaryAccessOnlyFrom.enabled (eq .Values.architecture "replication") }} + - from: + {{- $readPodLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.readReplicas.podLabels .Values.commonLabels ) "context" . ) }} + - podSelector: + matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $readPodLabels "context" $ ) | nindent 14 }} + app.kubernetes.io/component: read + ports: + - port: {{ .Values.containerPorts.postgresql }} + {{- end }} + {{- if .Values.networkPolicy.ingressRules.primaryAccessOnlyFrom.customRules }} + {{- include "common.tplvalues.render" (dict "value" .Values.networkPolicy.ingressRules.primaryAccessOnlyFrom.customRules "context" $) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/primary/servicemonitor.yaml b/charts/airflow/charts/postgresql/templates/primary/servicemonitor.yaml new file mode 100644 index 0000000..05d54f3 --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/primary/servicemonitor.yaml @@ -0,0 +1,46 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "postgresql.v1.primary.fullname" . }} + namespace: {{ default .Release.Namespace .Values.metrics.serviceMonitor.namespace | quote }} + {{- $labels := include "common.tplvalues.merge" ( dict "values" ( list .Values.metrics.serviceMonitor.labels .Values.commonLabels ) "context" . ) }} + labels: {{- include "common.labels.standard" ( dict "customLabels" $labels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: metrics + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + {{- if .Values.metrics.serviceMonitor.jobLabel }} + jobLabel: {{ .Values.metrics.serviceMonitor.jobLabel }} + {{- end }} + selector: + {{- $svcLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.metrics.serviceMonitor.selector .Values.commonLabels ) "context" . ) }} + matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $svcLabels "context" $ ) | nindent 6 }} + app.kubernetes.io/component: metrics + endpoints: + - port: http-metrics + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.relabelings }} + relabelings: {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.serviceMonitor.relabelings "context" $) | nindent 6 }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.serviceMonitor.metricRelabelings "context" $) | nindent 6 }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.honorLabels }} + honorLabels: {{ .Values.metrics.serviceMonitor.honorLabels }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace | quote }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/primary/statefulset.yaml b/charts/airflow/charts/postgresql/templates/primary/statefulset.yaml new file mode 100644 index 0000000..cb9374d --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/primary/statefulset.yaml @@ -0,0 +1,661 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- $customUser := include "postgresql.v1.username" . }} +apiVersion: {{ include "common.capabilities.statefulset.apiVersion" . }} +kind: StatefulSet +metadata: + name: {{ include "postgresql.v1.primary.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + {{- $labels := include "common.tplvalues.merge" ( dict "values" ( list .Values.primary.labels .Values.commonLabels ) "context" . ) }} + labels: {{- include "common.labels.standard" ( dict "customLabels" $labels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: primary + {{- if or .Values.commonAnnotations .Values.primary.annotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.primary.annotations .Values.commonAnnotations ) "context" . ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +spec: + replicas: 1 + serviceName: {{ include "postgresql.v1.primary.svc.headless" . }} + {{- if .Values.primary.updateStrategy }} + updateStrategy: {{- toYaml .Values.primary.updateStrategy | nindent 4 }} + {{- end }} + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.primary.podLabels .Values.commonLabels ) "context" . ) }} + selector: + matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} + app.kubernetes.io/component: primary + template: + metadata: + name: {{ include "postgresql.v1.primary.fullname" . }} + labels: {{- include "common.labels.standard" ( dict "customLabels" $podLabels "context" $ ) | nindent 8 }} + app.kubernetes.io/component: primary + {{- if or (include "postgresql.v1.primary.createConfigmap" .) (include "postgresql.v1.primary.createExtendedConfigmap" .) .Values.primary.podAnnotations }} + annotations: + {{- if (include "postgresql.v1.primary.createConfigmap" .) }} + checksum/configuration: {{ pick (include (print $.Template.BasePath "/primary/configmap.yaml") . | fromYaml) "data" | toYaml | sha256sum }} + {{- end }} + {{- if (include "postgresql.v1.primary.createExtendedConfigmap" .) }} + checksum/extended-configuration: {{ pick (include (print $.Template.BasePath "/primary/extended-configmap.yaml") . | fromYaml) "data" | toYaml | sha256sum }} + {{- end }} + {{- if .Values.primary.podAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.primary.podAnnotations "context" $ ) | nindent 8 }} + {{- end }} + {{- end }} + spec: + {{- if .Values.primary.extraPodSpec }} + {{- include "common.tplvalues.render" (dict "value" .Values.primary.extraPodSpec "context" $) | nindent 6 }} + {{- end }} + serviceAccountName: {{ include "postgresql.v1.serviceAccountName" . }} + {{- include "postgresql.v1.imagePullSecrets" . | nindent 6 }} + {{- if .Values.primary.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.primary.hostAliases "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.primary.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.primary.affinity "context" $) | nindent 8 }} + {{- else }} + affinity: + podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.primary.podAffinityPreset "component" "primary" "customLabels" $podLabels "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.primary.podAntiAffinityPreset "component" "primary" "customLabels" $podLabels "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.primary.nodeAffinityPreset.type "key" .Values.primary.nodeAffinityPreset.key "values" .Values.primary.nodeAffinityPreset.values) | nindent 10 }} + {{- end }} + {{- if .Values.primary.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.primary.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.primary.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.primary.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.primary.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.primary.topologySpreadConstraints "context" .) | nindent 8 }} + {{- end }} + {{- if .Values.primary.priorityClassName }} + priorityClassName: {{ .Values.primary.priorityClassName }} + {{- end }} + {{- if .Values.primary.schedulerName }} + schedulerName: {{ .Values.primary.schedulerName | quote }} + {{- end }} + {{- if .Values.primary.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ .Values.primary.terminationGracePeriodSeconds }} + {{- end }} + {{- if .Values.primary.podSecurityContext.enabled }} + securityContext: {{- omit .Values.primary.podSecurityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + hostNetwork: {{ .Values.primary.hostNetwork }} + hostIPC: {{ .Values.primary.hostIPC }} + {{- if or (and .Values.tls.enabled (not .Values.volumePermissions.enabled)) (and .Values.volumePermissions.enabled (or .Values.primary.persistence.enabled .Values.shmVolume.enabled)) .Values.primary.initContainers }} + initContainers: + {{- if and .Values.tls.enabled (not .Values.volumePermissions.enabled) }} + - name: copy-certs + image: {{ include "postgresql.v1.volumePermissions.image" . }} + imagePullPolicy: {{ .Values.volumePermissions.image.pullPolicy | quote }} + {{- if .Values.primary.resources }} + resources: {{- toYaml .Values.primary.resources | nindent 12 }} + {{- end }} + # We don't require a privileged container in this case + {{- if .Values.primary.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.primary.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + command: + - /bin/sh + - -ec + - | + cp /tmp/certs/* /opt/bitnami/postgresql/certs/ + chmod 600 {{ include "postgresql.v1.tlsCertKey" . }} + volumeMounts: + - name: raw-certificates + mountPath: /tmp/certs + - name: postgresql-certificates + mountPath: /opt/bitnami/postgresql/certs + {{- else if and .Values.volumePermissions.enabled (or .Values.primary.persistence.enabled .Values.shmVolume.enabled) }} + - name: init-chmod-data + image: {{ include "postgresql.v1.volumePermissions.image" . }} + imagePullPolicy: {{ .Values.volumePermissions.image.pullPolicy | quote }} + {{- if .Values.volumePermissions.resources }} + resources: {{- toYaml .Values.volumePermissions.resources | nindent 12 }} + {{- end }} + command: + - /bin/sh + - -ec + - | + {{- if .Values.primary.persistence.enabled }} + {{- if eq ( toString ( .Values.volumePermissions.containerSecurityContext.runAsUser )) "auto" }} + chown `id -u`:`id -G | cut -d " " -f2` {{ .Values.primary.persistence.mountPath }} + {{- else }} + chown {{ .Values.primary.containerSecurityContext.runAsUser }}:{{ .Values.primary.podSecurityContext.fsGroup }} {{ .Values.primary.persistence.mountPath }} + {{- end }} + mkdir -p {{ .Values.primary.persistence.mountPath }}/data {{- if (include "postgresql.v1.mountConfigurationCM" .) }} {{ .Values.primary.persistence.mountPath }}/conf {{- end }} + chmod 700 {{ .Values.primary.persistence.mountPath }}/data {{- if (include "postgresql.v1.mountConfigurationCM" .) }} {{ .Values.primary.persistence.mountPath }}/conf {{- end }} + find {{ .Values.primary.persistence.mountPath }} -mindepth 1 -maxdepth 1 {{- if not (include "postgresql.v1.mountConfigurationCM" .) }} -not -name "conf" {{- end }} -not -name ".snapshot" -not -name "lost+found" | \ + {{- if eq ( toString ( .Values.volumePermissions.containerSecurityContext.runAsUser )) "auto" }} + xargs -r chown -R `id -u`:`id -G | cut -d " " -f2` + {{- else }} + xargs -r chown -R {{ .Values.primary.containerSecurityContext.runAsUser }}:{{ .Values.primary.podSecurityContext.fsGroup }} + {{- end }} + {{- end }} + {{- if .Values.shmVolume.enabled }} + chmod -R 777 /dev/shm + {{- end }} + {{- if .Values.tls.enabled }} + cp /tmp/certs/* /opt/bitnami/postgresql/certs/ + {{- if eq ( toString ( .Values.volumePermissions.containerSecurityContext.runAsUser )) "auto" }} + chown -R `id -u`:`id -G | cut -d " " -f2` /opt/bitnami/postgresql/certs/ + {{- else }} + chown -R {{ .Values.primary.containerSecurityContext.runAsUser }}:{{ .Values.primary.podSecurityContext.fsGroup }} /opt/bitnami/postgresql/certs/ + {{- end }} + chmod 600 {{ include "postgresql.v1.tlsCertKey" . }} + {{- end }} + {{- if eq ( toString ( .Values.volumePermissions.containerSecurityContext.runAsUser )) "auto" }} + securityContext: {{- omit .Values.volumePermissions.containerSecurityContext "runAsUser" | toYaml | nindent 12 }} + {{- else }} + securityContext: {{- .Values.volumePermissions.containerSecurityContext | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + {{- if .Values.primary.persistence.enabled }} + - name: data + mountPath: {{ .Values.primary.persistence.mountPath }} + {{- if .Values.primary.persistence.subPath }} + subPath: {{ .Values.primary.persistence.subPath }} + {{- end }} + {{- end }} + {{- if .Values.shmVolume.enabled }} + - name: dshm + mountPath: /dev/shm + {{- end }} + {{- if .Values.tls.enabled }} + - name: raw-certificates + mountPath: /tmp/certs + - name: postgresql-certificates + mountPath: /opt/bitnami/postgresql/certs + {{- end }} + {{- end }} + {{- if .Values.primary.initContainers }} + {{- include "common.tplvalues.render" ( dict "value" .Values.primary.initContainers "context" $ ) | nindent 8 }} + {{- end }} + {{- end }} + containers: + - name: postgresql + image: {{ include "postgresql.v1.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.primary.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.primary.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} + {{- else if .Values.primary.command }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.primary.command "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.diagnosticMode.enabled }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} + {{- else if .Values.primary.args }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.primary.args "context" $) | nindent 12 }} + {{- end }} + env: + - name: BITNAMI_DEBUG + value: {{ ternary "true" "false" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} + - name: POSTGRESQL_PORT_NUMBER + value: {{ .Values.containerPorts.postgresql | quote }} + - name: POSTGRESQL_VOLUME_DIR + value: {{ .Values.primary.persistence.mountPath | quote }} + {{- if .Values.primary.persistence.mountPath }} + - name: PGDATA + value: {{ .Values.postgresqlDataDir | quote }} + {{- end }} + # Authentication + {{- if or (eq $customUser "postgres") (empty $customUser) }} + {{- if .Values.auth.enablePostgresUser }} + {{- if .Values.auth.usePasswordFiles }} + - name: POSTGRES_PASSWORD_FILE + value: {{ printf "/opt/bitnami/postgresql/secrets/%s" (include "postgresql.v1.adminPasswordKey" .) }} + {{- else }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" . }} + key: {{ include "postgresql.v1.adminPasswordKey" . }} + {{- end }} + {{- else }} + - name: ALLOW_EMPTY_PASSWORD + value: "true" + {{- end }} + {{- else }} + - name: POSTGRES_USER + value: {{ $customUser | quote }} + {{- if .Values.auth.usePasswordFiles }} + - name: POSTGRES_PASSWORD_FILE + value: {{ printf "/opt/bitnami/postgresql/secrets/%s" (include "postgresql.v1.userPasswordKey" .) }} + {{- else }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" . }} + key: {{ include "postgresql.v1.userPasswordKey" . }} + {{- end }} + {{- if .Values.auth.enablePostgresUser }} + {{- if .Values.auth.usePasswordFiles }} + - name: POSTGRES_POSTGRES_PASSWORD_FILE + value: {{ printf "/opt/bitnami/postgresql/secrets/%s" (include "postgresql.v1.adminPasswordKey" .) }} + {{- else }} + - name: POSTGRES_POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" . }} + key: {{ include "postgresql.v1.adminPasswordKey" . }} + {{- end }} + {{- end }} + {{- end }} + {{- if (include "postgresql.v1.database" .) }} + - name: POSTGRES_DATABASE + value: {{ (include "postgresql.v1.database" .) | quote }} + {{- end }} + # Replication + {{- if or (eq .Values.architecture "replication") .Values.primary.standby.enabled }} + - name: POSTGRES_REPLICATION_MODE + value: {{ ternary "slave" "master" .Values.primary.standby.enabled | quote }} + - name: POSTGRES_REPLICATION_USER + value: {{ .Values.auth.replicationUsername | quote }} + {{- if .Values.auth.usePasswordFiles }} + - name: POSTGRES_REPLICATION_PASSWORD_FILE + value: {{ printf "/opt/bitnami/postgresql/secrets/%s" (include "postgresql.v1.replicationPasswordKey" .) }} + {{- else }} + - name: POSTGRES_REPLICATION_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" . }} + key: {{ include "postgresql.v1.replicationPasswordKey" . }} + {{- end }} + {{- if ne .Values.replication.synchronousCommit "off" }} + - name: POSTGRES_SYNCHRONOUS_COMMIT_MODE + value: {{ .Values.replication.synchronousCommit | quote }} + - name: POSTGRES_NUM_SYNCHRONOUS_REPLICAS + value: {{ .Values.replication.numSynchronousReplicas | quote }} + {{- end }} + - name: POSTGRES_CLUSTER_APP_NAME + value: {{ .Values.replication.applicationName }} + {{- end }} + # Initdb + {{- if .Values.primary.initdb.args }} + - name: POSTGRES_INITDB_ARGS + value: {{ .Values.primary.initdb.args | quote }} + {{- end }} + {{- if .Values.primary.initdb.postgresqlWalDir }} + - name: POSTGRES_INITDB_WALDIR + value: {{ .Values.primary.initdb.postgresqlWalDir | quote }} + {{- end }} + {{- if .Values.primary.initdb.user }} + - name: POSTGRES_INITSCRIPTS_USERNAME + value: {{ .Values.primary.initdb.user }} + {{- end }} + {{- if .Values.primary.initdb.password }} + - name: POSTGRES_INITSCRIPTS_PASSWORD + value: {{ .Values.primary.initdb.password | quote }} + {{- end }} + # Standby + {{- if .Values.primary.standby.enabled }} + - name: POSTGRES_MASTER_HOST + value: {{ .Values.primary.standby.primaryHost }} + - name: POSTGRES_MASTER_PORT_NUMBER + value: {{ .Values.primary.standby.primaryPort | quote }} + {{- end }} + # LDAP + - name: POSTGRESQL_ENABLE_LDAP + value: {{ ternary "yes" "no" .Values.ldap.enabled | quote }} + {{- if .Values.ldap.enabled }} + {{- if or .Values.ldap.url .Values.ldap.uri }} + - name: POSTGRESQL_LDAP_URL + value: {{ coalesce .Values.ldap.url .Values.ldap.uri }} + {{- else }} + - name: POSTGRESQL_LDAP_SERVER + value: {{ .Values.ldap.server }} + - name: POSTGRESQL_LDAP_PORT + value: {{ .Values.ldap.port | quote }} + - name: POSTGRESQL_LDAP_SCHEME + value: {{ .Values.ldap.scheme }} + {{- if (include "postgresql.v1.ldap.tls.enabled" .) }} + - name: POSTGRESQL_LDAP_TLS + value: "1" + {{- end }} + - name: POSTGRESQL_LDAP_PREFIX + value: {{ .Values.ldap.prefix | quote }} + - name: POSTGRESQL_LDAP_SUFFIX + value: {{ .Values.ldap.suffix | quote }} + - name: POSTGRESQL_LDAP_BASE_DN + value: {{ coalesce .Values.ldap.baseDN .Values.ldap.basedn }} + - name: POSTGRESQL_LDAP_BIND_DN + value: {{ coalesce .Values.ldap.bindDN .Values.ldap.binddn}} + {{- if or (not (empty .Values.ldap.bind_password)) (not (empty .Values.ldap.bindpw)) }} + - name: POSTGRESQL_LDAP_BIND_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" . }} + key: ldap-password + {{- end }} + - name: POSTGRESQL_LDAP_SEARCH_ATTR + value: {{ coalesce .Values.ldap.search_attr .Values.ldap.searchAttribute }} + - name: POSTGRESQL_LDAP_SEARCH_FILTER + value: {{ coalesce .Values.ldap.search_filter .Values.ldap.searchFilter }} + {{- end }} + {{- end }} + # TLS + - name: POSTGRESQL_ENABLE_TLS + value: {{ ternary "yes" "no" .Values.tls.enabled | quote }} + {{- if .Values.tls.enabled }} + - name: POSTGRESQL_TLS_PREFER_SERVER_CIPHERS + value: {{ ternary "yes" "no" .Values.tls.preferServerCiphers | quote }} + - name: POSTGRESQL_TLS_CERT_FILE + value: {{ include "postgresql.v1.tlsCert" . }} + - name: POSTGRESQL_TLS_KEY_FILE + value: {{ include "postgresql.v1.tlsCertKey" . }} + {{- if .Values.tls.certCAFilename }} + - name: POSTGRESQL_TLS_CA_FILE + value: {{ include "postgresql.v1.tlsCACert" . }} + {{- end }} + {{- if .Values.tls.crlFilename }} + - name: POSTGRESQL_TLS_CRL_FILE + value: {{ include "postgresql.v1.tlsCRL" . }} + {{- end }} + {{- end }} + # Audit + - name: POSTGRESQL_LOG_HOSTNAME + value: {{ .Values.audit.logHostname | quote }} + - name: POSTGRESQL_LOG_CONNECTIONS + value: {{ .Values.audit.logConnections | quote }} + - name: POSTGRESQL_LOG_DISCONNECTIONS + value: {{ .Values.audit.logDisconnections | quote }} + {{- if .Values.audit.logLinePrefix }} + - name: POSTGRESQL_LOG_LINE_PREFIX + value: {{ .Values.audit.logLinePrefix | quote }} + {{- end }} + {{- if .Values.audit.logTimezone }} + - name: POSTGRESQL_LOG_TIMEZONE + value: {{ .Values.audit.logTimezone | quote }} + {{- end }} + {{- if .Values.audit.pgAuditLog }} + - name: POSTGRESQL_PGAUDIT_LOG + value: {{ .Values.audit.pgAuditLog | quote }} + {{- end }} + - name: POSTGRESQL_PGAUDIT_LOG_CATALOG + value: {{ .Values.audit.pgAuditLogCatalog | quote }} + # Others + - name: POSTGRESQL_CLIENT_MIN_MESSAGES + value: {{ .Values.audit.clientMinMessages | quote }} + - name: POSTGRESQL_SHARED_PRELOAD_LIBRARIES + value: {{ .Values.postgresqlSharedPreloadLibraries | quote }} + {{- if .Values.primary.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.primary.extraEnvVars "context" $) | nindent 12 }} + {{- end }} + {{- if or .Values.primary.extraEnvVarsCM .Values.primary.extraEnvVarsSecret }} + envFrom: + {{- if .Values.primary.extraEnvVarsCM }} + - configMapRef: + name: {{ .Values.primary.extraEnvVarsCM }} + {{- end }} + {{- if .Values.primary.extraEnvVarsSecret }} + - secretRef: + name: {{ .Values.primary.extraEnvVarsSecret }} + {{- end }} + {{- end }} + ports: + - name: tcp-postgresql + containerPort: {{ .Values.containerPorts.postgresql }} + {{- if not .Values.diagnosticMode.enabled }} + {{- if .Values.primary.customStartupProbe }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.primary.customStartupProbe "context" $) | nindent 12 }} + {{- else if .Values.primary.startupProbe.enabled }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.primary.startupProbe "enabled") "context" $) | nindent 12 }} + exec: + command: + - /bin/sh + - -c + {{- if (include "postgresql.v1.database" .) }} + - exec pg_isready -U {{ default "postgres" $customUser | quote }} -d "dbname={{ include "postgresql.v1.database" . }} {{- if and .Values.tls.enabled .Values.tls.certCAFilename }} sslcert={{ include "postgresql.v1.tlsCert" . }} sslkey={{ include "postgresql.v1.tlsCertKey" . }}{{- end }}" -h 127.0.0.1 -p {{ .Values.containerPorts.postgresql }} + {{- else }} + - exec pg_isready -U {{ default "postgres" $customUser | quote }} {{- if and .Values.tls.enabled .Values.tls.certCAFilename }} -d "sslcert={{ include "postgresql.v1.tlsCert" . }} sslkey={{ include "postgresql.v1.tlsCertKey" . }}"{{- end }} -h 127.0.0.1 -p {{ .Values.containerPorts.postgresql }} + {{- end }} + {{- end }} + {{- if .Values.primary.customLivenessProbe }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.primary.customLivenessProbe "context" $) | nindent 12 }} + {{- else if .Values.primary.livenessProbe.enabled }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.primary.livenessProbe "enabled") "context" $) | nindent 12 }} + exec: + command: + - /bin/sh + - -c + {{- if (include "postgresql.v1.database" .) }} + - exec pg_isready -U {{ default "postgres" $customUser | quote }} -d "dbname={{ include "postgresql.v1.database" . }} {{- if and .Values.tls.enabled .Values.tls.certCAFilename }} sslcert={{ include "postgresql.v1.tlsCert" . }} sslkey={{ include "postgresql.v1.tlsCertKey" . }}{{- end }}" -h 127.0.0.1 -p {{ .Values.containerPorts.postgresql }} + {{- else }} + - exec pg_isready -U {{ default "postgres" $customUser | quote }} {{- if and .Values.tls.enabled .Values.tls.certCAFilename }} -d "sslcert={{ include "postgresql.v1.tlsCert" . }} sslkey={{ include "postgresql.v1.tlsCertKey" . }}"{{- end }} -h 127.0.0.1 -p {{ .Values.containerPorts.postgresql }} + {{- end }} + {{- end }} + {{- if .Values.primary.customReadinessProbe }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.primary.customReadinessProbe "context" $) | nindent 12 }} + {{- else if .Values.primary.readinessProbe.enabled }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.primary.readinessProbe "enabled") "context" $) | nindent 12 }} + exec: + command: + - /bin/sh + - -c + - -e + {{- include "postgresql.v1.readinessProbeCommand" . | nindent 16 }} + {{- end }} + {{- end }} + {{- if .Values.primary.resources }} + resources: {{- toYaml .Values.primary.resources | nindent 12 }} + {{- end }} + {{- if .Values.primary.lifecycleHooks }} + lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.primary.lifecycleHooks "context" $) | nindent 12 }} + {{- end }} + volumeMounts: + {{- if or .Values.primary.initdb.scriptsConfigMap .Values.primary.initdb.scripts }} + - name: custom-init-scripts + mountPath: /docker-entrypoint-initdb.d/ + {{- end }} + {{- if .Values.primary.initdb.scriptsSecret }} + - name: custom-init-scripts-secret + mountPath: /docker-entrypoint-initdb.d/secret + {{- end }} + {{- if or .Values.primary.extendedConfiguration .Values.primary.existingExtendedConfigmap }} + - name: postgresql-extended-config + mountPath: {{ .Values.primary.persistence.mountPath }}/conf/conf.d/ + {{- end }} + {{- if .Values.auth.usePasswordFiles }} + - name: postgresql-password + mountPath: /opt/bitnami/postgresql/secrets/ + {{- end }} + {{- if .Values.tls.enabled }} + - name: postgresql-certificates + mountPath: /opt/bitnami/postgresql/certs + readOnly: true + {{- end }} + {{- if .Values.shmVolume.enabled }} + - name: dshm + mountPath: /dev/shm + {{- end }} + {{- if .Values.primary.persistence.enabled }} + - name: data + mountPath: {{ .Values.primary.persistence.mountPath }} + {{- if .Values.primary.persistence.subPath }} + subPath: {{ .Values.primary.persistence.subPath }} + {{- end }} + {{- end }} + {{- if or .Values.primary.configuration .Values.primary.pgHbaConfiguration .Values.primary.existingConfigmap }} + - name: postgresql-config + mountPath: {{ .Values.primary.persistence.mountPath }}/conf + {{- end }} + {{- if .Values.primary.extraVolumeMounts }} + {{- include "common.tplvalues.render" (dict "value" .Values.primary.extraVolumeMounts "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.metrics.enabled }} + - name: metrics + image: {{ include "postgresql.v1.metrics.image" . }} + imagePullPolicy: {{ .Values.metrics.image.pullPolicy | quote }} + {{- if .Values.metrics.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.metrics.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} + {{- else if or .Values.metrics.customMetrics .Values.metrics.collectors }} + args: + {{- if .Values.metrics.customMetrics }} + - --extend.query-path + - /conf/custom-metrics.yaml + {{- end }} + {{- range $name, $enabled := .Values.metrics.collectors }} + - --{{ if not $enabled }}no-{{ end }}collector.{{ $name }} + {{- end }} + {{- end }} + env: + {{- $database := required "In order to enable metrics you need to specify a database (.Values.auth.database or .Values.global.postgresql.auth.database)" (include "postgresql.v1.database" .) }} + - name: DATA_SOURCE_URI + value: {{ printf "127.0.0.1:%d/%s?sslmode=disable" (int (include "postgresql.v1.service.port" .)) $database }} + {{- $pwdKey := ternary (include "postgresql.v1.adminPasswordKey" .) (include "postgresql.v1.userPasswordKey" .) (or (eq $customUser "postgres") (empty $customUser)) }} + {{- if .Values.auth.usePasswordFiles }} + - name: DATA_SOURCE_PASS_FILE + value: {{ printf "/opt/bitnami/postgresql/secrets/%s" $pwdKey }} + {{- else }} + - name: DATA_SOURCE_PASS + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" . }} + key: {{ $pwdKey }} + {{- end }} + - name: DATA_SOURCE_USER + value: {{ default "postgres" $customUser | quote }} + {{- if .Values.metrics.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.metrics.extraEnvVars "context" $) | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: {{ .Values.metrics.containerPorts.metrics }} + {{- if not .Values.diagnosticMode.enabled }} + {{- if .Values.metrics.customStartupProbe }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.metrics.customStartupProbe "context" $) | nindent 12 }} + {{- else if .Values.metrics.startupProbe.enabled }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.metrics.startupProbe "enabled") "context" $) | nindent 12 }} + tcpSocket: + port: http-metrics + {{- end }} + {{- if .Values.metrics.customLivenessProbe }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.metrics.customLivenessProbe "context" $) | nindent 12 }} + {{- else if .Values.metrics.livenessProbe.enabled }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.metrics.livenessProbe "enabled") "context" $) | nindent 12 }} + httpGet: + path: / + port: http-metrics + {{- end }} + {{- if .Values.metrics.customReadinessProbe }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.metrics.customReadinessProbe "context" $) | nindent 12 }} + {{- else if .Values.metrics.readinessProbe.enabled }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.metrics.readinessProbe "enabled") "context" $) | nindent 12 }} + httpGet: + path: / + port: http-metrics + {{- end }} + {{- end }} + volumeMounts: + {{- if .Values.auth.usePasswordFiles }} + - name: postgresql-password + mountPath: /opt/bitnami/postgresql/secrets/ + {{- end }} + {{- if .Values.metrics.customMetrics }} + - name: custom-metrics + mountPath: /conf + readOnly: true + {{- end }} + {{- if .Values.metrics.resources }} + resources: {{- toYaml .Values.metrics.resources | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.primary.sidecars }} + {{- include "common.tplvalues.render" ( dict "value" .Values.primary.sidecars "context" $ ) | nindent 8 }} + {{- end }} + volumes: + {{- if or .Values.primary.configuration .Values.primary.pgHbaConfiguration .Values.primary.existingConfigmap }} + - name: postgresql-config + configMap: + name: {{ include "postgresql.v1.primary.configmapName" . }} + {{- end }} + {{- if or .Values.primary.extendedConfiguration .Values.primary.existingExtendedConfigmap }} + - name: postgresql-extended-config + configMap: + name: {{ include "postgresql.v1.primary.extendedConfigmapName" . }} + {{- end }} + {{- if .Values.auth.usePasswordFiles }} + - name: postgresql-password + secret: + secretName: {{ include "postgresql.v1.secretName" . }} + {{- end }} + {{- if or .Values.primary.initdb.scriptsConfigMap .Values.primary.initdb.scripts }} + - name: custom-init-scripts + configMap: + name: {{ include "postgresql.v1.initdb.scriptsCM" . }} + {{- end }} + {{- if .Values.primary.initdb.scriptsSecret }} + - name: custom-init-scripts-secret + secret: + secretName: {{ tpl .Values.primary.initdb.scriptsSecret $ }} + {{- end }} + {{- if .Values.tls.enabled }} + - name: raw-certificates + secret: + secretName: {{ include "postgresql.v1.tlsSecretName" . }} + - name: postgresql-certificates + emptyDir: {} + {{- end }} + {{- if .Values.primary.extraVolumes }} + {{- include "common.tplvalues.render" ( dict "value" .Values.primary.extraVolumes "context" $ ) | nindent 8 }} + {{- end }} + {{- if and .Values.metrics.enabled .Values.metrics.customMetrics }} + - name: custom-metrics + configMap: + name: {{ printf "%s-metrics" (include "postgresql.v1.primary.fullname" .) }} + {{- end }} + {{- if .Values.shmVolume.enabled }} + - name: dshm + emptyDir: + medium: Memory + {{- if .Values.shmVolume.sizeLimit }} + sizeLimit: {{ .Values.shmVolume.sizeLimit }} + {{- end }} + {{- end }} + {{- if and .Values.primary.persistence.enabled .Values.primary.persistence.existingClaim }} + - name: data + persistentVolumeClaim: + claimName: {{ tpl .Values.primary.persistence.existingClaim $ }} + {{- else if not .Values.primary.persistence.enabled }} + - name: data + emptyDir: {} + {{- else }} + {{- if .Values.primary.persistentVolumeClaimRetentionPolicy.enabled }} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ .Values.primary.persistentVolumeClaimRetentionPolicy.whenDeleted }} + whenScaled: {{ .Values.primary.persistentVolumeClaimRetentionPolicy.whenScaled }} + {{- end }} + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data + {{- if .Values.primary.persistence.annotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.primary.persistence.annotations "context" $) | nindent 10 }} + {{- end }} + {{- if .Values.primary.persistence.labels }} + labels: {{- include "common.tplvalues.render" (dict "value" .Values.primary.persistence.labels "context" $) | nindent 10 }} + {{- end }} + spec: + accessModes: + {{- range .Values.primary.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + {{- if .Values.primary.persistence.dataSource }} + dataSource: {{- include "common.tplvalues.render" (dict "value" .Values.primary.persistence.dataSource "context" $) | nindent 10 }} + {{- end }} + resources: + requests: + storage: {{ .Values.primary.persistence.size | quote }} + {{- if .Values.primary.persistence.selector }} + selector: {{- include "common.tplvalues.render" (dict "value" .Values.primary.persistence.selector "context" $) | nindent 10 }} + {{- end }} + {{- include "common.storage.class" (dict "persistence" .Values.primary.persistence "global" .Values.global) | nindent 8 }} + {{- end }} diff --git a/charts/airflow/charts/postgresql/templates/primary/svc-headless.yaml b/charts/airflow/charts/postgresql/templates/primary/svc-headless.yaml new file mode 100644 index 0000000..b18565a --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/primary/svc-headless.yaml @@ -0,0 +1,36 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "postgresql.v1.primary.svc.headless" . }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: primary + annotations: + {{- if or .Values.primary.service.headless.annotations .Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.primary.service.headless.annotations .Values.commonAnnotations ) "context" . ) }} + {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} + # Use this annotation in addition to the actual publishNotReadyAddresses + # field below because the annotation will stop being respected soon but the + # field is broken in some versions of Kubernetes: + # https://github.com/kubernetes/kubernetes/issues/58662 + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" +spec: + type: ClusterIP + clusterIP: None + # We want all pods in the StatefulSet to have their addresses published for + # the sake of the other Postgresql pods even before they're ready, since they + # have to be able to talk to each other in order to become ready. + publishNotReadyAddresses: true + ports: + - name: tcp-postgresql + port: {{ template "postgresql.v1.service.port" . }} + targetPort: tcp-postgresql + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.primary.podLabels .Values.commonLabels ) "context" . ) }} + selector: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: primary diff --git a/charts/airflow/charts/postgresql/templates/primary/svc.yaml b/charts/airflow/charts/postgresql/templates/primary/svc.yaml new file mode 100644 index 0000000..90f7e46 --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/primary/svc.yaml @@ -0,0 +1,51 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "postgresql.v1.primary.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: primary + {{- if or .Values.commonAnnotations .Values.primary.service.annotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.primary.service.annotations .Values.commonAnnotations ) "context" . ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.primary.service.type }} + {{- if or (eq .Values.primary.service.type "LoadBalancer") (eq .Values.primary.service.type "NodePort") }} + externalTrafficPolicy: {{ .Values.primary.service.externalTrafficPolicy | quote }} + {{- end }} + {{- if and (eq .Values.primary.service.type "LoadBalancer") (not (empty .Values.primary.service.loadBalancerSourceRanges)) }} + loadBalancerSourceRanges: {{ .Values.primary.service.loadBalancerSourceRanges | toJson}} + {{- end }} + {{- if and (eq .Values.primary.service.type "LoadBalancer") (not (empty .Values.primary.service.loadBalancerIP)) }} + loadBalancerIP: {{ .Values.primary.service.loadBalancerIP }} + {{- end }} + {{- if and .Values.primary.service.clusterIP (eq .Values.primary.service.type "ClusterIP") }} + clusterIP: {{ .Values.primary.service.clusterIP }} + {{- end }} + {{- if .Values.primary.service.sessionAffinity }} + sessionAffinity: {{ .Values.primary.service.sessionAffinity }} + {{- end }} + {{- if .Values.primary.service.sessionAffinityConfig }} + sessionAffinityConfig: {{- include "common.tplvalues.render" (dict "value" .Values.primary.service.sessionAffinityConfig "context" $) | nindent 4 }} + {{- end }} + ports: + - name: tcp-postgresql + port: {{ template "postgresql.v1.service.port" . }} + targetPort: tcp-postgresql + {{- if and (or (eq .Values.primary.service.type "NodePort") (eq .Values.primary.service.type "LoadBalancer")) (not (empty .Values.primary.service.nodePorts.postgresql)) }} + nodePort: {{ .Values.primary.service.nodePorts.postgresql }} + {{- else if eq .Values.primary.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- if .Values.primary.service.extraPorts }} + {{- include "common.tplvalues.render" (dict "value" .Values.primary.service.extraPorts "context" $) | nindent 4 }} + {{- end }} + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.primary.podLabels .Values.commonLabels ) "context" . ) }} + selector: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: primary diff --git a/charts/airflow/charts/postgresql/templates/prometheusrule.yaml b/charts/airflow/charts/postgresql/templates/prometheusrule.yaml new file mode 100644 index 0000000..6cdb087 --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/prometheusrule.yaml @@ -0,0 +1,22 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.metrics.enabled .Values.metrics.prometheusRule.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ default .Release.Namespace .Values.metrics.prometheusRule.namespace | quote }} + {{- $labels := include "common.tplvalues.merge" ( dict "values" ( list .Values.metrics.prometheusRule.labels .Values.commonLabels ) "context" . ) }} + labels: {{- include "common.labels.standard" ( dict "customLabels" $labels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: metrics + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + groups: + - name: {{ include "common.names.fullname" . }} + rules: {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.prometheusRule.rules "context" $ ) | nindent 8 }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/psp.yaml b/charts/airflow/charts/postgresql/templates/psp.yaml new file mode 100644 index 0000000..2cc1bbf --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/psp.yaml @@ -0,0 +1,42 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and (include "common.capabilities.psp.supported" .) .Values.psp.create }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + privileged: false + volumes: + - 'configMap' + - 'secret' + - 'persistentVolumeClaim' + - 'emptyDir' + - 'projected' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/read/extended-configmap.yaml b/charts/airflow/charts/postgresql/templates/read/extended-configmap.yaml new file mode 100644 index 0000000..efa87bb --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/read/extended-configmap.yaml @@ -0,0 +1,20 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if (include "postgresql.v1.readReplicas.createExtendedConfigmap" .) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-extended-configuration" (include "postgresql.v1.readReplica.fullname" .) }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: read + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + override.conf: |- + {{- include "common.tplvalues.render" ( dict "value" .Values.readReplicas.extendedConfiguration "context" $ ) | nindent 4 }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/read/metrics-configmap.yaml b/charts/airflow/charts/postgresql/templates/read/metrics-configmap.yaml new file mode 100644 index 0000000..a1e06bf --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/read/metrics-configmap.yaml @@ -0,0 +1,18 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.metrics.enabled .Values.metrics.customMetrics (eq .Values.architecture "replication") }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-metrics" (include "postgresql.v1.readReplica.fullname" .) }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + custom-metrics.yaml: {{ toYaml .Values.metrics.customMetrics | quote }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/read/metrics-svc.yaml b/charts/airflow/charts/postgresql/templates/read/metrics-svc.yaml new file mode 100644 index 0000000..e9f13e0 --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/read/metrics-svc.yaml @@ -0,0 +1,31 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.metrics.enabled (eq .Values.architecture "replication") }} +apiVersion: v1 +kind: Service +metadata: + name: {{ printf "%s-metrics" (include "postgresql.v1.readReplica.fullname" .) }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: metrics-read + {{- if or .Values.commonAnnotations .Values.metrics.service.annotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.metrics.service.annotations .Values.commonAnnotations ) "context" . ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + sessionAffinity: {{ .Values.metrics.service.sessionAffinity }} + {{- if .Values.metrics.service.clusterIP }} + clusterIP: {{ .Values.metrics.service.clusterIP }} + {{- end }} + ports: + - name: http-metrics + port: {{ .Values.metrics.service.ports.metrics }} + targetPort: http-metrics + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.readReplicas.podLabels .Values.commonLabels ) "context" . ) }} + selector: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: read +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/read/networkpolicy.yaml b/charts/airflow/charts/postgresql/templates/read/networkpolicy.yaml new file mode 100644 index 0000000..79d3a5a --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/read/networkpolicy.yaml @@ -0,0 +1,39 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.networkPolicy.enabled (eq .Values.architecture "replication") .Values.networkPolicy.ingressRules.readReplicasAccessOnlyFrom.enabled }} +apiVersion: {{ include "common.capabilities.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + name: {{ printf "%s-ingress" (include "postgresql.v1.readReplica.fullname" .) }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: read + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.readReplicas.podLabels .Values.commonLabels ) "context" . ) }} + podSelector: + matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} + app.kubernetes.io/component: read + ingress: + {{- if and .Values.networkPolicy.ingressRules.readReplicasAccessOnlyFrom.enabled (or .Values.networkPolicy.ingressRules.readReplicasAccessOnlyFrom.namespaceSelector .Values.networkPolicy.ingressRules.readReplicasAccessOnlyFrom.podSelector) }} + - from: + {{- if .Values.networkPolicy.ingressRules.readReplicasAccessOnlyFrom.namespaceSelector }} + - namespaceSelector: + matchLabels: {{- include "common.tplvalues.render" (dict "value" .Values.networkPolicy.ingressRules.readReplicasAccessOnlyFrom.namespaceSelector "context" $) | nindent 14 }} + {{- end }} + {{- if .Values.networkPolicy.ingressRules.readReplicasAccessOnlyFrom.podSelector }} + - podSelector: + matchLabels: {{- include "common.tplvalues.render" (dict "value" .Values.networkPolicy.ingressRules.readReplicasAccessOnlyFrom.podSelector "context" $) | nindent 14 }} + {{- end }} + ports: + - port: {{ .Values.containerPorts.postgresql }} + {{- end }} + {{- if .Values.networkPolicy.ingressRules.readReplicasAccessOnlyFrom.customRules }} + {{- include "common.tplvalues.render" (dict "value" .Values.networkPolicy.ingressRules.readReplicasAccessOnlyFrom.customRules "context" $) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/read/servicemonitor.yaml b/charts/airflow/charts/postgresql/templates/read/servicemonitor.yaml new file mode 100644 index 0000000..845734b --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/read/servicemonitor.yaml @@ -0,0 +1,46 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled (eq .Values.architecture "replication") }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "postgresql.v1.readReplica.fullname" . }} + namespace: {{ default .Release.Namespace .Values.metrics.serviceMonitor.namespace | quote }} + {{- $labels := include "common.tplvalues.merge" ( dict "values" ( list .Values.metrics.serviceMonitor.labels .Values.commonLabels ) "context" . ) }} + labels: {{- include "common.labels.standard" ( dict "customLabels" $labels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: metrics-read + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + {{- if .Values.metrics.serviceMonitor.jobLabel }} + jobLabel: {{ .Values.metrics.serviceMonitor.jobLabel }} + {{- end }} + selector: + {{- $svcLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.metrics.serviceMonitor.selector .Values.commonLabels ) "context" . ) }} + matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $svcLabels "context" $ ) | nindent 6 }} + app.kubernetes.io/component: metrics-read + endpoints: + - port: http-metrics + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.relabelings }} + relabelings: {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.serviceMonitor.relabelings "context" $) | nindent 6 }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.serviceMonitor.metricRelabelings "context" $) | nindent 6 }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.honorLabels }} + honorLabels: {{ .Values.metrics.serviceMonitor.honorLabels }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace | quote }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/read/statefulset.yaml b/charts/airflow/charts/postgresql/templates/read/statefulset.yaml new file mode 100644 index 0000000..8268700 --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/read/statefulset.yaml @@ -0,0 +1,552 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if eq .Values.architecture "replication" }} +{{- $customUser := include "postgresql.v1.username" . }} +apiVersion: {{ include "common.capabilities.statefulset.apiVersion" . }} +kind: StatefulSet +metadata: + name: {{ include "postgresql.v1.readReplica.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + {{- $labels := include "common.tplvalues.merge" ( dict "values" ( list .Values.readReplicas.labels .Values.commonLabels ) "context" . ) }} + labels: {{- include "common.labels.standard" ( dict "customLabels" $labels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: read + {{- if or .Values.commonAnnotations .Values.readReplicas.annotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.readReplicas.annotations .Values.commonAnnotations ) "context" . ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.readReplicas.replicaCount }} + serviceName: {{ include "postgresql.v1.readReplica.svc.headless" . }} + {{- if .Values.readReplicas.updateStrategy }} + updateStrategy: {{- toYaml .Values.readReplicas.updateStrategy | nindent 4 }} + {{- end }} + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.readReplicas.podLabels .Values.commonLabels ) "context" . ) }} + selector: + matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} + app.kubernetes.io/component: read + template: + metadata: + name: {{ include "postgresql.v1.readReplica.fullname" . }} + labels: {{- include "common.labels.standard" ( dict "customLabels" $podLabels "context" $ ) | nindent 8 }} + app.kubernetes.io/component: read + {{- if or (include "postgresql.v1.readReplicas.createExtendedConfigmap" .) .Values.readReplicas.podAnnotations }} + annotations: + {{- if (include "postgresql.v1.readReplicas.createExtendedConfigmap" .) }} + checksum/extended-configuration: {{ pick (include (print $.Template.BasePath "/primary/extended-configmap.yaml") . | fromYaml) "data" | toYaml | sha256sum }} + {{- end }} + {{- if .Values.readReplicas.podAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.readReplicas.podAnnotations "context" $ ) | nindent 8 }} + {{- end }} + {{- end }} + spec: + {{- if .Values.readReplicas.extraPodSpec }} + {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.extraPodSpec "context" $) | nindent 6 }} + {{- end }} + serviceAccountName: {{ include "postgresql.v1.serviceAccountName" . }} + {{- include "postgresql.v1.imagePullSecrets" . | nindent 6 }} + {{- if .Values.readReplicas.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.hostAliases "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.readReplicas.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.affinity "context" $) | nindent 8 }} + {{- else }} + affinity: + podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.readReplicas.podAffinityPreset "component" "read" "customLabels" $podLabels "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.readReplicas.podAntiAffinityPreset "component" "read" "customLabels" $podLabels "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.readReplicas.nodeAffinityPreset.type "key" .Values.readReplicas.nodeAffinityPreset.key "values" .Values.readReplicas.nodeAffinityPreset.values) | nindent 10 }} + {{- end }} + {{- if .Values.readReplicas.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.readReplicas.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.readReplicas.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.topologySpreadConstraints "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.readReplicas.priorityClassName }} + priorityClassName: {{ .Values.readReplicas.priorityClassName }} + {{- end }} + {{- if .Values.readReplicas.schedulerName }} + schedulerName: {{ .Values.readReplicas.schedulerName | quote }} + {{- end }} + {{- if .Values.readReplicas.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ .Values.readReplicas.terminationGracePeriodSeconds }} + {{- end }} + {{- if .Values.readReplicas.podSecurityContext.enabled }} + securityContext: {{- omit .Values.readReplicas.podSecurityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + hostNetwork: {{ .Values.readReplicas.hostNetwork }} + hostIPC: {{ .Values.readReplicas.hostIPC }} + {{- if or (and .Values.tls.enabled (not .Values.volumePermissions.enabled)) (and .Values.volumePermissions.enabled (or .Values.readReplicas.persistence.enabled .Values.shmVolume.enabled)) .Values.readReplicas.initContainers }} + initContainers: + {{- if and .Values.tls.enabled (not .Values.volumePermissions.enabled) }} + - name: copy-certs + image: {{ include "postgresql.v1.volumePermissions.image" . }} + imagePullPolicy: {{ .Values.volumePermissions.image.pullPolicy | quote }} + {{- if .Values.readReplicas.resources }} + resources: {{- toYaml .Values.readReplicas.resources | nindent 12 }} + {{- end }} + # We don't require a privileged container in this case + {{- if .Values.readReplicas.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.readReplicas.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + command: + - /bin/sh + - -ec + - | + cp /tmp/certs/* /opt/bitnami/postgresql/certs/ + chmod 600 {{ include "postgresql.v1.tlsCertKey" . }} + volumeMounts: + - name: raw-certificates + mountPath: /tmp/certs + - name: postgresql-certificates + mountPath: /opt/bitnami/postgresql/certs + {{- else if and .Values.volumePermissions.enabled (or .Values.readReplicas.persistence.enabled .Values.shmVolume.enabled) }} + - name: init-chmod-data + image: {{ include "postgresql.v1.volumePermissions.image" . }} + imagePullPolicy: {{ .Values.volumePermissions.image.pullPolicy | quote }} + {{- if .Values.readReplicas.resources }} + resources: {{- toYaml .Values.readReplicas.resources | nindent 12 }} + {{- end }} + command: + - /bin/sh + - -ec + - | + {{- if .Values.readReplicas.persistence.enabled }} + {{- if eq ( toString ( .Values.volumePermissions.containerSecurityContext.runAsUser )) "auto" }} + chown `id -u`:`id -G | cut -d " " -f2` {{ .Values.readReplicas.persistence.mountPath }} + {{- else }} + chown {{ .Values.readReplicas.containerSecurityContext.runAsUser }}:{{ .Values.readReplicas.podSecurityContext.fsGroup }} {{ .Values.readReplicas.persistence.mountPath }} + {{- end }} + mkdir -p {{ .Values.readReplicas.persistence.mountPath }}/data {{- if (include "postgresql.v1.mountConfigurationCM" .) }} {{ .Values.readReplicas.persistence.mountPath }}/conf {{- end }} + chmod 700 {{ .Values.readReplicas.persistence.mountPath }}/data {{- if (include "postgresql.v1.mountConfigurationCM" .) }} {{ .Values.readReplicas.persistence.mountPath }}/conf {{- end }} + find {{ .Values.readReplicas.persistence.mountPath }} -mindepth 1 -maxdepth 1 {{- if not (include "postgresql.v1.mountConfigurationCM" .) }} -not -name "conf" {{- end }} -not -name ".snapshot" -not -name "lost+found" | \ + {{- if eq ( toString ( .Values.volumePermissions.containerSecurityContext.runAsUser )) "auto" }} + xargs -r chown -R `id -u`:`id -G | cut -d " " -f2` + {{- else }} + xargs -r chown -R {{ .Values.readReplicas.containerSecurityContext.runAsUser }}:{{ .Values.readReplicas.podSecurityContext.fsGroup }} + {{- end }} + {{- end }} + {{- if .Values.shmVolume.enabled }} + chmod -R 777 /dev/shm + {{- end }} + {{- if .Values.tls.enabled }} + cp /tmp/certs/* /opt/bitnami/postgresql/certs/ + {{- if eq ( toString ( .Values.volumePermissions.containerSecurityContext.runAsUser )) "auto" }} + chown -R `id -u`:`id -G | cut -d " " -f2` /opt/bitnami/postgresql/certs/ + {{- else }} + chown -R {{ .Values.readReplicas.containerSecurityContext.runAsUser }}:{{ .Values.readReplicas.podSecurityContext.fsGroup }} /opt/bitnami/postgresql/certs/ + {{- end }} + chmod 600 {{ include "postgresql.v1.tlsCertKey" . }} + {{- end }} + {{- if eq ( toString ( .Values.volumePermissions.containerSecurityContext.runAsUser )) "auto" }} + securityContext: {{- omit .Values.volumePermissions.containerSecurityContext "runAsUser" | toYaml | nindent 12 }} + {{- else }} + securityContext: {{- .Values.volumePermissions.containerSecurityContext | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + {{ if .Values.readReplicas.persistence.enabled }} + - name: data + mountPath: {{ .Values.readReplicas.persistence.mountPath }} + {{- if .Values.readReplicas.persistence.subPath }} + subPath: {{ .Values.readReplicas.persistence.subPath }} + {{- end }} + {{- end }} + {{- if .Values.shmVolume.enabled }} + - name: dshm + mountPath: /dev/shm + {{- end }} + {{- if .Values.tls.enabled }} + - name: raw-certificates + mountPath: /tmp/certs + - name: postgresql-certificates + mountPath: /opt/bitnami/postgresql/certs + {{- end }} + {{- end }} + {{- if .Values.readReplicas.initContainers }} + {{- include "common.tplvalues.render" ( dict "value" .Values.readReplicas.initContainers "context" $ ) | nindent 8 }} + {{- end }} + {{- end }} + containers: + - name: postgresql + image: {{ include "postgresql.v1.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.readReplicas.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.readReplicas.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} + {{- else if .Values.readReplicas.command }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.command "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.diagnosticMode.enabled }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} + {{- else if .Values.readReplicas.args }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.args "context" $) | nindent 12 }} + {{- end }} + env: + - name: BITNAMI_DEBUG + value: {{ ternary "true" "false" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} + - name: POSTGRESQL_PORT_NUMBER + value: {{ .Values.containerPorts.postgresql | quote }} + - name: POSTGRESQL_VOLUME_DIR + value: {{ .Values.readReplicas.persistence.mountPath | quote }} + {{- if .Values.readReplicas.persistence.mountPath }} + - name: PGDATA + value: {{ .Values.postgresqlDataDir | quote }} + {{- end }} + # Authentication + {{- if or (eq $customUser "postgres") (empty $customUser) }} + {{- if .Values.auth.enablePostgresUser }} + {{- if .Values.auth.usePasswordFiles }} + - name: POSTGRES_PASSWORD_FILE + value: {{ printf "/opt/bitnami/postgresql/secrets/%s" (include "postgresql.v1.adminPasswordKey" .) }} + {{- else }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" . }} + key: {{ include "postgresql.v1.adminPasswordKey" . }} + {{- end }} + {{- else }} + - name: ALLOW_EMPTY_PASSWORD + value: "true" + {{- end }} + {{- else }} + - name: POSTGRES_USER + value: {{ $customUser | quote }} + {{- if .Values.auth.usePasswordFiles }} + - name: POSTGRES_PASSWORD_FILE + value: {{ printf "/opt/bitnami/postgresql/secrets/%s" (include "postgresql.v1.userPasswordKey" .) }} + {{- else }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" . }} + key: {{ include "postgresql.v1.userPasswordKey" . }} + {{- end }} + {{- if .Values.auth.enablePostgresUser }} + {{- if .Values.auth.usePasswordFiles }} + - name: POSTGRES_POSTGRES_PASSWORD_FILE + value: {{ printf "/opt/bitnami/postgresql/secrets/%s" (include "postgresql.v1.adminPasswordKey" .) }} + {{- else }} + - name: POSTGRES_POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" . }} + key: {{ include "postgresql.v1.adminPasswordKey" . }} + {{- end }} + {{- end }} + {{- end }} + # Replication + - name: POSTGRES_REPLICATION_MODE + value: "slave" + - name: POSTGRES_REPLICATION_USER + value: {{ .Values.auth.replicationUsername | quote }} + {{- if .Values.auth.usePasswordFiles }} + - name: POSTGRES_REPLICATION_PASSWORD_FILE + value: {{ printf "/opt/bitnami/postgresql/secrets/%s" (include "postgresql.v1.replicationPasswordKey" .) }} + {{- else }} + - name: POSTGRES_REPLICATION_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" . }} + key: {{ include "postgresql.v1.replicationPasswordKey" . }} + {{- end }} + - name: POSTGRES_CLUSTER_APP_NAME + value: {{ .Values.replication.applicationName }} + - name: POSTGRES_MASTER_HOST + value: {{ include "postgresql.v1.primary.fullname" . }} + - name: POSTGRES_MASTER_PORT_NUMBER + value: {{ include "postgresql.v1.service.port" . | quote }} + # TLS + - name: POSTGRESQL_ENABLE_TLS + value: {{ ternary "yes" "no" .Values.tls.enabled | quote }} + {{- if .Values.tls.enabled }} + - name: POSTGRESQL_TLS_PREFER_SERVER_CIPHERS + value: {{ ternary "yes" "no" .Values.tls.preferServerCiphers | quote }} + - name: POSTGRESQL_TLS_CERT_FILE + value: {{ include "postgresql.v1.tlsCert" . }} + - name: POSTGRESQL_TLS_KEY_FILE + value: {{ include "postgresql.v1.tlsCertKey" . }} + {{- if .Values.tls.certCAFilename }} + - name: POSTGRESQL_TLS_CA_FILE + value: {{ include "postgresql.v1.tlsCACert" . }} + {{- end }} + {{- if .Values.tls.crlFilename }} + - name: POSTGRESQL_TLS_CRL_FILE + value: {{ include "postgresql.v1.tlsCRL" . }} + {{- end }} + {{- end }} + # Audit + - name: POSTGRESQL_LOG_HOSTNAME + value: {{ .Values.audit.logHostname | quote }} + - name: POSTGRESQL_LOG_CONNECTIONS + value: {{ .Values.audit.logConnections | quote }} + - name: POSTGRESQL_LOG_DISCONNECTIONS + value: {{ .Values.audit.logDisconnections | quote }} + {{- if .Values.audit.logLinePrefix }} + - name: POSTGRESQL_LOG_LINE_PREFIX + value: {{ .Values.audit.logLinePrefix | quote }} + {{- end }} + {{- if .Values.audit.logTimezone }} + - name: POSTGRESQL_LOG_TIMEZONE + value: {{ .Values.audit.logTimezone | quote }} + {{- end }} + {{- if .Values.audit.pgAuditLog }} + - name: POSTGRESQL_PGAUDIT_LOG + value: {{ .Values.audit.pgAuditLog | quote }} + {{- end }} + - name: POSTGRESQL_PGAUDIT_LOG_CATALOG + value: {{ .Values.audit.pgAuditLogCatalog | quote }} + # Others + - name: POSTGRESQL_CLIENT_MIN_MESSAGES + value: {{ .Values.audit.clientMinMessages | quote }} + - name: POSTGRESQL_SHARED_PRELOAD_LIBRARIES + value: {{ .Values.postgresqlSharedPreloadLibraries | quote }} + {{- if .Values.readReplicas.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.extraEnvVars "context" $) | nindent 12 }} + {{- end }} + {{- if or .Values.readReplicas.extraEnvVarsCM .Values.readReplicas.extraEnvVarsSecret }} + envFrom: + {{- if .Values.readReplicas.extraEnvVarsCM }} + - configMapRef: + name: {{ .Values.readReplicas.extraEnvVarsCM }} + {{- end }} + {{- if .Values.readReplicas.extraEnvVarsSecret }} + - secretRef: + name: {{ .Values.readReplicas.extraEnvVarsSecret }} + {{- end }} + {{- end }} + ports: + - name: tcp-postgresql + containerPort: {{ .Values.containerPorts.postgresql }} + {{- if not .Values.diagnosticMode.enabled }} + {{- if .Values.readReplicas.customStartupProbe }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.customStartupProbe "context" $) | nindent 12 }} + {{- else if .Values.readReplicas.startupProbe.enabled }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.readReplicas.startupProbe "enabled") "context" $) | nindent 12 }} + exec: + command: + - /bin/sh + - -c + {{- if (include "postgresql.v1.database" .) }} + - exec pg_isready -U {{ default "postgres" $customUser| quote }} -d "dbname={{ include "postgresql.v1.database" . }} {{- if and .Values.tls.enabled .Values.tls.certCAFilename }} sslcert={{ include "postgresql.v1.tlsCert" . }} sslkey={{ include "postgresql.v1.tlsCertKey" . }}{{- end }}" -h 127.0.0.1 -p {{ .Values.containerPorts.postgresql }} + {{- else }} + - exec pg_isready -U {{ default "postgres" $customUser | quote }} {{- if and .Values.tls.enabled .Values.tls.certCAFilename }} -d "sslcert={{ include "postgresql.v1.tlsCert" . }} sslkey={{ include "postgresql.v1.tlsCertKey" . }}"{{- end }} -h 127.0.0.1 -p {{ .Values.containerPorts.postgresql }} + {{- end }} + {{- end }} + {{- if .Values.readReplicas.customLivenessProbe }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.customLivenessProbe "context" $) | nindent 12 }} + {{- else if .Values.readReplicas.livenessProbe.enabled }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.readReplicas.livenessProbe "enabled") "context" $) | nindent 12 }} + exec: + command: + - /bin/sh + - -c + {{- if (include "postgresql.v1.database" .) }} + - exec pg_isready -U {{ default "postgres" $customUser | quote }} -d "dbname={{ include "postgresql.v1.database" . }} {{- if and .Values.tls.enabled .Values.tls.certCAFilename }} sslcert={{ include "postgresql.v1.tlsCert" . }} sslkey={{ include "postgresql.v1.tlsCertKey" . }}{{- end }}" -h 127.0.0.1 -p {{ .Values.containerPorts.postgresql }} + {{- else }} + - exec pg_isready -U {{default "postgres" $customUser | quote }} {{- if and .Values.tls.enabled .Values.tls.certCAFilename }} -d "sslcert={{ include "postgresql.v1.tlsCert" . }} sslkey={{ include "postgresql.v1.tlsCertKey" . }}"{{- end }} -h 127.0.0.1 -p {{ .Values.containerPorts.postgresql }} + {{- end }} + {{- end }} + {{- if .Values.readReplicas.customReadinessProbe }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.customReadinessProbe "context" $) | nindent 12 }} + {{- else if .Values.readReplicas.readinessProbe.enabled }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.readReplicas.readinessProbe "enabled") "context" $) | nindent 12 }} + exec: + command: + - /bin/sh + - -c + - -e + {{- include "postgresql.v1.readinessProbeCommand" . | nindent 16 }} + {{- end }} + {{- end }} + {{- if .Values.readReplicas.resources }} + resources: {{- toYaml .Values.readReplicas.resources | nindent 12 }} + {{- end }} + {{- if .Values.readReplicas.lifecycleHooks }} + lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.lifecycleHooks "context" $) | nindent 12 }} + {{- end }} + volumeMounts: + {{- if .Values.auth.usePasswordFiles }} + - name: postgresql-password + mountPath: /opt/bitnami/postgresql/secrets/ + {{- end }} + {{- if .Values.readReplicas.extendedConfiguration }} + - name: postgresql-extended-config + mountPath: {{ .Values.readReplicas.persistence.mountPath }}/conf/conf.d/ + {{- end }} + {{- if .Values.tls.enabled }} + - name: postgresql-certificates + mountPath: /opt/bitnami/postgresql/certs + readOnly: true + {{- end }} + {{- if .Values.shmVolume.enabled }} + - name: dshm + mountPath: /dev/shm + {{- end }} + {{- if .Values.readReplicas.persistence.enabled }} + - name: data + mountPath: {{ .Values.readReplicas.persistence.mountPath }} + {{- if .Values.readReplicas.persistence.subPath }} + subPath: {{ .Values.readReplicas.persistence.subPath }} + {{- end }} + {{- end }} + {{- if .Values.readReplicas.extraVolumeMounts }} + {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.extraVolumeMounts "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.metrics.enabled }} + - name: metrics + image: {{ include "postgresql.v1.metrics.image" . }} + imagePullPolicy: {{ .Values.metrics.image.pullPolicy | quote }} + {{- if .Values.metrics.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.metrics.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} + {{- else if .Values.metrics.customMetrics }} + args: [ "--extend.query-path", "/conf/custom-metrics.yaml" ] + {{- end }} + env: + {{- $database := required "In order to enable metrics you need to specify a database (.Values.auth.database or .Values.global.postgresql.auth.database)" (include "postgresql.v1.database" .) }} + - name: DATA_SOURCE_URI + value: {{ printf "127.0.0.1:%d/%s?sslmode=disable" (int (include "postgresql.v1.service.port" .)) $database }} + {{- if .Values.auth.usePasswordFiles }} + - name: DATA_SOURCE_PASS_FILE + value: {{ printf "/opt/bitnami/postgresql/secrets/%s" (include "postgresql.v1.userPasswordKey" .) }} + {{- else }} + - name: DATA_SOURCE_PASS + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" . }} + key: {{ include "postgresql.v1.userPasswordKey" . }} + {{- end }} + - name: DATA_SOURCE_USER + value: {{ default "postgres" $customUser | quote }} + {{- if .Values.metrics.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.metrics.extraEnvVars "context" $) | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: {{ .Values.metrics.containerPorts.metrics }} + {{- if not .Values.diagnosticMode.enabled }} + {{- if .Values.metrics.customStartupProbe }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.metrics.customStartupProbe "context" $) | nindent 12 }} + {{- else if .Values.metrics.startupProbe.enabled }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.metrics.startupProbe "enabled") "context" $) | nindent 12 }} + tcpSocket: + port: http-metrics + {{- end }} + {{- if .Values.metrics.customLivenessProbe }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.metrics.customLivenessProbe "context" $) | nindent 12 }} + {{- else if .Values.metrics.livenessProbe.enabled }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.metrics.livenessProbe "enabled") "context" $) | nindent 12 }} + httpGet: + path: / + port: http-metrics + {{- end }} + {{- if .Values.metrics.customReadinessProbe }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.metrics.customReadinessProbe "context" $) | nindent 12 }} + {{- else if .Values.metrics.readinessProbe.enabled }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.metrics.readinessProbe "enabled") "context" $) | nindent 12 }} + httpGet: + path: / + port: http-metrics + {{- end }} + {{- end }} + volumeMounts: + {{- if .Values.auth.usePasswordFiles }} + - name: postgresql-password + mountPath: /opt/bitnami/postgresql/secrets/ + {{- end }} + {{- if .Values.metrics.customMetrics }} + - name: custom-metrics + mountPath: /conf + readOnly: true + {{- end }} + {{- if .Values.metrics.resources }} + resources: {{- toYaml .Values.metrics.resources | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.readReplicas.sidecars }} + {{- include "common.tplvalues.render" ( dict "value" .Values.readReplicas.sidecars "context" $ ) | nindent 8 }} + {{- end }} + volumes: + {{- if .Values.readReplicas.extendedConfiguration }} + - name: postgresql-extended-config + configMap: + name: {{ include "postgresql.v1.readReplicas.extendedConfigmapName" . }} + {{- end }} + {{- if .Values.auth.usePasswordFiles }} + - name: postgresql-password + secret: + secretName: {{ include "postgresql.v1.secretName" . }} + {{- end }} + {{- if .Values.tls.enabled }} + - name: raw-certificates + secret: + secretName: {{ include "postgresql.v1.tlsSecretName" . }} + - name: postgresql-certificates + emptyDir: {} + {{- end }} + {{- if and .Values.metrics.enabled .Values.metrics.customMetrics }} + - name: custom-metrics + configMap: + name: {{ printf "%s-metrics" (include "postgresql.v1.readReplica.fullname" .) }} + {{- end }} + {{- if .Values.shmVolume.enabled }} + - name: dshm + emptyDir: + medium: Memory + {{- if .Values.shmVolume.sizeLimit }} + sizeLimit: {{ .Values.shmVolume.sizeLimit }} + {{- end }} + {{- end }} + {{- if .Values.readReplicas.extraVolumes }} + {{- include "common.tplvalues.render" ( dict "value" .Values.readReplicas.extraVolumes "context" $ ) | nindent 8 }} + {{- end }} + {{- if and .Values.readReplicas.persistence.enabled .Values.readReplicas.persistence.existingClaim }} + - name: data + persistentVolumeClaim: + claimName: {{ tpl .Values.readReplicas.persistence.existingClaim $ }} + {{- else if not .Values.readReplicas.persistence.enabled }} + - name: data + emptyDir: {} + {{- else }} + {{- if .Values.readReplicas.persistentVolumeClaimRetentionPolicy.enabled }} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ .Values.readReplicas.persistentVolumeClaimRetentionPolicy.whenDeleted }} + whenScaled: {{ .Values.readReplicas.persistentVolumeClaimRetentionPolicy.whenScaled }} + {{- end }} + volumeClaimTemplates: + - metadata: + name: data + {{- if .Values.readReplicas.persistence.annotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.persistence.annotations "context" $) | nindent 10 }} + {{- end }} + {{- if .Values.readReplicas.persistence.labels }} + labels: {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.persistence.labels "context" $) | nindent 10 }} + {{- end }} + spec: + accessModes: + {{- range .Values.readReplicas.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + {{- if .Values.readReplicas.persistence.dataSource }} + dataSource: {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.persistence.dataSource "context" $) | nindent 10 }} + {{- end }} + resources: + requests: + storage: {{ .Values.readReplicas.persistence.size | quote }} + {{- if .Values.readReplicas.persistence.selector }} + selector: {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.persistence.selector "context" $) | nindent 10 }} + {{- end -}} + {{- include "common.storage.class" (dict "persistence" .Values.readReplicas.persistence "global" .Values.global) | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/read/svc-headless.yaml b/charts/airflow/charts/postgresql/templates/read/svc-headless.yaml new file mode 100644 index 0000000..249af5f --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/read/svc-headless.yaml @@ -0,0 +1,38 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if eq .Values.architecture "replication" }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "postgresql.v1.readReplica.svc.headless" . }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: read + annotations: + {{- if or .Values.readReplicas.service.headless.annotations .Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.readReplicas.service.headless.annotations .Values.commonAnnotations ) "context" . ) }} + {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} + # Use this annotation in addition to the actual publishNotReadyAddresses + # field below because the annotation will stop being respected soon but the + # field is broken in some versions of Kubernetes: + # https://github.com/kubernetes/kubernetes/issues/58662 + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" +spec: + type: ClusterIP + clusterIP: None + # We want all pods in the StatefulSet to have their addresses published for + # the sake of the other Postgresql pods even before they're ready, since they + # have to be able to talk to each other in order to become ready. + publishNotReadyAddresses: true + ports: + - name: tcp-postgresql + port: {{ include "postgresql.v1.readReplica.service.port" . }} + targetPort: tcp-postgresql + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.readReplicas.podLabels .Values.commonLabels ) "context" . ) }} + selector: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: read +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/read/svc.yaml b/charts/airflow/charts/postgresql/templates/read/svc.yaml new file mode 100644 index 0000000..d92c523 --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/read/svc.yaml @@ -0,0 +1,53 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if eq .Values.architecture "replication" }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "postgresql.v1.readReplica.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: read + {{- if or .Values.commonAnnotations .Values.readReplicas.service.annotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.readReplicas.service.annotations .Values.commonAnnotations ) "context" . ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.readReplicas.service.type }} + {{- if or (eq .Values.readReplicas.service.type "LoadBalancer") (eq .Values.readReplicas.service.type "NodePort") }} + externalTrafficPolicy: {{ .Values.readReplicas.service.externalTrafficPolicy | quote }} + {{- end }} + {{- if and (eq .Values.readReplicas.service.type "LoadBalancer") (not (empty .Values.readReplicas.service.loadBalancerSourceRanges)) }} + loadBalancerSourceRanges: {{ .Values.readReplicas.service.loadBalancerSourceRanges }} + {{- end }} + {{- if and (eq .Values.readReplicas.service.type "LoadBalancer") (not (empty .Values.readReplicas.service.loadBalancerIP)) }} + loadBalancerIP: {{ .Values.readReplicas.service.loadBalancerIP }} + {{- end }} + {{- if and .Values.readReplicas.service.clusterIP (eq .Values.readReplicas.service.type "ClusterIP") }} + clusterIP: {{ .Values.readReplicas.service.clusterIP }} + {{- end }} + {{- if .Values.readReplicas.service.sessionAffinity }} + sessionAffinity: {{ .Values.readReplicas.service.sessionAffinity }} + {{- end }} + {{- if .Values.readReplicas.service.sessionAffinityConfig }} + sessionAffinityConfig: {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.service.sessionAffinityConfig "context" $) | nindent 4 }} + {{- end }} + ports: + - name: tcp-postgresql + port: {{ include "postgresql.v1.readReplica.service.port" . }} + targetPort: tcp-postgresql + {{- if and (or (eq .Values.readReplicas.service.type "NodePort") (eq .Values.readReplicas.service.type "LoadBalancer")) (not (empty .Values.readReplicas.service.nodePorts.postgresql)) }} + nodePort: {{ .Values.readReplicas.service.nodePorts.postgresql }} + {{- else if eq .Values.readReplicas.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- if .Values.readReplicas.service.extraPorts }} + {{- include "common.tplvalues.render" (dict "value" .Values.readReplicas.service.extraPorts "context" $) | nindent 4 }} + {{- end }} + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.readReplicas.podLabels .Values.commonLabels ) "context" . ) }} + selector: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: read +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/role.yaml b/charts/airflow/charts/postgresql/templates/role.yaml new file mode 100644 index 0000000..0ae728a --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/role.yaml @@ -0,0 +1,32 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if .Values.rbac.create }} +kind: Role +apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }} +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +# yamllint disable rule:indentation +rules: + {{- if and (include "common.capabilities.psp.supported" .) .Values.psp.create }} + - apiGroups: + - 'policy' + resources: + - 'podsecuritypolicies' + verbs: + - 'use' + resourceNames: + - {{ include "common.names.fullname" . }} + {{- end }} + {{- if .Values.rbac.rules }} + {{- include "common.tplvalues.render" ( dict "value" .Values.rbac.rules "context" $ ) | nindent 2 }} + {{- end }} +# yamllint enable rule:indentation +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/rolebinding.yaml b/charts/airflow/charts/postgresql/templates/rolebinding.yaml new file mode 100644 index 0000000..04323a0 --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/rolebinding.yaml @@ -0,0 +1,24 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if .Values.rbac.create }} +kind: RoleBinding +apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }} +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +roleRef: + kind: Role + name: {{ include "common.names.fullname" . }} + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: {{ include "postgresql.v1.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/secrets.yaml b/charts/airflow/charts/postgresql/templates/secrets.yaml new file mode 100644 index 0000000..b4267ab --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/secrets.yaml @@ -0,0 +1,99 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- $host := include "postgresql.v1.primary.fullname" . }} +{{- $port := include "postgresql.v1.service.port" . }} +{{- $customUser := include "postgresql.v1.username" . }} +{{- $postgresPassword := include "common.secrets.lookup" (dict "secret" (include "postgresql.v1.secretName" .) "key" (coalesce .Values.global.postgresql.auth.secretKeys.adminPasswordKey .Values.auth.secretKeys.adminPasswordKey) "defaultValue" (ternary (coalesce .Values.global.postgresql.auth.password .Values.auth.password .Values.global.postgresql.auth.postgresPassword .Values.auth.postgresPassword) (coalesce .Values.global.postgresql.auth.postgresPassword .Values.auth.postgresPassword) (or (empty $customUser) (eq $customUser "postgres"))) "context" $) | trimAll "\"" | b64dec }} +{{- if and (not $postgresPassword) .Values.auth.enablePostgresUser }} +{{- $postgresPassword = randAlphaNum 10 }} +{{- end }} +{{- $replicationPassword := "" }} +{{- if eq .Values.architecture "replication" }} +{{- $replicationPassword = include "common.secrets.passwords.manage" (dict "secret" (include "postgresql.v1.secretName" .) "key" (coalesce .Values.global.postgresql.auth.secretKeys.replicationPasswordKey .Values.auth.secretKeys.replicationPasswordKey) "providedValues" (list "auth.replicationPassword") "context" $) | trimAll "\"" | b64dec }} +{{- end }} +{{- $ldapPassword := "" }} +{{- if and .Values.ldap.enabled (or .Values.ldap.bind_password .Values.ldap.bindpw) }} +{{- $ldapPassword = coalesce .Values.ldap.bind_password .Values.ldap.bindpw }} +{{- end }} +{{- $password := "" }} +{{- if and (not (empty $customUser)) (ne $customUser "postgres") }} +{{- $password = include "common.secrets.passwords.manage" (dict "secret" (include "postgresql.v1.secretName" .) "key" (coalesce .Values.global.postgresql.auth.secretKeys.userPasswordKey .Values.auth.secretKeys.userPasswordKey) "providedValues" (list "global.postgresql.auth.password" "auth.password") "context" $) | trimAll "\"" | b64dec }} +{{- end }} +{{- $database := include "postgresql.v1.database" . }} +{{- if (include "postgresql.v1.createSecret" .) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- if $postgresPassword }} + postgres-password: {{ $postgresPassword | b64enc | quote }} + {{- end }} + {{- if $password }} + password: {{ $password | b64enc | quote }} + {{- end }} + {{- if $replicationPassword }} + replication-password: {{ $replicationPassword | b64enc | quote }} + {{- end }} + # We don't auto-generate LDAP password when it's not provided as we do for other passwords + {{- if and .Values.ldap.enabled (or .Values.ldap.bind_password .Values.ldap.bindpw) }} + ldap-password: {{ $ldapPassword | b64enc | quote }} + {{- end }} +{{- end }} +{{- if .Values.serviceBindings.enabled }} +{{- if $postgresPassword }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "common.names.fullname" . }}-svcbind-postgres + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +type: servicebinding.io/postgresql +data: + provider: {{ print "bitnami" | b64enc | quote }} + type: {{ print "postgresql" | b64enc | quote }} + host: {{ $host | b64enc | quote }} + port: {{ $port | b64enc | quote }} + username: {{ print "postgres" | b64enc | quote }} + database: {{ print "postgres" | b64enc | quote }} + password: {{ $postgresPassword | b64enc | quote }} + uri: {{ printf "postgresql://postgres:%s@%s:%s/postgres" $postgresPassword $host $port | b64enc | quote }} +{{- end }} +{{- if $password }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "common.names.fullname" . }}-svcbind-custom-user + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +type: servicebinding.io/postgresql +data: + provider: {{ print "bitnami" | b64enc | quote }} + type: {{ print "postgresql" | b64enc | quote }} + host: {{ $host | b64enc | quote }} + port: {{ $port | b64enc | quote }} + username: {{ $customUser | b64enc | quote }} + password: {{ $password | b64enc | quote }} + {{- if $database }} + database: {{ $database | b64enc | quote }} + {{- end }} + uri: {{ printf "postgresql://%s:%s@%s:%s/%s" $customUser $password $host $port $database | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/serviceaccount.yaml b/charts/airflow/charts/postgresql/templates/serviceaccount.yaml new file mode 100644 index 0000000..8886bff --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "postgresql.v1.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if or .Values.serviceAccount.annotations .Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.serviceAccount.annotations .Values.commonAnnotations ) "context" . ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/templates/tls-secrets.yaml b/charts/airflow/charts/postgresql/templates/tls-secrets.yaml new file mode 100644 index 0000000..7e44a43 --- /dev/null +++ b/charts/airflow/charts/postgresql/templates/tls-secrets.yaml @@ -0,0 +1,30 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if (include "postgresql.v1.createTlsSecret" . ) }} +{{- $secretName := printf "%s-crt" (include "common.names.fullname" .) }} +{{- $ca := genCA "postgresql-ca" 365 }} +{{- $fullname := include "common.names.fullname" . }} +{{- $releaseNamespace := .Release.Namespace }} +{{- $clusterDomain := .Values.clusterDomain }} +{{- $primaryHeadlessServiceName := include "postgresql.v1.primary.svc.headless" . }} +{{- $readHeadlessServiceName := include "postgresql.v1.readReplica.svc.headless" . }} +{{- $altNames := list (printf "*.%s.%s.svc.%s" $fullname $releaseNamespace $clusterDomain) (printf "%s.%s.svc.%s" $fullname $releaseNamespace $clusterDomain) (printf "*.%s.%s.svc.%s" $primaryHeadlessServiceName $releaseNamespace $clusterDomain) (printf "%s.%s.svc.%s" $primaryHeadlessServiceName $releaseNamespace $clusterDomain) (printf "*.%s.%s.svc.%s" $readHeadlessServiceName $releaseNamespace $clusterDomain) (printf "%s.%s.svc.%s" $readHeadlessServiceName $releaseNamespace $clusterDomain) $fullname }} +{{- $cert := genSignedCert $fullname nil $altNames 365 $ca }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + tls.crt: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "tls.crt" "defaultValue" $cert.Cert "context" $) }} + tls.key: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "tls.key" "defaultValue" $cert.Key "context" $) }} + ca.crt: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "ca.crt" "defaultValue" $ca.Cert "context" $) }} +{{- end }} diff --git a/charts/airflow/charts/postgresql/values.schema.json b/charts/airflow/charts/postgresql/values.schema.json new file mode 100644 index 0000000..fc41483 --- /dev/null +++ b/charts/airflow/charts/postgresql/values.schema.json @@ -0,0 +1,156 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "architecture": { + "type": "string", + "title": "PostgreSQL architecture", + "form": true, + "description": "Allowed values: `standalone` or `replication`" + }, + "auth": { + "type": "object", + "title": "Authentication configuration", + "form": true, + "properties": { + "enablePostgresUser": { + "type": "boolean", + "title": "Enable \"postgres\" admin user", + "description": "Assign a password to the \"postgres\" admin user. Otherwise, remote access will be blocked for this user", + "form": true + }, + "postgresPassword": { + "type": "string", + "title": "Password for the \"postgres\" admin user", + "description": "Defaults to a random 10-character alphanumeric string if not set", + "form": true + }, + "database": { + "type": "string", + "title": "PostgreSQL custom database", + "description": "Name of the custom database to be created during the 1st initialization of PostgreSQL", + "form": true + }, + "username": { + "type": "string", + "title": "PostgreSQL custom user", + "description": "Name of the custom user to be created during the 1st initialization of PostgreSQL. This user only has permissions on the PostgreSQL custom database", + "form": true + }, + "password": { + "type": "string", + "title": "Password for the custom user to create", + "description": "Defaults to a random 10-character alphanumeric string if not set", + "form": true + }, + "replicationUsername": { + "type": "string", + "title": "PostgreSQL replication user", + "description": "Name of user used to manage replication.", + "form": true, + "hidden": { + "value": "standalone", + "path": "architecture" + } + }, + "replicationPassword": { + "type": "string", + "title": "Password for PostgreSQL replication user", + "description": "Defaults to a random 10-character alphanumeric string if not set", + "form": true, + "hidden": { + "value": "standalone", + "path": "architecture" + } + } + } + }, + "persistence": { + "type": "object", + "properties": { + "size": { + "type": "string", + "title": "Persistent Volume Size", + "form": true, + "render": "slider", + "sliderMin": 1, + "sliderMax": 100, + "sliderUnit": "Gi" + } + } + }, + "resources": { + "type": "object", + "title": "Required Resources", + "description": "Configure resource requests", + "form": true, + "properties": { + "requests": { + "type": "object", + "properties": { + "memory": { + "type": "string", + "form": true, + "render": "slider", + "title": "Memory Request", + "sliderMin": 10, + "sliderMax": 2048, + "sliderUnit": "Mi" + }, + "cpu": { + "type": "string", + "form": true, + "render": "slider", + "title": "CPU Request", + "sliderMin": 10, + "sliderMax": 2000, + "sliderUnit": "m" + } + } + } + } + }, + "replication": { + "type": "object", + "form": true, + "title": "Replication Details", + "properties": { + "enabled": { + "type": "boolean", + "title": "Enable Replication", + "form": true + }, + "readReplicas": { + "type": "integer", + "title": "read Replicas", + "form": true, + "hidden": { + "value": "standalone", + "path": "architecture" + } + } + } + }, + "volumePermissions": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "form": true, + "title": "Enable Init Containers", + "description": "Change the owner of the persist volume mountpoint to RunAsUser:fsGroup" + } + } + }, + "metrics": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "title": "Configure metrics exporter", + "form": true + } + } + } + } +} diff --git a/charts/airflow/charts/postgresql/values.yaml b/charts/airflow/charts/postgresql/values.yaml new file mode 100644 index 0000000..15d87d4 --- /dev/null +++ b/charts/airflow/charts/postgresql/values.yaml @@ -0,0 +1,1613 @@ +# Copyright VMware, Inc. +# SPDX-License-Identifier: APACHE-2.0 + +## @section Global parameters +## Please, note that this will override the parameters, including dependencies, configured to use the global value +## +global: + ## @param global.imageRegistry Global Docker image registry + ## + imageRegistry: "" + ## @param global.imagePullSecrets Global Docker registry secret names as an array + ## e.g. + ## imagePullSecrets: + ## - myRegistryKeySecretName + ## + imagePullSecrets: [] + ## @param global.storageClass Global StorageClass for Persistent Volume(s) + ## + storageClass: "" + postgresql: + ## @param global.postgresql.auth.postgresPassword Password for the "postgres" admin user (overrides `auth.postgresPassword`) + ## @param global.postgresql.auth.username Name for a custom user to create (overrides `auth.username`) + ## @param global.postgresql.auth.password Password for the custom user to create (overrides `auth.password`) + ## @param global.postgresql.auth.database Name for a custom database to create (overrides `auth.database`) + ## @param global.postgresql.auth.existingSecret Name of existing secret to use for PostgreSQL credentials (overrides `auth.existingSecret`). + ## @param global.postgresql.auth.secretKeys.adminPasswordKey Name of key in existing secret to use for PostgreSQL credentials (overrides `auth.secretKeys.adminPasswordKey`). Only used when `global.postgresql.auth.existingSecret` is set. + ## @param global.postgresql.auth.secretKeys.userPasswordKey Name of key in existing secret to use for PostgreSQL credentials (overrides `auth.secretKeys.userPasswordKey`). Only used when `global.postgresql.auth.existingSecret` is set. + ## @param global.postgresql.auth.secretKeys.replicationPasswordKey Name of key in existing secret to use for PostgreSQL credentials (overrides `auth.secretKeys.replicationPasswordKey`). Only used when `global.postgresql.auth.existingSecret` is set. + ## + auth: + postgresPassword: "" + username: "" + password: "" + database: "" + existingSecret: "" + secretKeys: + adminPasswordKey: "" + userPasswordKey: "" + replicationPasswordKey: "" + ## @param global.postgresql.service.ports.postgresql PostgreSQL service port (overrides `service.ports.postgresql`) + ## + service: + ports: + postgresql: "" + +## @section Common parameters +## + +## @param kubeVersion Override Kubernetes version +## +kubeVersion: "" +## @param nameOverride String to partially override common.names.fullname template (will maintain the release name) +## +nameOverride: "" +## @param fullnameOverride String to fully override common.names.fullname template +## +fullnameOverride: "" +## @param clusterDomain Kubernetes Cluster Domain +## +clusterDomain: cluster.local +## @param extraDeploy Array of extra objects to deploy with the release (evaluated as a template) +## +extraDeploy: [] +## @param commonLabels Add labels to all the deployed resources +## +commonLabels: {} +## @param commonAnnotations Add annotations to all the deployed resources +## +commonAnnotations: {} +## Enable diagnostic mode in the statefulset +## +diagnosticMode: + ## @param diagnosticMode.enabled Enable diagnostic mode (all probes will be disabled and the command will be overridden) + ## + enabled: false + ## @param diagnosticMode.command Command to override all containers in the statefulset + ## + command: + - sleep + ## @param diagnosticMode.args Args to override all containers in the statefulset + ## + args: + - infinity + +## @section PostgreSQL common parameters +## + +## Bitnami PostgreSQL image version +## ref: https://hub.docker.com/r/bitnami/postgresql/tags/ +## @param image.registry [default: REGISTRY_NAME] PostgreSQL image registry +## @param image.repository [default: REPOSITORY_NAME/postgresql] PostgreSQL image repository +## @skip image.tag PostgreSQL image tag (immutable tags are recommended) +## @param image.digest PostgreSQL image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag +## @param image.pullPolicy PostgreSQL image pull policy +## @param image.pullSecrets Specify image pull secrets +## @param image.debug Specify if debug values should be set +## +image: + registry: docker.io + repository: bitnami/postgresql + tag: 16.1.0-debian-11-r15 + digest: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## Set to true if you would like to see extra information on logs + ## + debug: false +## Authentication parameters +## ref: https://github.com/bitnami/containers/tree/main/bitnami/postgresql#setting-the-root-password-on-first-run +## ref: https://github.com/bitnami/containers/tree/main/bitnami/postgresql#creating-a-database-on-first-run +## ref: https://github.com/bitnami/containers/tree/main/bitnami/postgresql#creating-a-database-user-on-first-run +## +auth: + ## @param auth.enablePostgresUser Assign a password to the "postgres" admin user. Otherwise, remote access will be blocked for this user + ## + enablePostgresUser: true + ## @param auth.postgresPassword Password for the "postgres" admin user. Ignored if `auth.existingSecret` is provided + ## + postgresPassword: "" + ## @param auth.username Name for a custom user to create + ## + username: "" + ## @param auth.password Password for the custom user to create. Ignored if `auth.existingSecret` is provided + ## + password: "" + ## @param auth.database Name for a custom database to create + ## + database: "" + ## @param auth.replicationUsername Name of the replication user + ## + replicationUsername: repl_user + ## @param auth.replicationPassword Password for the replication user. Ignored if `auth.existingSecret` is provided + ## + replicationPassword: "" + ## @param auth.existingSecret Name of existing secret to use for PostgreSQL credentials. `auth.postgresPassword`, `auth.password`, and `auth.replicationPassword` will be ignored and picked up from this secret. The secret might also contains the key `ldap-password` if LDAP is enabled. `ldap.bind_password` will be ignored and picked from this secret in this case. + ## + existingSecret: "" + ## @param auth.secretKeys.adminPasswordKey Name of key in existing secret to use for PostgreSQL credentials. Only used when `auth.existingSecret` is set. + ## @param auth.secretKeys.userPasswordKey Name of key in existing secret to use for PostgreSQL credentials. Only used when `auth.existingSecret` is set. + ## @param auth.secretKeys.replicationPasswordKey Name of key in existing secret to use for PostgreSQL credentials. Only used when `auth.existingSecret` is set. + ## + secretKeys: + adminPasswordKey: postgres-password + userPasswordKey: password + replicationPasswordKey: replication-password + ## @param auth.usePasswordFiles Mount credentials as a files instead of using an environment variable + ## + usePasswordFiles: false +## @param architecture PostgreSQL architecture (`standalone` or `replication`) +## +architecture: standalone +## Replication configuration +## Ignored if `architecture` is `standalone` +## +replication: + ## @param replication.synchronousCommit Set synchronous commit mode. Allowed values: `on`, `remote_apply`, `remote_write`, `local` and `off` + ## @param replication.numSynchronousReplicas Number of replicas that will have synchronous replication. Note: Cannot be greater than `readReplicas.replicaCount`. + ## ref: https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-SYNCHRONOUS-COMMIT + ## + synchronousCommit: "off" + numSynchronousReplicas: 0 + ## @param replication.applicationName Cluster application name. Useful for advanced replication settings + ## + applicationName: my_application +## @param containerPorts.postgresql PostgreSQL container port +## +containerPorts: + postgresql: 5432 +## Audit settings +## https://github.com/bitnami/containers/tree/main/bitnami/postgresql#auditing +## @param audit.logHostname Log client hostnames +## @param audit.logConnections Add client log-in operations to the log file +## @param audit.logDisconnections Add client log-outs operations to the log file +## @param audit.pgAuditLog Add operations to log using the pgAudit extension +## @param audit.pgAuditLogCatalog Log catalog using pgAudit +## @param audit.clientMinMessages Message log level to share with the user +## @param audit.logLinePrefix Template for log line prefix (default if not set) +## @param audit.logTimezone Timezone for the log timestamps +## +audit: + logHostname: false + logConnections: false + logDisconnections: false + pgAuditLog: "" + pgAuditLogCatalog: "off" + clientMinMessages: error + logLinePrefix: "" + logTimezone: "" +## LDAP configuration +## @param ldap.enabled Enable LDAP support +## DEPRECATED ldap.url It will removed in a future, please use 'ldap.uri' instead +## @param ldap.server IP address or name of the LDAP server. +## @param ldap.port Port number on the LDAP server to connect to +## @param ldap.prefix String to prepend to the user name when forming the DN to bind +## @param ldap.suffix String to append to the user name when forming the DN to bind +## DEPRECATED ldap.baseDN It will removed in a future, please use 'ldap.basedn' instead +## DEPRECATED ldap.bindDN It will removed in a future, please use 'ldap.binddn' instead +## DEPRECATED ldap.bind_password It will removed in a future, please use 'ldap.bindpw' instead +## @param ldap.basedn Root DN to begin the search for the user in +## @param ldap.binddn DN of user to bind to LDAP +## @param ldap.bindpw Password for the user to bind to LDAP +## DEPRECATED ldap.search_attr It will removed in a future, please use 'ldap.searchAttribute' instead +## DEPRECATED ldap.search_filter It will removed in a future, please use 'ldap.searchFilter' instead +## @param ldap.searchAttribute Attribute to match against the user name in the search +## @param ldap.searchFilter The search filter to use when doing search+bind authentication +## @param ldap.scheme Set to `ldaps` to use LDAPS +## DEPRECATED ldap.tls as string is deprecated，please use 'ldap.tls.enabled' instead +## @param ldap.tls.enabled Se to true to enable TLS encryption +## +ldap: + enabled: false + server: "" + port: "" + prefix: "" + suffix: "" + basedn: "" + binddn: "" + bindpw: "" + searchAttribute: "" + searchFilter: "" + scheme: "" + tls: + enabled: false + ## @param ldap.uri LDAP URL beginning in the form `ldap[s]://host[:port]/basedn`. If provided, all the other LDAP parameters will be ignored. + ## Ref: https://www.postgresql.org/docs/current/auth-ldap.html + ## + uri: "" +## @param postgresqlDataDir PostgreSQL data dir folder +## +postgresqlDataDir: /bitnami/postgresql/data +## @param postgresqlSharedPreloadLibraries Shared preload libraries (comma-separated list) +## +postgresqlSharedPreloadLibraries: "pgaudit" +## Start PostgreSQL pod(s) without limitations on shm memory. +## By default docker and containerd (and possibly other container runtimes) limit `/dev/shm` to `64M` +## ref: https://github.com/docker-library/postgres/issues/416 +## ref: https://github.com/containerd/containerd/issues/3654 +## +shmVolume: + ## @param shmVolume.enabled Enable emptyDir volume for /dev/shm for PostgreSQL pod(s) + ## + enabled: true + ## @param shmVolume.sizeLimit Set this to enable a size limit on the shm tmpfs + ## Note: the size of the tmpfs counts against container's memory limit + ## e.g: + ## sizeLimit: 1Gi + ## + sizeLimit: "" +## TLS configuration +## +tls: + ## @param tls.enabled Enable TLS traffic support + ## + enabled: false + ## @param tls.autoGenerated Generate automatically self-signed TLS certificates + ## + autoGenerated: false + ## @param tls.preferServerCiphers Whether to use the server's TLS cipher preferences rather than the client's + ## + preferServerCiphers: true + ## @param tls.certificatesSecret Name of an existing secret that contains the certificates + ## + certificatesSecret: "" + ## @param tls.certFilename Certificate filename + ## + certFilename: "" + ## @param tls.certKeyFilename Certificate key filename + ## + certKeyFilename: "" + ## @param tls.certCAFilename CA Certificate filename + ## If provided, PostgreSQL will authenticate TLS/SSL clients by requesting them a certificate + ## ref: https://www.postgresql.org/docs/9.6/auth-methods.html + ## + certCAFilename: "" + ## @param tls.crlFilename File containing a Certificate Revocation List + ## + crlFilename: "" + +## @section PostgreSQL Primary parameters +## +primary: + ## @param primary.name Name of the primary database (eg primary, master, leader, ...) + ## + name: primary + ## @param primary.configuration PostgreSQL Primary main configuration to be injected as ConfigMap + ## ref: https://www.postgresql.org/docs/current/static/runtime-config.html + ## + configuration: "" + ## @param primary.pgHbaConfiguration PostgreSQL Primary client authentication configuration + ## ref: https://www.postgresql.org/docs/current/static/auth-pg-hba-conf.html + ## e.g:# + ## pgHbaConfiguration: |- + ## local all all trust + ## host all all localhost trust + ## host mydatabase mysuser 192.168.0.0/24 md5 + ## + pgHbaConfiguration: "" + ## @param primary.existingConfigmap Name of an existing ConfigMap with PostgreSQL Primary configuration + ## NOTE: `primary.configuration` and `primary.pgHbaConfiguration` will be ignored + ## + existingConfigmap: "" + ## @param primary.extendedConfiguration Extended PostgreSQL Primary configuration (appended to main or default configuration) + ## ref: https://github.com/bitnami/containers/tree/main/bitnami/postgresql#allow-settings-to-be-loaded-from-files-other-than-the-default-postgresqlconf + ## + extendedConfiguration: "" + ## @param primary.existingExtendedConfigmap Name of an existing ConfigMap with PostgreSQL Primary extended configuration + ## NOTE: `primary.extendedConfiguration` will be ignored + ## + existingExtendedConfigmap: "" + ## Initdb configuration + ## ref: https://github.com/bitnami/containers/tree/main/bitnami/postgresql#specifying-initdb-arguments + ## + initdb: + ## @param primary.initdb.args PostgreSQL initdb extra arguments + ## + args: "" + ## @param primary.initdb.postgresqlWalDir Specify a custom location for the PostgreSQL transaction log + ## + postgresqlWalDir: "" + ## @param primary.initdb.scripts Dictionary of initdb scripts + ## Specify dictionary of scripts to be run at first boot + ## e.g: + ## scripts: + ## my_init_script.sh: | + ## #!/bin/sh + ## echo "Do something." + ## + scripts: {} + ## @param primary.initdb.scriptsConfigMap ConfigMap with scripts to be run at first boot + ## NOTE: This will override `primary.initdb.scripts` + ## + scriptsConfigMap: "" + ## @param primary.initdb.scriptsSecret Secret with scripts to be run at first boot (in case it contains sensitive information) + ## NOTE: This can work along `primary.initdb.scripts` or `primary.initdb.scriptsConfigMap` + ## + scriptsSecret: "" + ## @param primary.initdb.user Specify the PostgreSQL username to execute the initdb scripts + ## + user: "" + ## @param primary.initdb.password Specify the PostgreSQL password to execute the initdb scripts + ## + password: "" + ## Configure current cluster's primary server to be the standby server in other cluster. + ## This will allow cross cluster replication and provide cross cluster high availability. + ## You will need to configure pgHbaConfiguration if you want to enable this feature with local cluster replication enabled. + ## @param primary.standby.enabled Whether to enable current cluster's primary as standby server of another cluster or not + ## @param primary.standby.primaryHost The Host of replication primary in the other cluster + ## @param primary.standby.primaryPort The Port of replication primary in the other cluster + ## + standby: + enabled: false + primaryHost: "" + primaryPort: "" + ## @param primary.extraEnvVars Array with extra environment variables to add to PostgreSQL Primary nodes + ## e.g: + ## extraEnvVars: + ## - name: FOO + ## value: "bar" + ## + extraEnvVars: [] + ## @param primary.extraEnvVarsCM Name of existing ConfigMap containing extra env vars for PostgreSQL Primary nodes + ## + extraEnvVarsCM: "" + ## @param primary.extraEnvVarsSecret Name of existing Secret containing extra env vars for PostgreSQL Primary nodes + ## + extraEnvVarsSecret: "" + ## @param primary.command Override default container command (useful when using custom images) + ## + command: [] + ## @param primary.args Override default container args (useful when using custom images) + ## + args: [] + ## Configure extra options for PostgreSQL Primary containers' liveness, readiness and startup probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes + ## @param primary.livenessProbe.enabled Enable livenessProbe on PostgreSQL Primary containers + ## @param primary.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe + ## @param primary.livenessProbe.periodSeconds Period seconds for livenessProbe + ## @param primary.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe + ## @param primary.livenessProbe.failureThreshold Failure threshold for livenessProbe + ## @param primary.livenessProbe.successThreshold Success threshold for livenessProbe + ## + livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + ## @param primary.readinessProbe.enabled Enable readinessProbe on PostgreSQL Primary containers + ## @param primary.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe + ## @param primary.readinessProbe.periodSeconds Period seconds for readinessProbe + ## @param primary.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe + ## @param primary.readinessProbe.failureThreshold Failure threshold for readinessProbe + ## @param primary.readinessProbe.successThreshold Success threshold for readinessProbe + ## + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + ## @param primary.startupProbe.enabled Enable startupProbe on PostgreSQL Primary containers + ## @param primary.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe + ## @param primary.startupProbe.periodSeconds Period seconds for startupProbe + ## @param primary.startupProbe.timeoutSeconds Timeout seconds for startupProbe + ## @param primary.startupProbe.failureThreshold Failure threshold for startupProbe + ## @param primary.startupProbe.successThreshold Success threshold for startupProbe + ## + startupProbe: + enabled: false + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 15 + successThreshold: 1 + ## @param primary.customLivenessProbe Custom livenessProbe that overrides the default one + ## + customLivenessProbe: {} + ## @param primary.customReadinessProbe Custom readinessProbe that overrides the default one + ## + customReadinessProbe: {} + ## @param primary.customStartupProbe Custom startupProbe that overrides the default one + ## + customStartupProbe: {} + ## @param primary.lifecycleHooks for the PostgreSQL Primary container to automate configuration before or after startup + ## + lifecycleHooks: {} + ## PostgreSQL Primary resource requests and limits + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## @param primary.resources.limits The resources limits for the PostgreSQL Primary containers + ## @param primary.resources.requests.memory The requested memory for the PostgreSQL Primary containers + ## @param primary.resources.requests.cpu The requested cpu for the PostgreSQL Primary containers + ## + resources: + limits: {} + requests: + memory: 256Mi + cpu: 250m + ## Pod Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## @param primary.podSecurityContext.enabled Enable security context + ## @param primary.podSecurityContext.fsGroup Group ID for the pod + ## + podSecurityContext: + enabled: true + fsGroup: 1001 + ## Container Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## @param primary.containerSecurityContext.enabled Enabled containers' Security Context + ## @param primary.containerSecurityContext.runAsUser Set containers' Security Context runAsUser + ## @param primary.containerSecurityContext.runAsNonRoot Set container's Security Context runAsNonRoot + ## @param primary.containerSecurityContext.privileged Set container's Security Context privileged + ## @param primary.containerSecurityContext.readOnlyRootFilesystem Set container's Security Context readOnlyRootFilesystem + ## @param primary.containerSecurityContext.allowPrivilegeEscalation Set container's Security Context allowPrivilegeEscalation + ## @param primary.containerSecurityContext.capabilities.drop List of capabilities to be dropped + ## @param primary.containerSecurityContext.seccompProfile.type Set container's Security Context seccomp profile + ## + containerSecurityContext: + enabled: true + runAsUser: 1001 + runAsNonRoot: true + privileged: false + readOnlyRootFilesystem: false + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: "RuntimeDefault" + ## @param primary.hostAliases PostgreSQL primary pods host aliases + ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ + ## + hostAliases: [] + ## @param primary.hostNetwork Specify if host network should be enabled for PostgreSQL pod (postgresql primary) + ## + hostNetwork: false + ## @param primary.hostIPC Specify if host IPC should be enabled for PostgreSQL pod (postgresql primary) + ## + hostIPC: false + ## @param primary.labels Map of labels to add to the statefulset (postgresql primary) + ## + labels: {} + ## @param primary.annotations Annotations for PostgreSQL primary pods + ## + annotations: {} + ## @param primary.podLabels Map of labels to add to the pods (postgresql primary) + ## + podLabels: {} + ## @param primary.podAnnotations Map of annotations to add to the pods (postgresql primary) + ## + podAnnotations: {} + ## @param primary.podAffinityPreset PostgreSQL primary pod affinity preset. Ignored if `primary.affinity` is set. Allowed values: `soft` or `hard` + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## + podAffinityPreset: "" + ## @param primary.podAntiAffinityPreset PostgreSQL primary pod anti-affinity preset. Ignored if `primary.affinity` is set. Allowed values: `soft` or `hard` + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## + podAntiAffinityPreset: soft + ## PostgreSQL Primary node affinity preset + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity + ## + nodeAffinityPreset: + ## @param primary.nodeAffinityPreset.type PostgreSQL primary node affinity preset type. Ignored if `primary.affinity` is set. Allowed values: `soft` or `hard` + ## + type: "" + ## @param primary.nodeAffinityPreset.key PostgreSQL primary node label key to match Ignored if `primary.affinity` is set. + ## E.g. + ## key: "kubernetes.io/e2e-az-name" + ## + key: "" + ## @param primary.nodeAffinityPreset.values PostgreSQL primary node label values to match. Ignored if `primary.affinity` is set. + ## E.g. + ## values: + ## - e2e-az1 + ## - e2e-az2 + ## + values: [] + ## @param primary.affinity Affinity for PostgreSQL primary pods assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## Note: primary.podAffinityPreset, primary.podAntiAffinityPreset, and primary.nodeAffinityPreset will be ignored when it's set + ## + affinity: {} + ## @param primary.nodeSelector Node labels for PostgreSQL primary pods assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + ## @param primary.tolerations Tolerations for PostgreSQL primary pods assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## @param primary.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template + ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods + ## + topologySpreadConstraints: [] + ## @param primary.priorityClassName Priority Class to use for each pod (postgresql primary) + ## + priorityClassName: "" + ## @param primary.schedulerName Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + schedulerName: "" + ## @param primary.terminationGracePeriodSeconds Seconds PostgreSQL primary pod needs to terminate gracefully + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods + ## + terminationGracePeriodSeconds: "" + ## @param primary.updateStrategy.type PostgreSQL Primary statefulset strategy type + ## @param primary.updateStrategy.rollingUpdate PostgreSQL Primary statefulset rolling update configuration parameters + ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies + ## + updateStrategy: + type: RollingUpdate + rollingUpdate: {} + ## @param primary.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the PostgreSQL Primary container(s) + ## + extraVolumeMounts: [] + ## @param primary.extraVolumes Optionally specify extra list of additional volumes for the PostgreSQL Primary pod(s) + ## + extraVolumes: [] + ## @param primary.sidecars Add additional sidecar containers to the PostgreSQL Primary pod(s) + ## For example: + ## sidecars: + ## - name: your-image-name + ## image: your-image + ## imagePullPolicy: Always + ## ports: + ## - name: portname + ## containerPort: 1234 + ## + sidecars: [] + ## @param primary.initContainers Add additional init containers to the PostgreSQL Primary pod(s) + ## Example + ## + ## initContainers: + ## - name: do-something + ## image: busybox + ## command: ['do', 'something'] + ## + initContainers: [] + ## @param primary.extraPodSpec Optionally specify extra PodSpec for the PostgreSQL Primary pod(s) + ## + extraPodSpec: {} + ## PostgreSQL Primary service configuration + ## + service: + ## @param primary.service.type Kubernetes Service type + ## + type: ClusterIP + ## @param primary.service.ports.postgresql PostgreSQL service port + ## + ports: + postgresql: 5432 + ## Node ports to expose + ## NOTE: choose port between <30000-32767> + ## @param primary.service.nodePorts.postgresql Node port for PostgreSQL + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePorts: + postgresql: "" + ## @param primary.service.clusterIP Static clusterIP or None for headless services + ## e.g: + ## clusterIP: None + ## + clusterIP: "" + ## @param primary.service.annotations Annotations for PostgreSQL primary service + ## + annotations: {} + ## @param primary.service.loadBalancerIP Load balancer IP if service type is `LoadBalancer` + ## Set the LoadBalancer service type to internal only + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + loadBalancerIP: "" + ## @param primary.service.externalTrafficPolicy Enable client source IP preservation + ## ref https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + ## @param primary.service.loadBalancerSourceRanges Addresses that are allowed when service is LoadBalancer + ## https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## @param primary.service.extraPorts Extra ports to expose in the PostgreSQL primary service + ## + extraPorts: [] + ## @param primary.service.sessionAffinity Session Affinity for Kubernetes service, can be "None" or "ClientIP" + ## If "ClientIP", consecutive client requests will be directed to the same Pod + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + ## + sessionAffinity: None + ## @param primary.service.sessionAffinityConfig Additional settings for the sessionAffinity + ## sessionAffinityConfig: + ## clientIP: + ## timeoutSeconds: 300 + ## + sessionAffinityConfig: {} + ## Headless service properties + ## + headless: + ## @param primary.service.headless.annotations Additional custom annotations for headless PostgreSQL primary service + ## + annotations: {} + ## PostgreSQL Primary persistence configuration + ## + persistence: + ## @param primary.persistence.enabled Enable PostgreSQL Primary data persistence using PVC + ## + enabled: true + ## @param primary.persistence.existingClaim Name of an existing PVC to use + ## + existingClaim: "" + ## @param primary.persistence.mountPath The path the volume will be mounted at + ## Note: useful when using custom PostgreSQL images + ## + mountPath: /bitnami/postgresql + ## @param primary.persistence.subPath The subdirectory of the volume to mount to + ## Useful in dev environments and one PV for multiple services + ## + subPath: "" + ## @param primary.persistence.storageClass PVC Storage Class for PostgreSQL Primary data volume + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + storageClass: "" + ## @param primary.persistence.accessModes PVC Access Mode for PostgreSQL volume + ## + accessModes: + - ReadWriteOnce + ## @param primary.persistence.size PVC Storage Request for PostgreSQL volume + ## + size: 8Gi + ## @param primary.persistence.annotations Annotations for the PVC + ## + annotations: {} + ## @param primary.persistence.labels Labels for the PVC + ## + labels: {} + ## @param primary.persistence.selector Selector to match an existing Persistent Volume (this value is evaluated as a template) + ## selector: + ## matchLabels: + ## app: my-app + ## + selector: {} + ## @param primary.persistence.dataSource Custom PVC data source + ## + dataSource: {} + ## PostgreSQL Primary Persistent Volume Claim Retention Policy + ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention + ## + persistentVolumeClaimRetentionPolicy: + ## @param primary.persistentVolumeClaimRetentionPolicy.enabled Enable Persistent volume retention policy for Primary Statefulset + ## + enabled: false + ## @param primary.persistentVolumeClaimRetentionPolicy.whenScaled Volume retention behavior when the replica count of the StatefulSet is reduced + ## + whenScaled: Retain + ## @param primary.persistentVolumeClaimRetentionPolicy.whenDeleted Volume retention behavior that applies when the StatefulSet is deleted + ## + whenDeleted: Retain + +## @section PostgreSQL read only replica parameters (only used when `architecture` is set to `replication`) +## +readReplicas: + ## @param readReplicas.name Name of the read replicas database (eg secondary, slave, ...) + ## + name: read + ## @param readReplicas.replicaCount Number of PostgreSQL read only replicas + ## + replicaCount: 1 + ## @param readReplicas.extendedConfiguration Extended PostgreSQL read only replicas configuration (appended to main or default configuration) + ## ref: https://github.com/bitnami/containers/tree/main/bitnami/postgresql#allow-settings-to-be-loaded-from-files-other-than-the-default-postgresqlconf + ## + extendedConfiguration: "" + ## @param readReplicas.extraEnvVars Array with extra environment variables to add to PostgreSQL read only nodes + ## e.g: + ## extraEnvVars: + ## - name: FOO + ## value: "bar" + ## + extraEnvVars: [] + ## @param readReplicas.extraEnvVarsCM Name of existing ConfigMap containing extra env vars for PostgreSQL read only nodes + ## + extraEnvVarsCM: "" + ## @param readReplicas.extraEnvVarsSecret Name of existing Secret containing extra env vars for PostgreSQL read only nodes + ## + extraEnvVarsSecret: "" + ## @param readReplicas.command Override default container command (useful when using custom images) + ## + command: [] + ## @param readReplicas.args Override default container args (useful when using custom images) + ## + args: [] + ## Configure extra options for PostgreSQL read only containers' liveness, readiness and startup probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes + ## @param readReplicas.livenessProbe.enabled Enable livenessProbe on PostgreSQL read only containers + ## @param readReplicas.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe + ## @param readReplicas.livenessProbe.periodSeconds Period seconds for livenessProbe + ## @param readReplicas.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe + ## @param readReplicas.livenessProbe.failureThreshold Failure threshold for livenessProbe + ## @param readReplicas.livenessProbe.successThreshold Success threshold for livenessProbe + ## + livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + ## @param readReplicas.readinessProbe.enabled Enable readinessProbe on PostgreSQL read only containers + ## @param readReplicas.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe + ## @param readReplicas.readinessProbe.periodSeconds Period seconds for readinessProbe + ## @param readReplicas.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe + ## @param readReplicas.readinessProbe.failureThreshold Failure threshold for readinessProbe + ## @param readReplicas.readinessProbe.successThreshold Success threshold for readinessProbe + ## + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + ## @param readReplicas.startupProbe.enabled Enable startupProbe on PostgreSQL read only containers + ## @param readReplicas.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe + ## @param readReplicas.startupProbe.periodSeconds Period seconds for startupProbe + ## @param readReplicas.startupProbe.timeoutSeconds Timeout seconds for startupProbe + ## @param readReplicas.startupProbe.failureThreshold Failure threshold for startupProbe + ## @param readReplicas.startupProbe.successThreshold Success threshold for startupProbe + ## + startupProbe: + enabled: false + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 15 + successThreshold: 1 + ## @param readReplicas.customLivenessProbe Custom livenessProbe that overrides the default one + ## + customLivenessProbe: {} + ## @param readReplicas.customReadinessProbe Custom readinessProbe that overrides the default one + ## + customReadinessProbe: {} + ## @param readReplicas.customStartupProbe Custom startupProbe that overrides the default one + ## + customStartupProbe: {} + ## @param readReplicas.lifecycleHooks for the PostgreSQL read only container to automate configuration before or after startup + ## + lifecycleHooks: {} + ## PostgreSQL read only resource requests and limits + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## @param readReplicas.resources.limits The resources limits for the PostgreSQL read only containers + ## @param readReplicas.resources.requests.memory The requested memory for the PostgreSQL read only containers + ## @param readReplicas.resources.requests.cpu The requested cpu for the PostgreSQL read only containers + ## + resources: + limits: {} + requests: + memory: 256Mi + cpu: 250m + ## Pod Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## @param readReplicas.podSecurityContext.enabled Enable security context + ## @param readReplicas.podSecurityContext.fsGroup Group ID for the pod + ## + podSecurityContext: + enabled: true + fsGroup: 1001 + ## Container Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## @param readReplicas.containerSecurityContext.enabled Enabled containers' Security Context + ## @param readReplicas.containerSecurityContext.runAsUser Set containers' Security Context runAsUser + ## @param readReplicas.containerSecurityContext.runAsNonRoot Set container's Security Context runAsNonRoot + ## @param readReplicas.containerSecurityContext.privileged Set container's Security Context privileged + ## @param readReplicas.containerSecurityContext.readOnlyRootFilesystem Set container's Security Context readOnlyRootFilesystem + ## @param readReplicas.containerSecurityContext.allowPrivilegeEscalation Set container's Security Context allowPrivilegeEscalation + ## @param readReplicas.containerSecurityContext.capabilities.drop List of capabilities to be dropped + ## @param readReplicas.containerSecurityContext.seccompProfile.type Set container's Security Context seccomp profile + ## + containerSecurityContext: + enabled: true + runAsUser: 1001 + runAsNonRoot: true + privileged: false + readOnlyRootFilesystem: false + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: "RuntimeDefault" + ## @param readReplicas.hostAliases PostgreSQL read only pods host aliases + ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ + ## + hostAliases: [] + ## @param readReplicas.hostNetwork Specify if host network should be enabled for PostgreSQL pod (PostgreSQL read only) + ## + hostNetwork: false + ## @param readReplicas.hostIPC Specify if host IPC should be enabled for PostgreSQL pod (postgresql primary) + ## + hostIPC: false + ## @param readReplicas.labels Map of labels to add to the statefulset (PostgreSQL read only) + ## + labels: {} + ## @param readReplicas.annotations Annotations for PostgreSQL read only pods + ## + annotations: {} + ## @param readReplicas.podLabels Map of labels to add to the pods (PostgreSQL read only) + ## + podLabels: {} + ## @param readReplicas.podAnnotations Map of annotations to add to the pods (PostgreSQL read only) + ## + podAnnotations: {} + ## @param readReplicas.podAffinityPreset PostgreSQL read only pod affinity preset. Ignored if `primary.affinity` is set. Allowed values: `soft` or `hard` + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## + podAffinityPreset: "" + ## @param readReplicas.podAntiAffinityPreset PostgreSQL read only pod anti-affinity preset. Ignored if `primary.affinity` is set. Allowed values: `soft` or `hard` + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## + podAntiAffinityPreset: soft + ## PostgreSQL read only node affinity preset + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity + ## + nodeAffinityPreset: + ## @param readReplicas.nodeAffinityPreset.type PostgreSQL read only node affinity preset type. Ignored if `primary.affinity` is set. Allowed values: `soft` or `hard` + ## + type: "" + ## @param readReplicas.nodeAffinityPreset.key PostgreSQL read only node label key to match Ignored if `primary.affinity` is set. + ## E.g. + ## key: "kubernetes.io/e2e-az-name" + ## + key: "" + ## @param readReplicas.nodeAffinityPreset.values PostgreSQL read only node label values to match. Ignored if `primary.affinity` is set. + ## E.g. + ## values: + ## - e2e-az1 + ## - e2e-az2 + ## + values: [] + ## @param readReplicas.affinity Affinity for PostgreSQL read only pods assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## Note: primary.podAffinityPreset, primary.podAntiAffinityPreset, and primary.nodeAffinityPreset will be ignored when it's set + ## + affinity: {} + ## @param readReplicas.nodeSelector Node labels for PostgreSQL read only pods assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + ## @param readReplicas.tolerations Tolerations for PostgreSQL read only pods assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## @param readReplicas.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template + ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods + ## + topologySpreadConstraints: [] + ## @param readReplicas.priorityClassName Priority Class to use for each pod (PostgreSQL read only) + ## + priorityClassName: "" + ## @param readReplicas.schedulerName Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + schedulerName: "" + ## @param readReplicas.terminationGracePeriodSeconds Seconds PostgreSQL read only pod needs to terminate gracefully + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods + ## + terminationGracePeriodSeconds: "" + ## @param readReplicas.updateStrategy.type PostgreSQL read only statefulset strategy type + ## @param readReplicas.updateStrategy.rollingUpdate PostgreSQL read only statefulset rolling update configuration parameters + ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies + ## + updateStrategy: + type: RollingUpdate + rollingUpdate: {} + ## @param readReplicas.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the PostgreSQL read only container(s) + ## + extraVolumeMounts: [] + ## @param readReplicas.extraVolumes Optionally specify extra list of additional volumes for the PostgreSQL read only pod(s) + ## + extraVolumes: [] + ## @param readReplicas.sidecars Add additional sidecar containers to the PostgreSQL read only pod(s) + ## For example: + ## sidecars: + ## - name: your-image-name + ## image: your-image + ## imagePullPolicy: Always + ## ports: + ## - name: portname + ## containerPort: 1234 + ## + sidecars: [] + ## @param readReplicas.initContainers Add additional init containers to the PostgreSQL read only pod(s) + ## Example + ## + ## initContainers: + ## - name: do-something + ## image: busybox + ## command: ['do', 'something'] + ## + initContainers: [] + ## @param readReplicas.extraPodSpec Optionally specify extra PodSpec for the PostgreSQL read only pod(s) + ## + extraPodSpec: {} + ## PostgreSQL read only service configuration + ## + service: + ## @param readReplicas.service.type Kubernetes Service type + ## + type: ClusterIP + ## @param readReplicas.service.ports.postgresql PostgreSQL service port + ## + ports: + postgresql: 5432 + ## Node ports to expose + ## NOTE: choose port between <30000-32767> + ## @param readReplicas.service.nodePorts.postgresql Node port for PostgreSQL + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePorts: + postgresql: "" + ## @param readReplicas.service.clusterIP Static clusterIP or None for headless services + ## e.g: + ## clusterIP: None + ## + clusterIP: "" + ## @param readReplicas.service.annotations Annotations for PostgreSQL read only service + ## + annotations: {} + ## @param readReplicas.service.loadBalancerIP Load balancer IP if service type is `LoadBalancer` + ## Set the LoadBalancer service type to internal only + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + loadBalancerIP: "" + ## @param readReplicas.service.externalTrafficPolicy Enable client source IP preservation + ## ref https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + ## @param readReplicas.service.loadBalancerSourceRanges Addresses that are allowed when service is LoadBalancer + ## https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## @param readReplicas.service.extraPorts Extra ports to expose in the PostgreSQL read only service + ## + extraPorts: [] + ## @param readReplicas.service.sessionAffinity Session Affinity for Kubernetes service, can be "None" or "ClientIP" + ## If "ClientIP", consecutive client requests will be directed to the same Pod + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + ## + sessionAffinity: None + ## @param readReplicas.service.sessionAffinityConfig Additional settings for the sessionAffinity + ## sessionAffinityConfig: + ## clientIP: + ## timeoutSeconds: 300 + ## + sessionAffinityConfig: {} + ## Headless service properties + ## + headless: + ## @param readReplicas.service.headless.annotations Additional custom annotations for headless PostgreSQL read only service + ## + annotations: {} + ## PostgreSQL read only persistence configuration + ## + persistence: + ## @param readReplicas.persistence.enabled Enable PostgreSQL read only data persistence using PVC + ## + enabled: true + ## @param readReplicas.persistence.existingClaim Name of an existing PVC to use + ## + existingClaim: "" + ## @param readReplicas.persistence.mountPath The path the volume will be mounted at + ## Note: useful when using custom PostgreSQL images + ## + mountPath: /bitnami/postgresql + ## @param readReplicas.persistence.subPath The subdirectory of the volume to mount to + ## Useful in dev environments and one PV for multiple services + ## + subPath: "" + ## @param readReplicas.persistence.storageClass PVC Storage Class for PostgreSQL read only data volume + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + storageClass: "" + ## @param readReplicas.persistence.accessModes PVC Access Mode for PostgreSQL volume + ## + accessModes: + - ReadWriteOnce + ## @param readReplicas.persistence.size PVC Storage Request for PostgreSQL volume + ## + size: 8Gi + ## @param readReplicas.persistence.annotations Annotations for the PVC + ## + annotations: {} + ## @param readReplicas.persistence.labels Labels for the PVC + ## + labels: {} + ## @param readReplicas.persistence.selector Selector to match an existing Persistent Volume (this value is evaluated as a template) + ## selector: + ## matchLabels: + ## app: my-app + ## + selector: {} + ## @param readReplicas.persistence.dataSource Custom PVC data source + ## + dataSource: {} + ## PostgreSQL Read only Persistent Volume Claim Retention Policy + ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention + ## + persistentVolumeClaimRetentionPolicy: + ## @param readReplicas.persistentVolumeClaimRetentionPolicy.enabled Enable Persistent volume retention policy for read only Statefulset + ## + enabled: false + ## @param readReplicas.persistentVolumeClaimRetentionPolicy.whenScaled Volume retention behavior when the replica count of the StatefulSet is reduced + ## + whenScaled: Retain + ## @param readReplicas.persistentVolumeClaimRetentionPolicy.whenDeleted Volume retention behavior that applies when the StatefulSet is deleted + ## + whenDeleted: Retain + + +## @section Backup parameters +## This section implements a trivial logical dump cronjob of the database. +## This only comes with the consistency guarantees of the dump program. +## This is not a snapshot based roll forward/backward recovery backup. +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/ +backup: + ## @param backup.enabled Enable the logical dump of the database "regularly" + enabled: false + cronjob: + ## @param backup.cronjob.schedule Set the cronjob parameter schedule + schedule: "@daily" + ## @param backup.cronjob.timeZone Set the cronjob parameter timeZone + timeZone: "" + ## @param backup.cronjob.concurrencyPolicy Set the cronjob parameter concurrencyPolicy + concurrencyPolicy: Allow + ## @param backup.cronjob.failedJobsHistoryLimit Set the cronjob parameter failedJobsHistoryLimit + failedJobsHistoryLimit: 1 + ## @param backup.cronjob.successfulJobsHistoryLimit Set the cronjob parameter successfulJobsHistoryLimit + successfulJobsHistoryLimit: 3 + ## @param backup.cronjob.startingDeadlineSeconds Set the cronjob parameter startingDeadlineSeconds + startingDeadlineSeconds: "" + ## @param backup.cronjob.ttlSecondsAfterFinished Set the cronjob parameter ttlSecondsAfterFinished + ttlSecondsAfterFinished: "" + ## @param backup.cronjob.restartPolicy Set the cronjob parameter restartPolicy + restartPolicy: OnFailure + ## @param backup.cronjob.podSecurityContext.enabled Enable PodSecurityContext for CronJob/Backup + ## @param backup.cronjob.podSecurityContext.fsGroup Group ID for the CronJob + podSecurityContext: + enabled: true + fsGroup: 1001 + ## backup container's Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + ## @param backup.cronjob.containerSecurityContext.enabled Enabled containers' Security Context + ## @param backup.cronjob.containerSecurityContext.runAsUser Set containers' Security Context runAsUser + ## @param backup.cronjob.containerSecurityContext.runAsNonRoot Set container's Security Context runAsNonRoot + ## @param backup.cronjob.containerSecurityContext.privileged Set container's Security Context privileged + ## @param backup.cronjob.containerSecurityContext.readOnlyRootFilesystem Set container's Security Context readOnlyRootFilesystem + ## @param backup.cronjob.containerSecurityContext.allowPrivilegeEscalation Set container's Security Context allowPrivilegeEscalation + ## @param backup.cronjob.containerSecurityContext.capabilities.drop List of capabilities to be dropped + ## @param backup.cronjob.containerSecurityContext.seccompProfile.type Set container's Security Context seccomp profile + containerSecurityContext: + enabled: true + runAsUser: 1001 + runAsNonRoot: true + privileged: false + readOnlyRootFilesystem: false + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: "RuntimeDefault" + ## @param backup.cronjob.command Set backup container's command to run + command: + - /bin/sh + - -c + - "pg_dumpall --clean --if-exists --load-via-partition-root --quote-all-identifiers --no-password --file=${PGDUMP_DIR}/pg_dumpall-$(date '+%Y-%m-%d-%H-%M').pgdump" + + ## @param backup.cronjob.labels Set the cronjob labels + labels: {} + ## @param backup.cronjob.annotations Set the cronjob annotations + annotations: {} + ## @param backup.cronjob.nodeSelector Node labels for PostgreSQL backup CronJob pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + storage: + ## @param backup.cronjob.storage.existingClaim Provide an existing `PersistentVolumeClaim` (only when `architecture=standalone`) + ## If defined, PVC must be created manually before volume will be bound + ## + existingClaim: "" + ## @param backup.cronjob.storage.resourcePolicy Setting it to "keep" to avoid removing PVCs during a helm delete operation. Leaving it empty will delete PVCs after the chart deleted + ## + resourcePolicy: "" + ## @param backup.cronjob.storage.storageClass PVC Storage Class for the backup data volume + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. + ## + storageClass: "" + ## @param backup.cronjob.storage.accessModes PV Access Mode + ## + accessModes: + - ReadWriteOnce + ## @param backup.cronjob.storage.size PVC Storage Request for the backup data volume + ## + size: 8Gi + ## @param backup.cronjob.storage.annotations PVC annotations + ## + annotations: {} + ## @param backup.cronjob.storage.mountPath Path to mount the volume at + ## + mountPath: /backup/pgdump + ## @param backup.cronjob.storage.subPath Subdirectory of the volume to mount at + ## and one PV for multiple services. + ## + subPath: "" + ## Fine tuning for volumeClaimTemplates + ## + volumeClaimTemplates: + ## @param backup.cronjob.storage.volumeClaimTemplates.selector A label query over volumes to consider for binding (e.g. when using local volumes) + ## A label query over volumes to consider for binding (e.g. when using local volumes) + ## See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#labelselector-v1-meta for more details + ## + selector: {} + +## @section NetworkPolicy parameters +## + +## Add networkpolicies +## +networkPolicy: + ## @param networkPolicy.enabled Enable network policies + ## + enabled: false + ## @param networkPolicy.metrics.enabled Enable network policies for metrics (prometheus) + ## @param networkPolicy.metrics.namespaceSelector [object] Monitoring namespace selector labels. These labels will be used to identify the prometheus' namespace. + ## @param networkPolicy.metrics.podSelector [object] Monitoring pod selector labels. These labels will be used to identify the Prometheus pods. + ## + metrics: + enabled: false + ## e.g: + ## namespaceSelector: + ## label: monitoring + ## + namespaceSelector: {} + ## e.g: + ## podSelector: + ## label: monitoring + ## + podSelector: {} + ## Ingress Rules + ## + ingressRules: + ## @param networkPolicy.ingressRules.primaryAccessOnlyFrom.enabled Enable ingress rule that makes PostgreSQL primary node only accessible from a particular origin. + ## @param networkPolicy.ingressRules.primaryAccessOnlyFrom.namespaceSelector [object] Namespace selector label that is allowed to access the PostgreSQL primary node. This label will be used to identified the allowed namespace(s). + ## @param networkPolicy.ingressRules.primaryAccessOnlyFrom.podSelector [object] Pods selector label that is allowed to access the PostgreSQL primary node. This label will be used to identified the allowed pod(s). + ## @param networkPolicy.ingressRules.primaryAccessOnlyFrom.customRules Custom network policy for the PostgreSQL primary node. + ## + primaryAccessOnlyFrom: + enabled: false + ## e.g: + ## namespaceSelector: + ## label: ingress + ## + namespaceSelector: {} + ## e.g: + ## podSelector: + ## label: access + ## + podSelector: {} + ## custom ingress rules + ## e.g: + ## customRules: + ## - from: + ## - namespaceSelector: + ## matchLabels: + ## label: example + ## + customRules: [] + ## @param networkPolicy.ingressRules.readReplicasAccessOnlyFrom.enabled Enable ingress rule that makes PostgreSQL read-only nodes only accessible from a particular origin. + ## @param networkPolicy.ingressRules.readReplicasAccessOnlyFrom.namespaceSelector [object] Namespace selector label that is allowed to access the PostgreSQL read-only nodes. This label will be used to identified the allowed namespace(s). + ## @param networkPolicy.ingressRules.readReplicasAccessOnlyFrom.podSelector [object] Pods selector label that is allowed to access the PostgreSQL read-only nodes. This label will be used to identified the allowed pod(s). + ## @param networkPolicy.ingressRules.readReplicasAccessOnlyFrom.customRules Custom network policy for the PostgreSQL read-only nodes. + ## + readReplicasAccessOnlyFrom: + enabled: false + ## e.g: + ## namespaceSelector: + ## label: ingress + ## + namespaceSelector: {} + ## e.g: + ## podSelector: + ## label: access + ## + podSelector: {} + ## custom ingress rules + ## e.g: + ## CustomRules: + ## - from: + ## - namespaceSelector: + ## matchLabels: + ## label: example + ## + customRules: [] + ## @param networkPolicy.egressRules.denyConnectionsToExternal Enable egress rule that denies outgoing traffic outside the cluster, except for DNS (port 53). + ## @param networkPolicy.egressRules.customRules Custom network policy rule + ## + egressRules: + # Deny connections to external. This is not compatible with an external database. + denyConnectionsToExternal: false + ## Additional custom egress rules + ## e.g: + ## customRules: + ## - to: + ## - namespaceSelector: + ## matchLabels: + ## label: example + ## + customRules: [] + +## @section Volume Permissions parameters +## + +## Init containers parameters: +## volumePermissions: Change the owner and group of the persistent volume(s) mountpoint(s) to 'runAsUser:fsGroup' on each node +## +volumePermissions: + ## @param volumePermissions.enabled Enable init container that changes the owner and group of the persistent volume + ## + enabled: false + ## @param volumePermissions.image.registry [default: REGISTRY_NAME] Init container volume-permissions image registry + ## @param volumePermissions.image.repository [default: REPOSITORY_NAME/os-shell] Init container volume-permissions image repository + ## @skip volumePermissions.image.tag Init container volume-permissions image tag (immutable tags are recommended) + ## @param volumePermissions.image.digest Init container volume-permissions image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag + ## @param volumePermissions.image.pullPolicy Init container volume-permissions image pull policy + ## @param volumePermissions.image.pullSecrets Init container volume-permissions image pull secrets + ## + image: + registry: docker.io + repository: bitnami/os-shell + tag: 11-debian-11-r91 + digest: "" + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## Init container resource requests and limits + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## @param volumePermissions.resources.limits Init container volume-permissions resource limits + ## @param volumePermissions.resources.requests Init container volume-permissions resource requests + ## + resources: + limits: {} + requests: {} + ## Init container' Security Context + ## Note: the chown of the data folder is done to containerSecurityContext.runAsUser + ## and not the below volumePermissions.containerSecurityContext.runAsUser + ## @param volumePermissions.containerSecurityContext.runAsUser User ID for the init container + ## @param volumePermissions.containerSecurityContext.runAsGroup Group ID for the init container + ## @param volumePermissions.containerSecurityContext.runAsNonRoot runAsNonRoot for the init container + ## @param volumePermissions.containerSecurityContext.seccompProfile.type seccompProfile.type for the init container + ## + containerSecurityContext: + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false + seccompProfile: + type: RuntimeDefault +## @section Other Parameters +## + +## @param serviceBindings.enabled Create secret for service binding (Experimental) +## Ref: https://servicebinding.io/service-provider/ +## +serviceBindings: + enabled: false + +## Service account for PostgreSQL to use. +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +## +serviceAccount: + ## @param serviceAccount.create Enable creation of ServiceAccount for PostgreSQL pod + ## + create: false + ## @param serviceAccount.name The name of the ServiceAccount to use. + ## If not set and create is true, a name is generated using the common.names.fullname template + ## + name: "" + ## @param serviceAccount.automountServiceAccountToken Allows auto mount of ServiceAccountToken on the serviceAccount created + ## Can be set to false if pods using this serviceAccount do not need to use K8s API + ## + automountServiceAccountToken: true + ## @param serviceAccount.annotations Additional custom annotations for the ServiceAccount + ## + annotations: {} +## Creates role for ServiceAccount +## @param rbac.create Create Role and RoleBinding (required for PSP to work) +## +rbac: + create: false + ## @param rbac.rules Custom RBAC rules to set + ## e.g: + ## rules: + ## - apiGroups: + ## - "" + ## resources: + ## - pods + ## verbs: + ## - get + ## - list + ## + rules: [] +## Pod Security Policy +## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +## @param psp.create Whether to create a PodSecurityPolicy. WARNING: PodSecurityPolicy is deprecated in Kubernetes v1.21 or later, unavailable in v1.25 or later +## +psp: + create: false + +## @section Metrics Parameters +## + +metrics: + ## @param metrics.enabled Start a prometheus exporter + ## + enabled: false + ## @param metrics.image.registry [default: REGISTRY_NAME] PostgreSQL Prometheus Exporter image registry + ## @param metrics.image.repository [default: REPOSITORY_NAME/postgres-exporter] PostgreSQL Prometheus Exporter image repository + ## @skip metrics.image.tag PostgreSQL Prometheus Exporter image tag (immutable tags are recommended) + ## @param metrics.image.digest PostgreSQL image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag + ## @param metrics.image.pullPolicy PostgreSQL Prometheus Exporter image pull policy + ## @param metrics.image.pullSecrets Specify image pull secrets + ## + image: + registry: docker.io + repository: bitnami/postgres-exporter + tag: 0.15.0-debian-11-r2 + digest: "" + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param metrics.collectors Control enabled collectors + ## ref: https://github.com/prometheus-community/postgres_exporter#flags + ## Example: + ## collectors: + ## wal: false + collectors: {} + ## @param metrics.customMetrics Define additional custom metrics + ## ref: https://github.com/prometheus-community/postgres_exporter#adding-new-metrics-via-a-config-file-deprecated + ## customMetrics: + ## pg_database: + ## query: "SELECT d.datname AS name, CASE WHEN pg_catalog.has_database_privilege(d.datname, 'CONNECT') THEN pg_catalog.pg_database_size(d.datname) ELSE 0 END AS size_bytes FROM pg_catalog.pg_database d where datname not in ('template0', 'template1', 'postgres')" + ## metrics: + ## - name: + ## usage: "LABEL" + ## description: "Name of the database" + ## - size_bytes: + ## usage: "GAUGE" + ## description: "Size of the database in bytes" + ## + customMetrics: {} + ## @param metrics.extraEnvVars Extra environment variables to add to PostgreSQL Prometheus exporter + ## see: https://github.com/prometheus-community/postgres_exporter#environment-variables + ## For example: + ## extraEnvVars: + ## - name: PG_EXPORTER_DISABLE_DEFAULT_METRICS + ## value: "true" + ## + extraEnvVars: [] + ## PostgreSQL Prometheus exporter containers' Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + ## @param metrics.containerSecurityContext.enabled Enabled containers' Security Context + ## @param metrics.containerSecurityContext.runAsUser Set containers' Security Context runAsUser + ## @param metrics.containerSecurityContext.runAsNonRoot Set container's Security Context runAsNonRoot + ## @param metrics.containerSecurityContext.privileged Set container's Security Context privileged + ## @param metrics.containerSecurityContext.readOnlyRootFilesystem Set container's Security Context readOnlyRootFilesystem + ## @param metrics.containerSecurityContext.allowPrivilegeEscalation Set container's Security Context allowPrivilegeEscalation + ## @param metrics.containerSecurityContext.capabilities.drop List of capabilities to be dropped + ## @param metrics.containerSecurityContext.seccompProfile.type Set container's Security Context seccomp profile + ## + containerSecurityContext: + enabled: true + runAsUser: 1001 + runAsNonRoot: true + privileged: false + readOnlyRootFilesystem: false + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: "RuntimeDefault" + ## Configure extra options for PostgreSQL Prometheus exporter containers' liveness, readiness and startup probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes + ## @param metrics.livenessProbe.enabled Enable livenessProbe on PostgreSQL Prometheus exporter containers + ## @param metrics.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe + ## @param metrics.livenessProbe.periodSeconds Period seconds for livenessProbe + ## @param metrics.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe + ## @param metrics.livenessProbe.failureThreshold Failure threshold for livenessProbe + ## @param metrics.livenessProbe.successThreshold Success threshold for livenessProbe + ## + livenessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + ## @param metrics.readinessProbe.enabled Enable readinessProbe on PostgreSQL Prometheus exporter containers + ## @param metrics.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe + ## @param metrics.readinessProbe.periodSeconds Period seconds for readinessProbe + ## @param metrics.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe + ## @param metrics.readinessProbe.failureThreshold Failure threshold for readinessProbe + ## @param metrics.readinessProbe.successThreshold Success threshold for readinessProbe + ## + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + ## @param metrics.startupProbe.enabled Enable startupProbe on PostgreSQL Prometheus exporter containers + ## @param metrics.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe + ## @param metrics.startupProbe.periodSeconds Period seconds for startupProbe + ## @param metrics.startupProbe.timeoutSeconds Timeout seconds for startupProbe + ## @param metrics.startupProbe.failureThreshold Failure threshold for startupProbe + ## @param metrics.startupProbe.successThreshold Success threshold for startupProbe + ## + startupProbe: + enabled: false + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 15 + successThreshold: 1 + ## @param metrics.customLivenessProbe Custom livenessProbe that overrides the default one + ## + customLivenessProbe: {} + ## @param metrics.customReadinessProbe Custom readinessProbe that overrides the default one + ## + customReadinessProbe: {} + ## @param metrics.customStartupProbe Custom startupProbe that overrides the default one + ## + customStartupProbe: {} + ## @param metrics.containerPorts.metrics PostgreSQL Prometheus exporter metrics container port + ## + containerPorts: + metrics: 9187 + ## PostgreSQL Prometheus exporter resource requests and limits + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## @param metrics.resources.limits The resources limits for the PostgreSQL Prometheus exporter container + ## @param metrics.resources.requests The requested resources for the PostgreSQL Prometheus exporter container + ## + resources: + limits: {} + requests: {} + ## Service configuration + ## + service: + ## @param metrics.service.ports.metrics PostgreSQL Prometheus Exporter service port + ## + ports: + metrics: 9187 + ## @param metrics.service.clusterIP Static clusterIP or None for headless services + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address + ## + clusterIP: "" + ## @param metrics.service.sessionAffinity Control where client requests go, to the same pod or round-robin + ## Values: ClientIP or None + ## ref: https://kubernetes.io/docs/user-guide/services/ + ## + sessionAffinity: None + ## @param metrics.service.annotations [object] Annotations for Prometheus to auto-discover the metrics endpoint + ## + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.metrics.service.ports.metrics }}" + ## Prometheus Operator ServiceMonitor configuration + ## + serviceMonitor: + ## @param metrics.serviceMonitor.enabled Create ServiceMonitor Resource for scraping metrics using Prometheus Operator + ## + enabled: false + ## @param metrics.serviceMonitor.namespace Namespace for the ServiceMonitor Resource (defaults to the Release Namespace) + ## + namespace: "" + ## @param metrics.serviceMonitor.interval Interval at which metrics should be scraped. + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + interval: "" + ## @param metrics.serviceMonitor.scrapeTimeout Timeout after which the scrape is ended + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + scrapeTimeout: "" + ## @param metrics.serviceMonitor.labels Additional labels that can be used so ServiceMonitor will be discovered by Prometheus + ## + labels: {} + ## @param metrics.serviceMonitor.selector Prometheus instance selector labels + ## ref: https://github.com/bitnami/charts/tree/main/bitnami/prometheus-operator#prometheus-configuration + ## + selector: {} + ## @param metrics.serviceMonitor.relabelings RelabelConfigs to apply to samples before scraping + ## + relabelings: [] + ## @param metrics.serviceMonitor.metricRelabelings MetricRelabelConfigs to apply to samples before ingestion + ## + metricRelabelings: [] + ## @param metrics.serviceMonitor.honorLabels Specify honorLabels parameter to add the scrape endpoint + ## + honorLabels: false + ## @param metrics.serviceMonitor.jobLabel The name of the label on the target service to use as the job name in prometheus. + ## + jobLabel: "" + ## Custom PrometheusRule to be defined + ## The value is evaluated as a template, so, for example, the value can depend on .Release or .Chart + ## ref: https://github.com/coreos/prometheus-operator#customresourcedefinitions + ## + prometheusRule: + ## @param metrics.prometheusRule.enabled Create a PrometheusRule for Prometheus Operator + ## + enabled: false + ## @param metrics.prometheusRule.namespace Namespace for the PrometheusRule Resource (defaults to the Release Namespace) + ## + namespace: "" + ## @param metrics.prometheusRule.labels Additional labels that can be used so PrometheusRule will be discovered by Prometheus + ## + labels: {} + ## @param metrics.prometheusRule.rules PrometheusRule definitions + ## Make sure to constraint the rules to the current postgresql service. + ## rules: + ## - alert: HugeReplicationLag + ## expr: pg_replication_lag{service="{{ printf "%s-metrics" (include "common.names.fullname" .) }}"} / 3600 > 1 + ## for: 1m + ## labels: + ## severity: critical + ## annotations: + ## description: replication for {{ include "common.names.fullname" . }} PostgreSQL is lagging by {{ "{{ $value }}" }} hour(s). + ## summary: PostgreSQL replication is lagging by {{ "{{ $value }}" }} hour(s). + ## + rules: [] diff --git a/charts/airflow/dockerfiles/README.md b/charts/airflow/dockerfiles/README.md new file mode 100644 index 0000000..cd2a1e2 --- /dev/null +++ b/charts/airflow/dockerfiles/README.md @@ -0,0 +1,28 @@ + + +Those are images that are needed for the Helm Chart. + +In each of the images you can find "build_and_push.sh" script that builds and pushes the image. + +You need to be a PMC member with direct push access to "apache/airflow" DockerHub registry +to be able to push to the Airflow DockerHub registry. + +You can set the DOCKERHUB_USER variable to push to your own DockerHub user if you want + to test the image or build your own image. diff --git a/charts/airflow/dockerfiles/pgbouncer-exporter/Dockerfile b/charts/airflow/dockerfiles/pgbouncer-exporter/Dockerfile new file mode 100644 index 0000000..deea77f --- /dev/null +++ b/charts/airflow/dockerfiles/pgbouncer-exporter/Dockerfile @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG ALPINE_VERSION="3.19" +ARG GO_VERSION + +FROM golang:${GO_VERSION} AS builder + +ARG PGBOUNCER_EXPORTER_VERSION + +WORKDIR /usr/src/myapp + +SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] + +RUN URL="https://github.com/jbub/pgbouncer_exporter/archive/v${PGBOUNCER_EXPORTER_VERSION}.tar.gz" \ + && curl -L "${URL}" | tar -zx --strip-components 1 \ + && PLATFORM=$([ "$(uname -m)" = "aarch64" ] && echo "arm64" || echo "amd64" )\ + && GOOS=linux GOARCH="${PLATFORM}" CGO_ENABLED=0 go build -v + +FROM alpine:${ALPINE_VERSION} AS final + +# We want to make sure this one includes latest security fixes. +# "Pin versions in apk add" https://github.com/hadolint/hadolint/wiki/DL3018 +# hadolint ignore=DL3018 +RUN apk --no-cache add libressl libressl-dev openssl + +COPY --from=builder /usr/src/myapp/pgbouncer_exporter /bin + +ARG PGBOUNCER_EXPORTER_VERSION +ARG AIRFLOW_PGBOUNCER_EXPORTER_VERSION +ARG GO_VERSION +ARG COMMIT_SHA + +LABEL org.apache.airflow.component="pgbouncer-exporter" \ + org.apache.airflow.pgbouncer-exporter.version="${PGBOUNCER_EXPORTER_VERSION}" \ + org.apache.airflow.go.version="${GO_VERSION}" \ + org.apache.airflow.airflow-pgbouncer-exporter.version="${AIRFLOW_PGBOUNCER_EXPORTER_VERSION}" \ + org.apache.airflow.commit-sha="${COMMIT_SHA}" \ + maintainer="Apache Airflow Community " + +HEALTHCHECK CMD ["/bin/pgbouncer_exporter", "health"] + +USER nobody + +ENTRYPOINT ["/bin/pgbouncer_exporter"] +CMD ["server"] diff --git a/charts/airflow/dockerfiles/pgbouncer-exporter/build_and_push.sh b/charts/airflow/dockerfiles/pgbouncer-exporter/build_and_push.sh new file mode 100644 index 0000000..a177481 --- /dev/null +++ b/charts/airflow/dockerfiles/pgbouncer-exporter/build_and_push.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +set -euo pipefail +DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} +readonly DOCKERHUB_USER +DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"} +readonly DOCKERHUB_REPO + +PGBOUNCER_EXPORTER_VERSION="0.18.0" +readonly PGBOUNCER_EXPORTER_VERSION + +AIRFLOW_PGBOUNCER_EXPORTER_VERSION="2025.03.05" +readonly AIRFLOW_PGBOUNCER_EXPORTER_VERSION + +EXPECTED_GO_VERSION="1.23.7" +readonly EXPECTED_GO_VERSION + +COMMIT_SHA=$(git rev-parse HEAD) +readonly COMMIT_SHA + +TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:airflow-pgbouncer-exporter-${AIRFLOW_PGBOUNCER_EXPORTER_VERSION}-${PGBOUNCER_EXPORTER_VERSION}" +readonly TAG + +function center_text() { + columns=$(tput cols || echo 80) + printf "%*s\n" $(( (${#1} + columns) / 2)) "$1" +} + +cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1 + +center_text "Building image" + +# Note, you need buildx and qemu installed for your docker. They come pre-installed with docker-desktop, but +# as described in: +# * https://docs.docker.com/build/install-buildx/ +# * https://docs.docker.com/build/building/multi-platform/ +# You can also install them easily on all docker-based systems +# You might also need to create a different builder to build multi-platform images +# For example by running `docker buildx create --use` + +docker buildx build . \ + --platform linux/amd64,linux/arm64 \ + --pull \ + --push \ + --build-arg "PGBOUNCER_EXPORTER_VERSION=${PGBOUNCER_EXPORTER_VERSION}" \ + --build-arg "AIRFLOW_PGBOUNCER_EXPORTER_VERSION=${AIRFLOW_PGBOUNCER_EXPORTER_VERSION}"\ + --build-arg "COMMIT_SHA=${COMMIT_SHA}" \ + --build-arg "GO_VERSION=${EXPECTED_GO_VERSION}" \ + --tag "${TAG}" + +center_text "Checking image" + +docker run --rm "${TAG}" --version diff --git a/charts/airflow/dockerfiles/pgbouncer/Dockerfile b/charts/airflow/dockerfiles/pgbouncer/Dockerfile new file mode 100644 index 0000000..70236d2 --- /dev/null +++ b/charts/airflow/dockerfiles/pgbouncer/Dockerfile @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +ARG ALPINE_VERSION="3.19" +FROM alpine:${ALPINE_VERSION} AS builder +SHELL ["/bin/ash", "-e", "-x", "-c", "-o", "pipefail"] + +ARG PGBOUNCER_TAG +ARG PGBOUNCER_VERSION +ARG AIRFLOW_PGBOUNCER_VERSION + +ARG PGBOUNCER_SHA256 + +# Those are build deps only but still we want the latest versions of those +# "Pin versions in apk add" https://github.com/hadolint/hadolint/wiki/DL3018 +# hadolint ignore=DL3018 +RUN apk --no-cache add make pkgconfig build-base libtool wget gcc g++ libevent-dev openssl-dev c-ares-dev ca-certificates +# We are not using Dash so we can safely ignore the "Dash warning" +# "In dash, something is not supported." https://github.com/koalaman/shellcheck/wiki/SC2169 +# hadolint ignore=SC2169,SC3060 +RUN wget --progress=dot:giga "https://github.com/pgbouncer/pgbouncer/releases/download/${PGBOUNCER_TAG}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" \ + && echo "${PGBOUNCER_SHA256} pgbouncer-${PGBOUNCER_VERSION}.tar.gz" | sha256sum -c - \ + && tar -xzvf pgbouncer-$PGBOUNCER_VERSION.tar.gz + +WORKDIR /pgbouncer-$PGBOUNCER_VERSION +RUN ./configure --prefix=/usr --disable-debug && make && make install \ + && mkdir /etc/pgbouncer \ + && cp ./etc/pgbouncer.ini /etc/pgbouncer/ \ + && touch /etc/pgbouncer/userlist.txt \ + && sed -i -e "s|logfile = |#logfile = |" \ + -e "s|pidfile = |#pidfile = |" \ + -e "s|listen_addr = .*|listen_addr = 0.0.0.0|" \ + -e "s|auth_type = .*|auth_type = md5|" \ + /etc/pgbouncer/pgbouncer.ini + +FROM alpine:${ALPINE_VERSION} + +ARG PGBOUNCER_VERSION +ARG AIRFLOW_PGBOUNCER_VERSION +ARG COMMIT_SHA + + +# We want to make sure this one includes latest security fixes. +# "Pin versions in apk add" https://github.com/hadolint/hadolint/wiki/DL3018 +# hadolint ignore=DL3018 +RUN apk --no-cache add libevent libressl c-ares + +COPY --from=builder /etc/pgbouncer /etc/pgbouncer +COPY --from=builder /usr/bin/pgbouncer /usr/bin/pgbouncer + +LABEL org.apache.airflow.component="pgbouncer" \ + org.apache.airflow.pgbouncer.version="${PGBOUNCER_VERSION}" \ + org.apache.airflow.airflow-pgbouncer.version="${AIRFLOW_PGBOUNCER_VERSION}" \ + org.apache.airflow.commit-sha="${COMMIT_SHA}" \ + maintainer="Apache Airflow Community " + +# Healthcheck +HEALTHCHECK --interval=10s --timeout=3s CMD stat /tmp/.s.PGSQL.* + +EXPOSE 6432 + +USER nobody + +# pgbouncer can't run as root, so let's drop to 'nobody' +ENTRYPOINT ["/usr/bin/pgbouncer", "-u", "nobody", "/etc/pgbouncer/pgbouncer.ini" ] diff --git a/charts/airflow/dockerfiles/pgbouncer/build_and_push.sh b/charts/airflow/dockerfiles/pgbouncer/build_and_push.sh new file mode 100644 index 0000000..333e470 --- /dev/null +++ b/charts/airflow/dockerfiles/pgbouncer/build_and_push.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +set -euo pipefail +DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} +readonly DOCKERHUB_USER + +DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"} +readonly DOCKERHUB_REPO + +# Sometimes the pgbouncer tag does not reliably correspond with the version name +# For example, it may have a `-fixed` suffix +PGBOUNCER_TAG="pgbouncer_1_23_1-fixed" +readonly PGBOUNCER_TAG + +PGBOUNCER_VERSION="1.23.1" +readonly PGBOUNCER_VERSION + +PGBOUNCER_SHA256="1963b497231d9a560a62d266e4a2eae6881ab401853d93e5d292c3740eec5084" +readonly PGBOUNCER_SHA256 + +AIRFLOW_PGBOUNCER_VERSION="2025.03.05" +readonly AIRFLOW_PGBOUNCER_VERSION + +COMMIT_SHA=$(git rev-parse HEAD) +readonly COMMIT_SHA + +TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:airflow-pgbouncer-${AIRFLOW_PGBOUNCER_VERSION}-${PGBOUNCER_VERSION}" +readonly TAG + +function center_text() { + columns=$(tput cols || echo 80) + printf "%*s\n" $(( (${#1} + columns) / 2)) "$1" +} + +cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1 + +center_text "Building image" + +# Note, you need buildx and qemu installed for your docker. They come pre-installed with docker-desktop, but +# as described in: +# * https://docs.docker.com/build/install-buildx/ +# * https://docs.docker.com/build/building/multi-platform/ +# You can also install them easily on all docker-based systems +# You might also need to create a different builder to build multi-platform images +# For example by running `docker buildx create --use` + +docker buildx build . \ + --platform linux/amd64,linux/arm64 \ + --pull \ + --push \ + --build-arg "PGBOUNCER_TAG=${PGBOUNCER_TAG}" \ + --build-arg "PGBOUNCER_VERSION=${PGBOUNCER_VERSION}" \ + --build-arg "AIRFLOW_PGBOUNCER_VERSION=${AIRFLOW_PGBOUNCER_VERSION}"\ + --build-arg "PGBOUNCER_SHA256=${PGBOUNCER_SHA256}"\ + --build-arg "COMMIT_SHA=${COMMIT_SHA}" \ + --tag "${TAG}" + +center_text "Checking image" + +docker run --rm "${TAG}" pgbouncer --version diff --git a/charts/airflow/docs/adding-connections-and-variables.rst b/charts/airflow/docs/adding-connections-and-variables.rst new file mode 100644 index 0000000..8f1e3d2 --- /dev/null +++ b/charts/airflow/docs/adding-connections-and-variables.rst @@ -0,0 +1,89 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +Adding Connections, Variables and Environment Variables +======================================================= + +You can programmatically add Connections, Variables and arbitrary Environment Variables to your +Airflow deployment using the Helm Chart. + + +Connections and Sensitive Environment Variables +----------------------------------------------- +Under the ``secret`` and ``extraSecret`` sections of the ``values.yaml``, you can pass connection strings and sensitive +environment variables into Airflow using the Helm Chart. To illustrate, lets create a yaml file called ``override.yaml`` +to override values under these sections of the ``values.yaml`` file. + +.. code-block:: yaml + :caption: override.yaml + + secret: + - envName: "AIRFLOW_CONN_GCP" + secretName: "my-airflow-connections" + secretKey: "AIRFLOW_CONN_GCP" + - envName: "my-env" + secretName: "my-secret-name" + secretKey: "my-secret-key" + + extraSecrets: + my-airflow-connections: + data: | + AIRFLOW_CONN_GCP: 'base64_encoded_gcp_conn_string' + my-secret-name: + stringData: | + my-secret-key: my-secret + +.. warning:: + + Due to security concerns, it is not advised to define sensitive secrets values within ``values.yaml`` file. + +Variables +--------- +Airflow supports Variables which enable users to craft dynamic Dags. You can set Variables in Airflow in three ways - UI, +command line, and within your Dag file. See :doc:`apache-airflow:howto/variable` for more. + +With the Helm Chart, you can also inject environment variables into Airflow. In the ``override.yaml`` example file, +we can override values of interest in the ``env`` section of the ``values.yaml`` file. + +.. code-block:: yaml + :caption: override.yaml + + env: + - name: "AIRFLOW_VAR_KEY" + value: "value_1" + - name: "AIRFLOW_VAR_ANOTHER_KEY" + value: "value_2" + + +You can also utilize ``extraEnv`` and ``extraEnvFrom`` if you need the name or value to be templated. + +.. code-block:: yaml + :caption: override.yaml + + extraEnv: | + - name: AIRFLOW_VAR_HELM_RELEASE_NAME + value: '{{ .Release.Name }}' + + extraEnvFrom: | + - configMapRef: + name: '{{ .Release.Name }}-airflow-variables' + + extraConfigMaps: + '{{ .Release.Name }}-airflow-variables': + data: | + AIRFLOW_VAR_HELLO_MESSAGE: "Hi!" diff --git a/charts/airflow/docs/airflow-configuration.rst b/charts/airflow/docs/airflow-configuration.rst new file mode 100644 index 0000000..d8bda1c --- /dev/null +++ b/charts/airflow/docs/airflow-configuration.rst @@ -0,0 +1,42 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Configuring Airflow +=================== + +The Helm Chart allows for setting arbitrary Airflow configuration in values file under the ``config`` key. +Some of the defaults in the chart may differ from those of core Airflow and can be found in +`values.yaml `__. + +As an example of setting arbitrary configuration, the following ``override.yaml`` file demonstrates how one would +allow Airflow UI users to view the Airflow configuration directly via UI: + +.. code-block:: yaml + :caption: override.yaml + + config: + api: + expose_config: 'True' # by default this is 'False' + +Generally speaking, it is useful to familiarize oneself with the Airflow +configuration prior to installing and deploying the service. + +.. note:: + + The recommended way to load example Dags using the official Docker image and chart is to configure the ``AIRFLOW__CORE__LOAD_EXAMPLES`` environment variable + in ``extraEnv`` (see :doc:`Parameters reference `). The official Docker image has ``AIRFLOW__CORE__LOAD_EXAMPLES=False`` + set within the image, so you need to override it with an environment variable, when deploying the chart, in order for the examples to be present. diff --git a/charts/airflow/docs/conf.py b/charts/airflow/docs/conf.py new file mode 100644 index 0000000..97e2c9d --- /dev/null +++ b/charts/airflow/docs/conf.py @@ -0,0 +1,379 @@ +# Disable Flake8 because of all the sphinx imports +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Configuration of Airflow Chart Docs.""" + +from __future__ import annotations + +# Airflow documentation build configuration file, created by +# sphinx-quickstart on Thu Oct 9 20:50:01 2014. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. +import json +import logging +import os +import re +from typing import Any + +import yaml +from docs.utils.conf_constants import ( + AIRFLOW_FAVICON_PATH, + AIRFLOW_REPO_ROOT_PATH, + AUTOAPI_OPTIONS, + BASIC_AUTOAPI_IGNORE_PATTERNS, + BASIC_SPHINX_EXTENSIONS, + SMARTQUOTES_EXCLUDES, + SPELLING_WORDLIST_PATH, + SPHINX_DESIGN_STATIC_PATH, + SUPPRESS_WARNINGS, + filter_autoapi_ignore_entries, + get_autodoc_mock_imports, + get_html_context, + get_html_sidebars, + get_html_theme_options, + get_intersphinx_mapping, + get_rst_epilogue, +) +from packaging.version import parse as parse_version + +import airflow + +PACKAGE_NAME = "helm-chart" +CHART_ROOT_PATH = AIRFLOW_REPO_ROOT_PATH / "chart" +CHART_DOC_PATH = CHART_ROOT_PATH / "docs" +CHART_STATIC_PATH = CHART_DOC_PATH / "static" +os.environ["AIRFLOW_PACKAGE_NAME"] = PACKAGE_NAME + +CHART_YAML_FILE_PATH = CHART_ROOT_PATH / "Chart.yaml" +with CHART_YAML_FILE_PATH.open() as chart_file: + chart_yaml_contents = yaml.safe_load(chart_file) + +PACKAGE_VERSION: str = chart_yaml_contents["version"] + +# Adds to environment variables for easy access from other plugins like airflow_intersphinx. +os.environ["AIRFLOW_PACKAGE_NAME"] = PACKAGE_NAME + +# Hack to allow changing for piece of the code to behave differently while +# the docs are being built. The main objective was to alter the +# behavior of the utils.apply_default that was hiding function headers +os.environ["BUILDING_AIRFLOW_DOCS"] = "TRUE" + +# Use for generate rst_epilog and other post-generation substitutions +global_substitutions = { + "version": PACKAGE_VERSION, + "airflow-version": airflow.__version__, + "experimental": "This is an :ref:`experimental feature `.", +} + +# == Sphinx configuration ====================================================== + +# -- Project information ------------------------------------------------------- +# See: https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +# General information about the project. +project = PACKAGE_NAME +# # The version info for the project you're documenting +version = PACKAGE_VERSION +# The full version, including alpha/beta/rc tags. +release = PACKAGE_VERSION + +# -- General configuration ----------------------------------------------------- +# See: https://www.sphinx-doc.org/en/master/usage/configuration.html + +rst_epilog = get_rst_epilogue(PACKAGE_VERSION, False) + +smartquotes_excludes = SMARTQUOTES_EXCLUDES + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = BASIC_SPHINX_EXTENSIONS + +extensions.append("sphinx_jinja") + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns: list[str] = [] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["templates"] + +# If true, keep warnings as "system message" paragraphs in the built documents. +keep_warnings = True + +# -- Options for HTML output --------------------------------------------------- +# See: https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = "sphinx_airflow_theme" + +html_title = f"{PACKAGE_NAME} Documentation" + +conf_py_path = "/chart/docs/" +# A dictionary of values to pass into the template engine's context for all pages. +html_context = get_html_context(conf_py_path) + +# A shorter title for the navigation bar. Default is the same as html_title. +html_short_title = "" + +# given, this must be the name of an image file (path relative to the +# configuration directory) that is the favicon of the docs. Modern browsers +# use this as the icon for tabs, windows and bookmarks. It should be a +# Windows-style icon file (.ico), which is 16x16 or 32x32 pixels large. +html_favicon = AIRFLOW_FAVICON_PATH.as_posix() + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = [CHART_STATIC_PATH.as_posix(), SPHINX_DESIGN_STATIC_PATH.as_posix()] + +html_js_files = ["gh-jira-links.js"] + +html_css_files = ["custom.css"] + +# -- Theme configuration ------------------------------------------------------- +# Custom sidebar templates, maps document names to template names. +html_sidebars = get_html_sidebars(PACKAGE_VERSION) + +# If false, no index is generated. +html_use_index = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +html_show_copyright = False + +html_theme_options: dict[str, Any] = get_html_theme_options() + +# A dictionary of values to pass into the template engine's context for all pages. +html_context = get_html_context(conf_py_path) + +# == Extensions configuration ================================================== + +# -- Options for sphinx_jinja ------------------------------------------ +# See: https://github.com/tardyp/sphinx-jinja + +airflow_version = parse_version( + re.search( # type: ignore[union-attr,arg-type] + r"__version__ = \"([0-9\.]*)(\.dev[0-9]*)?\"", + (AIRFLOW_REPO_ROOT_PATH / "airflow-core" / "src" / "airflow" / "__init__.py").read_text(), + ).groups(0)[0] +) + + +def _str_representer(dumper, data): + style = "|" if "\n" in data else None # show as a block scalar if we have more than 1 line + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style) + + +yaml.add_representer(str, _str_representer) + + +def _format_default(value: Any) -> str: + if value == "": + return '""' + if value is None: + return "~" + return str(value) + + +def _format_examples(param_name: str, schema: dict) -> str | None: + if not schema.get("examples"): + return None + + # Nicer to have the parameter name shown as well + out = "" + for ex_data in schema["examples"]: + ex = [ex_data] if schema["type"] == "array" else ex_data + out += yaml.dump({param_name: ex}) + return out + + +def _get_params(root_schema: dict, prefix: str = "", default_section: str = "") -> list[dict]: + """ + Retrieve params. + + Given an jsonschema objects properties dict, return a flattened list of all parameters + from that object and any nested objects + """ + out = [] + for param_name, schema in root_schema.items(): + prefixed_name = f"{prefix}.{param_name}" if prefix else param_name + section_name = schema["x-docsSection"] if "x-docsSection" in schema else default_section + common_out = { + "section": section_name, + "name": prefixed_name, + } + if section_name and "description" in schema and schema["description"] and "default" in schema: + out.append( + { + **common_out, + "description": schema["description"], + "default": _format_default(schema["default"]), + "examples": _format_examples(param_name, schema), + } + ) + if schema.get("properties"): + out += _get_params(schema["properties"], prefixed_name, section_name) + items = schema.get("items") + if items: + out.extend(_process_array_items(items, prefixed_name, param_name, section_name)) + return out + + +def _process_array_items(items: dict, parent_name: str, param_name: str, default_section: str) -> list[dict]: + """Extract parameters from array item schemas.""" + item_prefix = f"{parent_name}[]" + section_name = items.get("x-docsSection", default_section) + + if items.get("properties"): + return _get_params(items["properties"], item_prefix, section_name) + + if section_name and items.get("description") and "default" in items: + return [ + { + "section": section_name, + "name": item_prefix, + "description": items["description"], + "default": _format_default(items["default"]), + "examples": _format_examples(param_name, items), + } + ] + + return [] + + +schema_file = CHART_ROOT_PATH / "values.schema.json" +with schema_file.open() as config_file: + chart_schema = json.load(config_file) + +params = _get_params(chart_schema["properties"]) + +# Now, split into sections +sections: dict[str, list[dict[str, str]]] = {} +for param in params: + if param["section"] not in sections: + sections[param["section"]] = [] + + sections[param["section"]].append(param) + +# and order each section +for section in sections.values(): # type: ignore + section.sort(key=lambda i: i["name"]) # type: ignore + +# and finally order the sections! +ordered_sections = [] +for name in chart_schema["x-docsSectionOrder"]: + if name not in sections: + raise ValueError(f"Unable to find any parameters for section: {name}") + ordered_sections.append({"name": name, "params": sections.pop(name)}) + +if sections: + raise ValueError(f"Found section(s) which were not in `section_order`: {list(sections.keys())}") + +jinja_contexts = { + "params_ctx": {"sections": ordered_sections}, + "official_download_page": { + "base_url": "https://downloads.apache.org/airflow/helm-chart", + "closer_lua_url": "https://www.apache.org/dyn/closer.lua/airflow/helm-chart", + "package_name": PACKAGE_NAME, + "package_version": PACKAGE_VERSION, + }, +} + + +# -- Options for sphinx.ext.autodoc -------------------------------------------- +# See: https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html + +# This value contains a list of modules to be mocked up. This is useful when some external dependencies +# are not met at build time and break the building process. +autodoc_mock_imports = get_autodoc_mock_imports() + +# The default options for autodoc directives. They are applied to all autodoc directives automatically. +autodoc_default_options = {"show-inheritance": True, "members": True} + +autodoc_typehints = "description" +autodoc_typehints_description_target = "documented" +autodoc_typehints_format = "short" + + +# -- Options for sphinx.ext.intersphinx ---------------------------------------- +# See: https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html + +# This config value contains names of other projects that should +# be linked to in this documentation. +# Inventories are only downloaded once by docs/exts/docs_build/fetch_inventories.py. +intersphinx_mapping = get_intersphinx_mapping() + +# -- Options for sphinx.ext.viewcode ------------------------------------------- +# See: https://www.sphinx-doc.org/es/master/usage/extensions/viewcode.html + +# If this is True, viewcode extension will emit viewcode-follow-imported event to resolve the name of +# the module by other extensions. The default is True. +viewcode_follow_imported_members = True + +# -- Options for sphinx-autoapi ------------------------------------------------ +# See: https://sphinx-autoapi.readthedocs.io/en/latest/config.html + +# Paths (relative or absolute) to the source code that you wish to generate +# your API documentation from. +autoapi_dirs = [CHART_ROOT_PATH.as_posix()] + +# A list of patterns to ignore when finding files +autoapi_ignore = BASIC_AUTOAPI_IGNORE_PATTERNS + +autoapi_log = logging.getLogger("sphinx.autoapi.mappers.base") +autoapi_log.addFilter(filter_autoapi_ignore_entries) + +# Keep the AutoAPI generated files on the filesystem after the run. +# Useful for debugging. +autoapi_keep_files = True + +# Relative path to output the AutoAPI files into. This can also be used to place the generated documentation +# anywhere in your documentation hierarchy. +autoapi_root = "_api" + +# Whether to insert the generated documentation into the TOC tree. If this is False, the default AutoAPI +# index page is not generated and you will need to include the generated documentation in a +# TOC tree entry yourself. +autoapi_add_toctree_entry = False + +# By default autoapi will include private members -- we don't want that! +autoapi_options = AUTOAPI_OPTIONS + +suppress_warnings = SUPPRESS_WARNINGS + +# -- Options for ext.exampleinclude -------------------------------------------- +exampleinclude_sourceroot = os.path.abspath("..") + +# -- Options for ext.redirects ------------------------------------------------- +redirects_file = "redirects.txt" + +# -- Options for sphinxcontrib-spelling ---------------------------------------- +spelling_word_list_filename = [SPELLING_WORDLIST_PATH.as_posix()] +spelling_exclude_patterns = ["changelog.rst"] +spelling_ignore_contributor_names = False +spelling_ignore_importable_modules = True + +graphviz_output_format = "svg" diff --git a/charts/airflow/docs/customizing-labels.rst b/charts/airflow/docs/customizing-labels.rst new file mode 100644 index 0000000..80b56ce --- /dev/null +++ b/charts/airflow/docs/customizing-labels.rst @@ -0,0 +1,125 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Customizing Labels and Annotations for Pods +=========================================== + +Customizing Pod Labels +---------------------- + +The Helm Chart allows you to customize labels for your Airflow objects. You can set global labels that apply to all objects and pods defined in the chart, as well as component-specific labels for individual Airflow components. + +Global Labels +~~~~~~~~~~~~~ + +Global labels can be set using the ``labels`` parameter in your values file. These labels will be applied to all Airflow objects and pods defined in the chart: + +.. code-block:: yaml + :caption: values.yaml + + labels: + environment: production + +Component-Specific Labels +~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can also set specific labels for individual Airflow components, which will be merged with the global labels. + +If the same label key exists in both global and component-specific labels, the component-specific value takes precedence (overrides the global value). + +This allows you to customize labels for specific components while still maintaining common global labels across all resources. +For example, to add specific labels to different components like scheduler or api-server: + +.. code-block:: yaml + :caption: values.yaml + + # Global labels applied to all pods + labels: + environment: production + + # Scheduler specific labels + scheduler: + labels: + role: scheduler + + # API Server specific labels + apiServer: + labels: + role: ui + +Customizing Pod Annotations +--------------------------- + +Pod annotations can be customized similarly to labels using ``podAnnotations`` and ``airflowPodAnnotations``. + +Global Pod Annotations +~~~~~~~~~~~~~~~~~~~~~~ + +Global pod annotations can be set using ``airflowPodAnnotations``. These are applied to all Airflow component pods (scheduler, api-server/webserver, triggerer, dag-processor and workers): + +.. code-block:: yaml + :caption: values.yaml + + airflowPodAnnotations: + example.com/team: data-platform + +Component-Specific Pod Annotations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Each component also supports its own ``podAnnotations``. Component-specific annotations take precedence over global ones: + +.. code-block:: yaml + :caption: values.yaml + + scheduler: + podAnnotations: + example.com/component: scheduler + +Templated Pod Annotations +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Both ``airflowPodAnnotations`` and ``podAnnotations`` support Helm template expressions. This allows annotations to reference release metadata or compute checksums of chart-managed resources, so that pods automatically restart when those resources change. + +For example, to restart scheduler pods whenever the chart's extra ConfigMaps change: + +.. code-block:: yaml + :caption: values.yaml + + extraConfigMaps: + my-listener-config: + data: | + listener.py: ... + + scheduler: + podAnnotations: + checksum/extra-configmaps: '{{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}' + +You can also reference release metadata: + +.. code-block:: yaml + :caption: values.yaml + + airflowPodAnnotations: + release: '{{ .Release.Name }}' + +.. note:: + + The ``include``/``sha256sum`` pattern only works for resources managed by this chart + (e.g., those created via ``extraConfigMaps`` or ``extraSecrets``). + For ConfigMaps or Secrets created outside the chart, consider using a tool like + `Stakater Reloader `__ to trigger pod restarts + automatically. diff --git a/charts/airflow/docs/customizing-workers.rst b/charts/airflow/docs/customizing-workers.rst new file mode 100644 index 0000000..2cab67b --- /dev/null +++ b/charts/airflow/docs/customizing-workers.rst @@ -0,0 +1,73 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Customizing Workers +=================== + +Both ``CeleryExecutor`` and ``KubernetesExecutor`` workers can be highly customized with the :ref:`workers parameters `. +For example, to set resources on workers: + +.. code-block:: yaml + :caption: values.yaml + + workers: + resources: + requests: + cpu: 1 + limits: + cpu: 1 + +One notable exception for ``KubernetesExecutor`` is that the default anti-affinity applied to ``CeleryExecutor`` workers to spread them across nodes +is not applied to ``KubernetesExecutor`` workers, as there is no reason to spread out per-task workers. + +Custom ``pod_template_file`` +---------------------------- + +With ``KubernetesExecutor`` or ``CeleryKubernetesExecutor`` you can also provide a complete ``pod_template_file`` +to fully override default Kubernetes workers configuration. This may be useful if you need different configuration between +worker types for ``CeleryKubernetesExecutor`` or if you need to customize something not possible with :ref:`workers parameters ` alone. + +.. note:: + + Some configuration options between Celery and Kubernetes workers can be overwritten by new ``workers.celery`` and ``workers.kubernetes`` sections. + Implementation of ``workers.celery`` and ``workers.kubernetes`` is not yet fully completed. + +As an example, let's say you want to set ``priorityClassName`` on your workers: + +.. note:: + + The following example is NOT functional, but meant to be illustrative of how you can provide a custom ``pod_template_file``. + You're better off starting with the default `pod_template_file`_ instead. + +.. _pod_template_file: https://github.com/apache/airflow/blob/main/chart/files/pod-template-file.kubernetes-helm-yaml + +.. code-block:: yaml + :caption: values.yaml + + podTemplate: | + apiVersion: v1 + kind: Pod + metadata: + name: placeholder-name + labels: + tier: airflow + component: worker + release: {{ .Release.Name }} + spec: + priorityClassName: high-priority + containers: + - name: base diff --git a/charts/airflow/docs/extending-the-chart.rst b/charts/airflow/docs/extending-the-chart.rst new file mode 100644 index 0000000..930743b --- /dev/null +++ b/charts/airflow/docs/extending-the-chart.rst @@ -0,0 +1,121 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Extending the Chart +=================== + +The Airflow Helm Chart can be easily extended by creating a custom chart which will depend on the Airflow chart. +That can be useful in cases where there is a need for custom templates deployment (e.g. maintenance CronJobs), +which are not directly related to the Airflow Helm Chart and should not be added to it in the source repository. +During installation of custom chart, the Airflow chart will also be installed too. + +You can extend the official Airflow chart by applying the following steps. + +Create your custom Helm Chart +----------------------------- + +First, you will need to create you own chart directory. You can do it by running the following command: + +.. code-block:: bash + + helm create my-custom-chart + + +This command will create a directory called ``my-custom-chart`` with the following structure: + +.. code-block:: none + + my-custom-chart/ + ├── .helmignore + ├── Chart.yaml + ├── values.yaml + ├── charts/ + └── templates/ + └── tests/ + +Add Airflow Helm Chart as dependency +------------------------------------ + +Second, you will need to add the Airflow chart as dependency to the custom chart. +This will give you the ability to add your custom templates without the need to modify the Airflow chart itself. +In order to add the Airflow chart as a dependency (often called ``subcharts``) to your chart, +add the following lines to your ``Chart.yaml`` file: + +.. code-block:: yaml + :caption: Chart.yaml + + dependencies: + - name: airflow + version: 1.11.0 + repository: https://airflow.apache.org + +.. note:: + + Make sure you have already added the Airflow repo locally by running: ``helm repo add apache-airflow https://airflow.apache.org``. + +.. tip:: + + You can also use the name of the repo instead of the URL by replacing + ``https://airflow.apache.org`` with ``"@apache-airflow"``. + +Adding the Airflow chart as a dependency means that it will be deployed together with your custom chart. +You can disable the installation of Airflow by adding the ``condition`` field to the ``dependencies`` section +like in the example below: + +.. code-block:: yaml + :caption: Chart.yaml + + dependencies: + - name: airflow + version: 1.11.0 + repository: https://airflow.apache.org + condition: airflow.enabled + +This will check if the value of ``airflow.enabled`` inside your ``values.yaml`` is ``true``. +If it is, the Airflow chart will be deployed together with your custom chart. +Otherwise, only your templates will be deployed. + +Download the Airflow Helm Chart +------------------------------- + +Third, after you have specified the Airflow chart inside the ``dependencies`` section in ``Chart.yaml`` file, +you can download it by running the following command: + +.. code-block:: bash + + helm dependency build + +.. note:: + + Make sure you are inside the directory which contains the ``Chart.yaml`` file. + +The chart will be downloaded and saved inside the ``charts/`` directory. + +Overriding default values +------------------------- + +When you add a chart as a subchart to your chart, +you have the ability to override the default values of the subchart in your ``values.yaml``. +This is useful when your chart needs a specific configuration for your custom chart. +E.g. if you want that the Airflow chart be installed with the ``KubernetesExecutor``, +you can do it by adding the following section to your ``values.yaml``: + +.. code-block:: yaml + :caption: values.yaml + + airflow: + executor: KubernetesExecutor diff --git a/charts/airflow/docs/img/helm-logo.svg b/charts/airflow/docs/img/helm-logo.svg new file mode 100644 index 0000000..1e2db8a --- /dev/null +++ b/charts/airflow/docs/img/helm-logo.svg @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/charts/airflow/docs/index.rst b/charts/airflow/docs/index.rst new file mode 100644 index 0000000..286183e --- /dev/null +++ b/charts/airflow/docs/index.rst @@ -0,0 +1,240 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. image:: /img/helm-logo.svg + :width: 100 + :class: no-scaled-link + +Helm Chart for Apache Airflow +============================= + +.. toctree:: + :hidden: + + Home + quick-start + airflow-configuration + adding-connections-and-variables + manage-dag-files + manage-logs + setting-resources-for-containers + keda + using-additional-containers + customizing-workers + customizing-labels + Installing from sources + Extending the Chart + +.. toctree:: + :hidden: + :caption: Guides + + production-guide + service-account-token-examples + +.. toctree:: + :hidden: + :caption: References + + Parameters + release_notes + + +This chart bootstraps an `Airflow `__ +deployment on a `Kubernetes `__ cluster using the +`Helm `__ package manager. + +Requirements +------------ + +- Kubernetes 1.30+ cluster +- Helm 3.10+ +- PV provisioner support in the underlying infrastructure (optionally) + +Features +-------- + +* Supported executors (all Airflow versions): ``LocalExecutor``, ``CeleryExecutor``, ``KubernetesExecutor`` +* Supported hybrid static executors (Airflow version ``2.11.X``): ``LocalKubernetesExecutor``, ``CeleryKubernetesExecutor`` +* Supported multiple Executors (``2.11+``) +* Supported AWS executors with AWS provider version ``8.21.0+``: + + * ``airflow.providers.amazon.aws.executors.batch.AwsBatchExecutor`` + * ``airflow.providers.amazon.aws.executors.ecs.AwsEcsExecutor`` + +* Supported AWS executors with AWS provider version ``9.9.0+``: + + * ``airflow.providers.amazon.aws.executors.aws_lambda.lambda_executor.AwsLambdaExecutor`` + +* Supported Edge executor with edge3 provider version ``1.0.0+``: + + * ``airflow.providers.edge3.executors.EdgeExecutor`` + +* Supported Airflow version: ``2.11+``, ``3.0+`` +* Supported database backend: ``PostgreSQL``, ``MySQL`` +* Autoscaling for ``CeleryExecutor`` provided by KEDA +* ``PostgreSQL`` and ``PgBouncer`` with a battle-tested configuration +* Monitoring: + + * StatsD/Prometheus metrics for Airflow + * Prometheus metrics for PgBouncer + * Flower + +* Automatic database migration after a new deployment +* Administrator account creation during deployment +* Kerberos secure configuration +* One-command deployment for any type of executor. You don't need to provide other services e.g. Redis/Database to test the Airflow + +.. _helm_chart_install: + +Installing the Helm Chart +------------------------- + +To install this chart using Helm 3, run the following commands: + +.. code-block:: bash + + helm repo add apache-airflow https://airflow.apache.org + helm upgrade --install airflow apache-airflow/airflow --namespace airflow --create-namespace + +The command deploys Airflow on the Kubernetes cluster with the default configuration in the airflow namespace. +The :doc:`parameters-ref` section lists the parameters that can be configured during installation. + + +.. tip:: + + List all releases using ``helm list``. + +Upgrading the deployed Helm Chart +--------------------------------- + +To upgrade the chart with the release name ``airflow``: + +.. code-block:: bash + + helm upgrade airflow apache-airflow/airflow --namespace airflow + +.. note:: + + To upgrade to a new version of the chart, run ``helm repo update`` first. + +.. tip:: + + By setting ``--install`` flag on the ``helm upgrade`` command, Helm Chart will be installed if it wasn't deployed before. + +Uninstalling the deployed Helm Chart +------------------------------------ + +To uninstall/delete the ``airflow`` deployment: + +.. code-block:: bash + + helm delete airflow --namespace airflow + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +.. note:: + + Some Kubernetes resources created by the chart `helm hooks `__ might be left in the namespace after executing ``helm uninstall``. + +Installing the Helm Chart with Argo CD, Flux, Rancher or Terraform +------------------------------------------------------------------ + +When installing the chart using Argo CD, Flux, Rancher or Terraform, you **must** set the four following values, or your application +will not start as the migrations will not be run: + +.. code-block:: yaml + :caption: values.yaml + + createUserJob: + useHelmHooks: false + applyCustomEnv: false + migrateDatabaseJob: + useHelmHooks: false + applyCustomEnv: false + +This is so these CI/CD services can perform updates without issues and preserve the immutability of Kubernetes Job manifests. + +.. note:: + + This applies to the chart installation with usage of ``--wait`` flag in ``helm install`` or ``helm upgrade`` command. + +.. note:: + + While deploying this Helm Chart with Argo, you might encounter issues with database migrations not running automatically on upgrade. + +To run database migrations with Argo CD automatically, you will need to add: + +.. code-block:: yaml + :caption: values.yaml + + migrateDatabaseJob: + jobAnnotations: + "argocd.argoproj.io/hook": Sync + +This will run database migrations every time there is a ``Sync`` event in Argo CD. While it is not ideal to run the migrations on every sync, it is a trade-off that allows them to be run automatically. + +If you use the ``CeleryExecutor`` or ``CeleryKubernetesExecutor`` with the built-in Redis, it is recommended that you set up a static Redis password either by supplying ``redis.passwordSecretName`` and ``data.brokerUrlSecretName`` or ``redis.password``. + +.. note:: + + By default, Helm hooks are also enabled for ``extraSecrets`` or ``extraConfigMaps``. When using the above CI/CD tools, you might encounter issues due to these default hooks. + +To avoid potential problems, it is recommended to disable these hooks by setting ``useHelmHooks=false`` as shown in the following examples: + +.. code-block:: yaml + :caption: values.yaml + + extraSecrets: + '{{ .Release.Name }}-example': + useHelmHooks: false + data: | + AIRFLOW_VAR_HELLO_MESSAGE: "Hi!" + + extraConfigMaps: + '{{ .Release.Name }}-example': + useHelmHooks: false + data: | + AIRFLOW_VAR_HELLO_MESSAGE: "Hi!" + +Naming Conventions +------------------ + +For new installations it is highly recommended to start using standard naming conventions. +It is not enabled by default as this may cause unexpected behaviours on existing installations. However you can enable it using ``useStandardNaming``: + +.. code-block:: yaml + :caption: values.yaml + + useStandardNaming: true + +For existing installations, all your resources will be recreated with a new name and helm will delete previous resources. + +This won't delete existing PVCs for logs used by StatefulSets/Deployments, but it will recreate them with brand new PVCs. +If you do want to preserve logs history you'll need to manually copy the data of these volumes into the new volumes after +deployment. Depending on what storage backend/class you're using, this procedure may vary. If you don't mind starting +with fresh logs/redis volumes, you can delete the old persistent volume claims, for example: + +.. code-block:: bash + + kubectl delete pvc -n airflow logs-gta-triggerer-0 + kubectl delete pvc -n airflow logs-gta-worker-0 + kubectl delete pvc -n airflow redis-db-gta-redis-0 + +.. note:: + + If you do not change ``useStandardNaming`` or ``fullnameOverride`` after upgrade, you can proceed as usual and no unexpected behaviours will be presented. diff --git a/charts/airflow/docs/installing-helm-chart-from-sources.rst b/charts/airflow/docs/installing-helm-chart-from-sources.rst new file mode 100644 index 0000000..98270b5 --- /dev/null +++ b/charts/airflow/docs/installing-helm-chart-from-sources.rst @@ -0,0 +1,128 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Installing Helm Chart from sources +================================== + +Released packages +----------------- + +.. jinja:: official_download_page + + This page describes downloading and verifying ``Apache Airflow Official Helm Chart`` version + ``{{ package_version }}`` using officially released source packages. You can also install the chart + directly from the ``airflow.apache.org`` repo as described in :ref:`helm_chart_install`. + You can choose different version of the chart by selecting different version from the drop-down at + the top-left of the page. + + +The sources and packages released are the "official" sources of installation that you can use if +you want to verify the origin of the packages and want to verify checksums and signatures of the packages. +The packages are available via the +`Official Apache Software Foundations Downloads `_ + +The downloads are available at: + +.. jinja:: official_download_page + + * `Sources package <{{ closer_lua_url }}/{{ package_version }}/airflow-chart-{{ package_version }}-source.tar.gz>`__ (`asc <{{ base_url }}/{{ package_version }}/airflow-chart-{{ package_version }}-source.tar.gz.asc>`__, `sha512 <{{ base_url }}/{{ package_version }}/airflow-chart-{{ package_version }}-source.tar.gz.sha512>`__) + * `Installable package <{{ closer_lua_url }}/{{ package_version }}/airflow-{{ package_version }}.tgz>`__ (`asc <{{ base_url }}/{{ package_version }}/airflow-{{ package_version }}.tgz.asc>`__, `sha512 <{{ base_url }}/{{ package_version }}/airflow-{{ package_version }}.tgz.sha512>`__) + +If you want to install from the source code, you can download from the sources link above, which will contain +an ``INSTALL`` file containing details on how you can build and install the chart. + +Release integrity +----------------- + +`PGP signatures KEYS `_ + +It is essential that you verify the integrity of the downloaded files using the PGP or SHA signatures. +The PGP signatures can be verified using GPG or PGP. Please download the KEYS as well as the asc +signature files for relevant distribution. It is recommended to get these files from the +main distribution directory and not from the mirrors. + +.. code-block:: bash + + gpg -i KEYS + +or + +.. code-block:: bash + + pgpk -a KEYS + +or + +.. code-block:: bash + + pgp -ka KEYS + +To verify the binaries/sources you can download the relevant asc files for it from main +distribution directory and follow the below guide. + +.. code-block:: bash + + gpg --verify airflow-********.asc airflow-********* + +or + +.. code-block:: bash + + pgpv airflow-********.asc + +or + +.. code-block:: bash + + pgp airflow-********.asc + +Example: + +.. jinja:: official_download_page + + .. code-block:: console + :substitutions: + + $ gpg --verify airflow-{{ package_version }}.tgz.asc airflow-{{ package_version }}.tgz + gpg: Signature made Sat 11 Sep 12:49:54 2021 BST + gpg: using RSA key CDE15C6E4D3A8EC4ECF4BA4B6674E08AD7DE406F + gpg: issuer "kaxilnaik@apache.org" + gpg: Good signature from "Kaxil Naik " [unknown] + gpg: aka "Kaxil Naik " [unknown] + gpg: WARNING: The key's User ID is not certified with a trusted signature! + gpg: There is no indication that the signature belongs to the owner. + Primary key fingerprint: CDE1 5C6E 4D3A 8EC4 ECF4 BA4B 6674 E08A D7DE 406F + + The "Good signature from ..." is indication that the signatures are correct. + Do not worry about the "not certified with a trusted signature" warning. Most of the certificates used + by release managers are self signed, that's why you get this warning. By importing the server in the + previous step and importing it via ID from ``KEYS`` page, you know that this is a valid Key already. + + For SHA512 sum check, download the relevant ``sha512`` and run the following: + + .. code-block:: bash + + shasum -a 512 airflow-******** | diff - airflow-********.sha512 + + The ``SHASUM`` of the file should match the one provided in ``.sha512`` file. + + Example: + + .. code-block:: bash + :substitutions: + + shasum -a 512 airflow-{{ package_version }}.tgz | diff - airflow-{{ package_version }}.tgz.sha512 diff --git a/charts/airflow/docs/keda.rst b/charts/airflow/docs/keda.rst new file mode 100644 index 0000000..83ce254 --- /dev/null +++ b/charts/airflow/docs/keda.rst @@ -0,0 +1,123 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Autoscaling with KEDA +===================== + +KEDA stands for Kubernetes Event Driven Autoscaling. +`KEDA `__ is a custom controller that +allows users to create custom bindings to the Kubernetes `Horizontal Pod +Autoscaler `__. +The autoscaler will adjust the number of active Celery workers based on the number +of tasks in ``queued`` or ``running`` state. + +One advantage of KEDA is that it allows you to scale your application to/from 0 workers, meaning no workers are idle when there are no tasks. + +KEDA Installation and usage +--------------------------- + +To install KEDA in your Kubernetes cluster, run the following commands: + +.. code-block:: bash + + helm repo add kedacore https://kedacore.github.io/charts + helm repo update + kubectl create namespace keda + helm install keda kedacore/keda \ + --namespace keda \ + --version "v2.0.0" + +To enable KEDA for the Airflow instance, it has to be enabled by setting ``workers.celery.keda.enabled=true`` +in your Helm command or in the ``values.yaml`` like: + +.. code-block:: bash + + kubectl create namespace airflow + helm repo add apache-airflow https://airflow.apache.org + helm install airflow apache-airflow/airflow \ + --namespace airflow \ + --set executor=CeleryExecutor \ + --set workers.celery.keda.enabled=true + +.. note:: + + Make sure ``values.yaml`` shows that either KEDA or HPA is enabled, but not both. It is recommended not + to use both KEDA and HPA to scale the same workload. They will compete with each other resulting in odd scaling behavior. + +After installation, the KEDA ``ScaledObject`` and an ``HPA`` will be created in the Airflow namespace. + +In the default configuration, KEDA will derive the desired number of Celery workers by querying Airflow metadata database with following SQL statement: + +.. code-block:: none + + SELECT + ceil(COUNT(*)::decimal / {{ .Values.config.celery.worker_concurrency }}) + FROM + task_instance + WHERE + (state='running' OR state='queued') + AND queue IN + +where ```` is a list of queue names used by +`Celery worker queues `_ +mechanism (with default configuration it has one element ``default``). + +.. note:: + + Set Celery worker concurrency through the Helm Chart value + ``config.celery.worker_concurrency`` (e.g. instead of airflow.cfg or + environment variables), so that the KEDA trigger will be consistent with + the worker concurrency setting. + +Triggers (aka Scalers) +---------------------- + +Triggers refer to the metrics (or formulae) that KEDA should refer to when scaling workers. + +It is recommended to use multiple triggers within a ScaledObject, rather than creating different objects for different triggers. +This keeps all your rules and formulae in one place, and it avoids multiple ScaledObjects being created by the same workload. + +ScaledObject +------------ + +To configure KEDA's triggers and scaling behaviors, you need to create a ScaledObject. Below ScaledObject parameters: + +* ``cooldownPeriod`` specifies the number of seconds to wait before downscaling to 0 workers, does not apply to downscaling to n workers while n >= 1. +* ``idleReplicaCount`` can be set to any number less than ``minReplicaCount``, but it must be set to 0, otherwise KEDA will not work. Change ``minReplicaCount`` to n > 0 if you need idle workers. + +Triggerers value ``targetQueryValue`` is used as ``TargetValue`` of workers, which must be between ScaledObject ``minReplicaCount`` and ``maxReplicaCount`` values. + +.. note:: + + To avoid strange behavior, best practice is to set ``cooldownPeriod`` to an integer slightly larger than ``terminationGracePeriodSeconds`` so that your cluster does not downscale to 0 workers before cleanup is finished. + +Metrics +------- + +The HPA controller, refreshes metrics defined in triggers every ``--horizontal-pod-autoscaler-sync-period`` and the values are routed to +KEDA Metrics Server directly. To reduce the load on the KEDA Scaler, you can set ``useCachedMetrics`` to true, to enabling reading metrics +from cache first. Cache is updated periodically every ``pollingInterval``. + +.. note:: + + When number of workers = 0, KEDA will still poll for metrics using ``pollingInterval``. + When number of workers >= 1, both KEDA and the HPA will poll your defined triggers. + +KEDA offers two ``metricTypes`` that provide more granular scaling control than the standard HPA ``Target`` metric: + +* AverageValue (default) controls a per-worker average. +* Value controls total system load. diff --git a/charts/airflow/docs/manage-dag-files.rst b/charts/airflow/docs/manage-dag-files.rst new file mode 100644 index 0000000..3ab6529 --- /dev/null +++ b/charts/airflow/docs/manage-dag-files.rst @@ -0,0 +1,261 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +Manage Dag files +================ + +When you create new or modify existing Dag files, it is necessary to deploy them into the environment. This section will describe some basic techniques which you can use. + +Bake Dags in docker image +------------------------- + +With this approach, you include your Dag files and related code in the Airflow image. + +This method requires redeploying the services in the helm chart with the new docker image in order to deploy the new Dag code. This can work well particularly if Dag code is not expected to change frequently. + +.. code-block:: bash + + docker build --pull --tag "my-company/airflow:8a0da78" . -f - <`_ for details), and specify it using ``--set registry.secretName`` like: + +.. code-block:: bash + + helm upgrade --install airflow apache-airflow/airflow \ + --set images.airflow.repository=my-company/airflow \ + --set images.airflow.tag=8a0da78 \ + --set images.airflow.pullPolicy=Always \ + --set registry.secretName=gitlab-registry-credentials + +Using Git-Sync +-------------- + +Mounting Dags using Git-Sync sidecar with persistence enabled +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This option will use a Persistent Volume Claim with ``ReadWriteMany`` access mode. +The dag-processor pod (if standalone dag-processor is disabled it will be scheduler pod) will sync Dags from +a git repository onto the PVC every configured number of seconds. The other pods will read the synced Dags. +Not all volume plugins have support for ``ReadWriteMany`` access mode. +Refer `Persistent Volume Access Modes `__ +for details. + +.. code-block:: bash + + helm upgrade --install airflow apache-airflow/airflow \ + --set dags.persistence.enabled=true \ + --set dags.gitSync.enabled=true + # You can also override the other persistence or gitSync values + # by setting the dags.persistence.* and dags.gitSync.* values + # Please refer to values.yaml for details + +Mounting Dags using Git-Sync sidecar without persistence +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This option will always use running Git-Sync sidecar on every dag-processor, worker and triggerer pods +(In Airflow 2.11, if separate dag-processor is not enabled, the Git-Sync sidecar will run on scheduler for Dag parsing as well). + +The Git-Sync sidecar containers will sync Dags from a git repository every configured number of +seconds. If you are using the ``KubernetesExecutor``, Git-Sync will run as an init container on your worker pods. + +.. code-block:: bash + + helm upgrade --install airflow apache-airflow/airflow \ + --set dags.persistence.enabled=false \ + --set dags.gitSync.enabled=true + # You can also override the other gitSync values + # by setting the dags.gitSync.* values + # Refer values.yaml for details + +Notes for combining Git-Sync and persistence +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +While using git-sync and persistence for Dags is possible, it is generally not recommended unless the +Deployment Manager carefully considered the trade-offs it brings. There are cases when git-sync without +persistence has other trade-offs (for example delays in synchronization of Dags vs. rate-limiting of Git +servers) that can often be mitigated (for example by sending signals to git-sync containers via web-hooks +when new commits are pushed to the repository), but there might be cases where you still might want to choose +git-sync and persistence together. + +Git-sync solution is primarily designed to be used for local, POSIX-compliant volumes to checkout Git +repositories. Part of the process of commits synchronization from git-sync involves checking out +new version of files in a freshly created folder and swapping symbolic links to the new folder, after the +checkout is complete. This is done to ensure that the whole Dags folder is consistent at all times. The way +git-sync works with symbolic-link swaps, makes sure that Parsing the Dags always work on a consistent +(single-commit-based) set of files in the whole Dag folder. + +This approach, however might have undesirable side effects when the folder that git-sync works on is not +a local volume, but is a persistent volume (so effectively a networked, distributed volume). Depending on +the technology behind the persistent volumes might handle git-sync approach differently and with non-obvious +consequences. There are a lot of persistence solutions available for various K8S installations and each of +them has different characteristics, so you need to carefully test and monitor your filesystem to make sure +those undesired side effects do not affect you. Those effects might change over time or depend on parameters +like how often the files are being scanned by the Dag Processor, the number and complexity of your +Dags, how remote and how distributed your persistent volumes are, how many IOPS you allocate for some of +the filesystem (usually highly paid feature of such filesystems is how many IOPS you can get) and many other +factors. + +The way git-sync works with symbolic links swapping generally causes a linear growth of the throughput and +potential delays in synchronization. The networking traffic from checkouts comes in bursts and the bursts +are linearly proportional to the number and size of files you have in the repository, makes it vulnerable +to pretty sudden and unexpected demand increase. Most of the persistence solution work "good enough" for +smaller/shorter burst of traffic, but when they outgrow certain thresholds, you need to upgrade the +networking to a much more capable and expensive options. This is difficult to control and impossible to +mitigate, so you might be suddenly faced with situation to pay a lot more for IOPS/persistence option to +keep your Dags sufficiently synchronized to avoid inconsistencies and delays in synchronization. + +The side-effects that you might observe are: + +* bursts of networking/communication at the moment when new commit is checked out, because of the quick + succession of deleting old files, creating new files, symbolic link swapping. +* temporary lack of consistency between files in Dag folders while Dags are being synced, because of delays + in distributing changes to individual files for various nodes in the cluster. +* visible drops of performance of the persistence solution when your Dag number grows, drops that might + amplify the side effects described above. +* some of persistence solutions might lack filesystem functionality that git-sync needs to perform the sync + (for example changing permissions or creating symbolic links). While those can often be mitigated, it is + only recommended to use git-sync with fully POSIX-filesystem compliant persistence filesystems. + +General recommendation is to use git-sync with local volumes only, and if you want to it with persistence, you +need to make sure that the persistence solution you use is POSIX-compliant and you monitor the side-effects +it might have. + +Synchronizing multiple Git repositories with Git-Sync +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Airflow git-sync integration in the Helm Chart does not allow synchronization of multiple repositories at +the same time. The Dag folder must come from single git repository. However, it is possible +to use `submodules `_ to create an "umbrella" repository +that you can use to bring a number of git repositories checked out together (with ``--submodules recursive`` +option). There are success stories of Airflow users using such approach with 100s of repositories put +together as submodules via such "umbrella" repo approach. When you choose this solution, you need to work out +the way how to link the submodules, when to update the umbrella repo when "submodule" repository change and +work out versioning approach and automate it. This might be as simple as always using latest versions of all +the submodule repositories, or as complex as managing versioning of shared libraries, Dags and code across +multiple teams and doing that following your release process. + +An example of such complex approach can found in this +`Manage Dags at scale `_ presentation from the Airflow +Summit. + +Mounting Dags from an externally populated PVC +---------------------------------------------- + +In this approach, Airflow will read the Dags from a PVC which has ``ReadOnlyMany`` or ``ReadWriteMany`` access mode. +You will have to ensure that the PVC is populated/updated with the required Dags (this won't be handled by the chart). +You can pass the name of the volume claim to the chart by using ``dags.persistence.existingClaim`` parameter: + +.. code-block:: bash + + helm upgrade --install airflow apache-airflow/airflow \ + --set dags.persistence.enabled=true \ + --set dags.persistence.existingClaim=my-volume-claim \ + --set dags.gitSync.enabled=false + +Mounting Dags from a private GitHub repo using Git-Sync sidecar +--------------------------------------------------------------- + +To configure mounting Dags from private GitHub repository, follow below steps: + +1. Create a private repo on GitHub if you have not created one already. +2. Then create your ssh keys: + + .. code-block:: bash + + ssh-keygen -t rsa -b 4096 -C "your_email@example.com" + +3. Add the public key to your private repo under ``Settings > Deploy keys``. +4. Convert the private ssh key to a base64 string and save it's value. + + .. note:: + + You can convert the private ssh key file like: + + .. code-block:: bash + + base64 -w 0 > temp.txt + + Then copy the string from the ``temp.txt`` file. + + The converted to base64 string will be used in the ``override-values.yaml`` file. + +5. Create a yaml file called ``override-values.yaml`` to override default values, instead of using ``--set``: + + .. code-block:: yaml + :caption: override-values.yaml + + dags: + gitSync: + enabled: true + repo: git@github.com:/.git + branch: + subPath: "" + sshKeySecret: airflow-ssh-secret + extraSecrets: + airflow-ssh-secret: + data: | + gitSshKey: '' + + Copied base64 string should be as a value for the ``gitSshKey`` key. + +6. Finally, from the context of your Airflow Helm chart directory, install Airflow: + + .. code-block:: bash + + helm upgrade --install airflow apache-airflow/airflow -f override-values.yaml + +If you have done everything correctly, Git-Sync will pick up the changes you make to the Dags +in your private GitHub repo. + +You should take this a step further and set ``dags.gitSync.knownHosts``, so you are not susceptible to man-in-the-middle +attacks. This process is documented in the :ref:`production guide `. diff --git a/charts/airflow/docs/manage-logs.rst b/charts/airflow/docs/manage-logs.rst new file mode 100644 index 0000000..09b38c2 --- /dev/null +++ b/charts/airflow/docs/manage-logs.rst @@ -0,0 +1,100 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Manage logs +=========== + +You have a number of options when it comes to managing your Airflow logs. + +No persistence +-------------- + +With this option, Airflow will log locally to each pod. As such, the logs will only be available during the lifetime of the pod. + +.. code-block:: bash + + helm upgrade --install airflow apache-airflow/airflow \ + --set logs.persistence.enabled=false + +.. note:: + + Setting the ``workers.celery.persistence.enabled=false`` is required when ``CeleryExecutor`` is used. + +Celery worker log persistence +----------------------------- + +If you are using ``CeleryExecutor``, workers persist logs by default to a volume claim created with a ``volumeClaimTemplate``. + +You can modify the template: + +.. code-block:: bash + + helm upgrade --install airflow apache-airflow/airflow \ + --set executor=CeleryExecutor \ + --set workers.celery.persistence.size=10Gi + +.. note:: + + With this option only task logs are persisted, unlike when log persistence is enabled which will also persist scheduler logs. + +Log persistence enabled +----------------------- + +This option will provision a ``PersistentVolumeClaim`` with an access mode of ``ReadWriteMany``. Each component of Airflow will +then log onto the same volume. + +Not all volume plugins have support for ``ReadWriteMany`` access mode. +Refer `Persistent Volume Access Modes `__ +for details. + +.. code-block:: bash + + helm upgrade --install airflow apache-airflow/airflow \ + --set logs.persistence.enabled=true + # You can also override the other persistence + # by setting the logs.persistence.* values + # Please refer to values.yaml for details + +Externally provisioned PVC +-------------------------- + +In this approach, Airflow will log to an existing ``ReadWriteMany`` PVC. You pass in the name of the volume claim to the chart +by using the ``logs.persistence.existingClaim`` parameter: + +.. code-block:: bash + + helm upgrade --install airflow apache-airflow/airflow \ + --set logs.persistence.enabled=true \ + --set logs.persistence.existingClaim=my-volume-claim + +.. note:: + + The volume has to be writable by the Airflow user. The easiest way to do this is to ensure GID ``0`` has a write permission. + More information can be found in the :ref:`Docker image entrypoint documentation `. + +Elasticsearch +------------- + +If your cluster forwards logs to Elasticsearch, you can configure Airflow to retrieve task logs from it. +See the :doc:`Elasticsearch providers guide ` for more details. + +.. code-block:: bash + + helm upgrade --install airflow apache-airflow/airflow \ + --set elasticsearch.enabled=true \ + --set elasticsearch.secretName=my-es-secret + # Other choices exist. Please refer to values.yaml for details. diff --git a/charts/airflow/docs/parameters-ref.rst b/charts/airflow/docs/parameters-ref.rst new file mode 100644 index 0000000..521bcd2 --- /dev/null +++ b/charts/airflow/docs/parameters-ref.rst @@ -0,0 +1,77 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Parameters reference +==================== + +The following tables lists the configurable parameters of the Airflow chart and their default values: + +.. jinja:: params_ctx + + {% for section in sections %} + + .. _parameters:{{ section["name"] }}: + + {{ section["name"] }} + {{ "=" * (section["name"]|length + 2) }} + + .. list-table:: + :widths: 15 10 30 + :header-rows: 1 + + * - Parameter + - Description + - Default + + {% for param in section["params"] %} + * - ``{{ param["name"] }}`` + - {{ param["description"] }} + - ``{{ param["default"] }}`` + {% if param["examples"] %} + + .. code-block:: yaml + :caption: Examples + + {{ param["examples"] | indent(width=10) }} + + {% endif %} + {% endfor %} + + {% endfor %} + + +Specify each parameter using the ``--set key=value[,key=value]`` argument to ``helm install``: + +.. code-block:: bash + + helm install my-release apache-airflow/airflow \ + --set executor=CeleryExecutor \ + --set enablePodLaunching=false + +or override values by using the ``override-values.yaml`` file: + +.. code-block:: yaml + :caption: override-values.yaml + + executor: CeleryExecutor + enablePodLaunching: false + +and install the chart: + +.. code-block:: bash + + helm install my-release apache-airflow/airflow --values override-values.yaml diff --git a/charts/airflow/docs/production-guide.rst b/charts/airflow/docs/production-guide.rst new file mode 100644 index 0000000..ee8f7ef --- /dev/null +++ b/charts/airflow/docs/production-guide.rst @@ -0,0 +1,892 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Production Guide +================ + +The following are things to consider when using this Helm chart in a production environment. + +Database +-------- + +It is advised to set up an external database for the Airflow metastore. The default Helm chart deploys a +Postgres database running in a container. For **production** usage, a database running on a dedicated machine or +leveraging a cloud provider's database service such as AWS RDS, should be used. Embedded Postgres +lacks stability, monitoring and persistence features that you need for a production database. It is only there to +make it easier to test the Helm Chart in a "standalone" version, but you might experience data loss when you +are using it. Supported databases and versions can be found at :doc:`Set up a Database Backend `. + + +.. note:: + + When using the helm chart, you do not need to initialize the db with ``airflow db migrate`` + as outlined in :doc:`Set up a Database Backend `. + +To disable deployment of Postgres pod, set below values in your ``values.yaml`` file: + +.. code-block:: yaml + :caption: values.yaml + + postgresql: + enabled: false + +To provide the database credentials to Airflow, you have 2 options - in your values file or in a Kubernetes Secret. + +Values file +^^^^^^^^^^^ + +This is the simpler options, as the chart will create a Kubernetes Secret for you. However, keep in mind your credentials will be in your values file. + +.. code-block:: yaml + :caption: values.yaml + + data: + metadataConnection: + user: + pass: + protocol: postgresql + host: + port: 5432 + db: + +.. warning:: + + Due to security concerns, it is not advised to store Airflow database user credentials directly in the ``values.yaml`` file. + +Kubernetes Secret +^^^^^^^^^^^^^^^^^ + +You can store the credentials in a Kubernetes Secret (it requires manual creation). + +.. note:: + + Any special character in the username/password must be URL encoded. + +.. code-block:: bash + + kubectl create secret generic mydatabase --from-literal=connection=postgresql://user:pass@host:5432/db + +After secret creation, configure the chart to use the secret: + +.. code-block:: yaml + :caption: values.yaml + + data: + metadataSecretName: mydatabase + +.. _production-guide:pgbouncer: + +Metadata DB Cleanup +^^^^^^^^^^^^^^^^^^^ + +It is recommended to periodically clean up the Airflow metadata database to remove old records and keep the database size manageable. +A Kubernetes CronJob can be enabled for this purpose: + +.. code-block:: yaml + :caption: values.yaml + + databaseCleanup: + enabled: true + retentionDays: 90 + +For details regarding the ``airflow db clean`` command, see :ref:`db clean usage ` and for additional options which +can be configured via helm chart values, see :doc:`parameters reference `. + +PgBouncer +--------- + +If you are using PostgreSQL as your database, you will likely want to enable `PgBouncer `_ as well. +Due to distributed nature of Airflow, it can open a lot of database connections. Using a connection pooler can significantly +reduce the number of open connections on the database. + +Database credentials stored Values file +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: yaml + :caption: values.yaml + + pgbouncer: + enabled: true + + +Database credentials stored Kubernetes Secret +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The default connection string in this case will not work. You need to modify accordingly the Kubernetes secret: + +.. code-block:: bash + + kubectl create secret generic mydatabase --from-literal=connection=postgresql://user:pass@pgbouncer_svc_name.deployment_namespace:6543/airflow-metadata + +Furthermore, two additional Kubernetes Secret are required for PgBouncer to be able to properly work: + +1. ``airflow-pgbouncer-stats`` secret: + + .. code-block:: bash + + kubectl create secret generic airflow-pgbouncer-stats --from-literal=connection=postgresql://user:pass@127.0.0.1:6543/pgbouncer?sslmode=disable + +2. ``airflow-pgbouncer-config`` secret: + + .. code-block:: yaml + :caption: airflow-pgbouncer-config + + apiVersion: v1 + kind: Secret + metadata: + name: airflow-pgbouncer-config + data: + pgbouncer.ini: dmFsdWUtMg0KDQo= + users.txt: dmFsdWUtMg0KDQo= + + where: + + 1. ``pgbouncer.ini`` value is equal to the base64 encoded version of below text: + + .. code-block:: text + :caption: pgbouncer.ini + + [databases] + airflow-metadata = host={external_database_host} dbname={external_database_dbname} port=5432 pool_size=10 + + [pgbouncer] + pool_mode = transaction + listen_port = 6543 + listen_addr = * + auth_type = scram-sha-256 + auth_file = /etc/pgbouncer/users.txt + stats_users = postgres + ignore_startup_parameters = extra_float_digits + max_client_conn = 100 + verbose = 0 + log_disconnections = 0 + log_connections = 0 + + server_tls_sslmode = prefer + server_tls_ciphers = normal + + 2. ``users.txt`` value is equal to the base64 encoded version of below text: + + .. code-block:: text + :caption: users.txt + + "{ external_database_username }" "{ external_database_pass }" + +In the ``values.yaml`` below secret-related parameters should be adjusted like: + +.. code-block:: yaml + :caption: values.yaml + + pgbouncer: + enabled: true + configSecretName: airflow-pgbouncer-config + metricsExporterSidecar: + statsSecretName: airflow-pgbouncer-stats + +.. note:: + + Depending on the size of your Airflow instance, you may want to adjust the following as well (defaults are shown): + + .. code-block:: yaml + :caption: values.yaml + + pgbouncer: + # The maximum number of connections to PgBouncer + maxClientConn: 100 + # The maximum number of server connections to the metadata database from PgBouncer + metadataPoolSize: 10 + # The maximum number of server connections to the result backend database from PgBouncer + resultBackendPoolSize: 5 + +API Secret Key +-------------- + +You should set a static API secret key when deploying with Airflow chart as it will help ensure +your Airflow components only restart when necessary. + +.. note:: + + This section also applies to the webserver for Airflow 2 (simply replace ``api`` with ``webserver``). + +.. warning:: + + You should use a different secret key for every instance you run, as this key is used to sign + session cookies and perform other security related functions. + +Follow below steps to create static API secret key: + +1. Generate a strong secret key: + + .. code-block:: bash + + python3 -c 'import secrets; print(secrets.token_hex(16))' + +2. Add the secret to your values file: + + .. code-block:: yaml + :caption: values.yaml + + apiSecretKey: + + or create a Kubernetes Secret and use ``apiSecretKeySecretName``: + + .. code-block:: yaml + :caption: values.yaml + + apiSecretKeySecretName: my-api-secret + # Where the random key is under `webserver-secret-key` in the k8s Secret + + .. warning:: + + Due to security concerns, it is advised to use Kubernetes Secret instead of setting API secret key directly in the values file. + +Example to create a Kubernetes Secret from ``kubectl``: + +.. code-block:: bash + + kubectl create secret generic my-api-secret --from-literal="api-secret-key=$(python3 -c 'import secrets; print(secrets.token_hex(16))')" + +The API secret key is also used to authorize requests to Celery workers when logs are retrieved. The token +generated using the secret key has a short expiry time though. Make sure that time on ALL the machines +that you run Airflow components on is synchronized (for example using ntpd). You might get +"forbidden" errors when the logs are accessed otherwise. + +JWT Secret +---------- + +You should set a static JWT Secret key when deploying with Airflow chart as it will increase environment +stability. It can be achieved by using ``jwtSecretName`` field in the ``values.yaml`` file. + +.. note:: + + For increase security of production setup, consider creating custom JWT Secret rollover procedure which will + not cause failures in dag runs due to mismatch in tokens. + +Eviction configuration +---------------------- +When running Airflow along with the `Kubernetes Cluster Autoscaler `_, it is important to configure whether pods can be safely evicted. +This setting can be configured in the Airflow chart at different levels: + +.. code-block:: yaml + :caption: values.yaml + + workers: + safeToEvict: true + scheduler: + safeToEvict: true + apiServer: + safeToEvict: true + +``workers.safeToEvict`` defaults to ``false``, and when using ``KubernetesExecutor`` +``workers.safeToEvict`` shouldn't be set to ``true`` as the workers may be removed before finishing. + +Extending and customizing Airflow Image +--------------------------------------- + +The Apache Airflow community, releases Docker Images which are ``reference images`` for Apache Airflow. +However, Airflow has more than 60 community managed providers (installable via extras) and some of the +default extras/providers installed are not used by everyone. Sometimes other extras/providers +are needed, sometimes (very often actually) you need to add your own custom dependencies, +packages or even custom providers, or add custom tools and binaries that are needed in +your deployment. + +In Kubernetes and Docker terms, this means that you need another image with your specific requirements. +This is why you should learn how to build your own ``Docker`` (or more properly ``Container``) image. + +Typical scenarios where you would like to use your custom image are adding: + +* ``apt`` packages, +* ``PyPI`` packages, +* binary resources necessary for your deployment, +* custom tools needed in your deployment. + +See :ref:`Extending Airflow Image ` and/or +`Building the image `_ for more +details on how you can extend, customize and test the modifications of Airflow image. + +Managing Dag Files +------------------ + +See :doc:`manage-dag-files`. + +.. _production-guide:knownhosts: + +knownHosts +^^^^^^^^^^ + +If you are using ``dags.gitSync.sshKeySecret``, you should also set ``dags.gitSync.knownHosts``. Here we will show the process +for GitHub, but the same can be done for any provider: + +1. Grab GitHub's public key: + + .. code-block:: bash + + ssh-keyscan -t rsa github.com > github_public_key + +2. Print the fingerprint for the public key: + + .. code-block:: bash + + ssh-keygen -lf github_public_key + +3. Compare that output with `GitHub's SSH key fingerprints `_. +4. If values are the same, add the public key to your values. It'll look something like this: + + .. code-block:: yaml + :caption: values.yaml + + dags: + gitSync: + knownHosts: | + github.com ssh-rsa AAAA...1/wsjk= + +External Scheduler +------------------ + +To use an external Scheduler instance: + +.. code-block:: yaml + :caption: values.yaml + + scheduler: + enabled: false + +Ensure that your external scheduler is connected to the same redis host as workers. + +Accessing the Airflow UI +------------------------ + +How you access the Airflow UI will depend on your environment; however, the chart does support various options. + +External API Server +^^^^^^^^^^^^^^^^^^^ + +To use an external API Server: + +.. code-block:: yaml + :caption: values.yaml + + apiServer: + enabled: false + +Ingress +^^^^^^^ + +You can create and configure ``Ingress`` objects. See the :ref:`Ingress chart parameters `. +For more information on ``Ingress``, see the +`Kubernetes Ingress documentation `_. + +LoadBalancer Service +^^^^^^^^^^^^^^^^^^^^ + +You can change the Service type for the API Server to be ``LoadBalancer``, and set any necessary annotations: + +.. code-block:: yaml + :caption: values.yaml + + apiServer: + service: + type: LoadBalancer + +For more information on ``LoadBalancer`` Services, see the `Kubernetes LoadBalancer Service Documentation +`_. + +Logging +------- + +Depending on your choice of executor, task logs may not work out of the box. All logging choices can be found +at :doc:`manage-logs`. + +Metrics +------- + +The chart supports sending metrics to an existing StatsD instance or provide a Prometheus endpoint. + +Prometheus Endpoint +^^^^^^^^^^^^^^^^^^^ + +The metrics endpoint is available at ``svc/{{ .Release.Name }}-statsd:9102/metrics``. + +External StatsD +^^^^^^^^^^^^^^^ + +To use an external StatsD instance: + +.. code-block:: yaml + :caption: values.yaml + + statsd: + enabled: false + config: + metrics: + statsd_on: true + statsd_host: ... + statsd_port: ... + +IPv6 StatsD +^^^^^^^^^^^ + +To use an StatsD instance with IPv6 address. Example with Kubernetes with IPv6 enabled: + +.. code-block:: yaml + :caption: values.yaml + + statsd: + enabled: true + config: + metrics: + statsd_on: 'True' + statsd_host: ... + statsd_ipv6: 'True' + statsd_port: ... + statsd_prefix: airflow + +Datadog +^^^^^^^ +If you are using a Datadog agent in your environment, this will enable Airflow to export metrics to the Datadog agent. + +.. code-block:: yaml + :caption: values.yaml + + statsd: + enabled: false + config: + metrics: + statsd_on: true + statsd_port: 8125 + extraEnv: |- + - name: AIRFLOW__METRICS__STATSD_HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + +Celery Backend +-------------- + +If you are using ``CeleryExecutor`` or ``CeleryKubernetesExecutor``, you can bring your own Celery backend. + +By default, the chart will deploy Redis. However, you can use any supported Celery backend instead: + +.. code-block:: yaml + :caption: values.yaml + + redis: + enabled: false + data: + brokerUrl: redis://redis-user:password@redis-host:6379/0 + +For more information about setting up a Celery broker, refer to the +exhaustive `Celery documentation on the topic `_. + +Security Context +---------------- + +Constraints +^^^^^^^^^^^ + +A ``Security Context Constraint`` (SCC) is a OpenShift construct that works as a RBAC rule. However, it targets Pods instead of users. +When defining a SCC, one can control actions and resources a POD can perform or access during startup and runtime. + +The SCCs are split into different levels or categories with the ``restricted`` SCC being the default one assigned to Pods. +When deploying Airflow to OpenShift, one can leverage the SCCs and allow the Pods to start containers utilizing the ``anyuid`` SCC. + +In order to enable the usage of SCCs, one must set the parameter ``rbac.createSCCRoleBinding`` to ``true`` as shown below: + +.. code-block:: yaml + :caption: values.yaml + + rbac: + create: true + createSCCRoleBinding: true + +In this chart, SCCs are bound to the Pods via RoleBindings meaning that the option ``rbac.create`` must also be set to ``true`` in order to fully enable the SCC usage. + +For more information about SCCs and what can be achieved with this construct, please refer to `Managing security context constraints `_. + +Configuration +^^^^^^^^^^^^^ + +In Kubernetes a ``securityContext`` can be used to define user ids, group ids and capabilities such as running a container in privileged mode. + +When deploying an application to Kubernetes, it is recommended to give the least privilege to containers +to reduce access and protect the host where the container is running. + +In the Airflow Helm chart, the ``securityContext`` can be configured in several ways: + +* :ref:`uid ` - configures the global uid or RunAsUser +* :ref:`gid ` - configures the global gid or fsGroup +* :ref:`securityContexts ` - same as ``uid``, but allows for setting all `Pod securityContext options `_ and `Container securityContext options `_ + +The same way one can configure the global :ref:`securityContexts `. It is also possible to configure different values for specific workloads by setting their local ``securityContexts`` as follows: + +.. code-block:: yaml + :caption: values.yaml + + scheduler: + securityContexts: + pod: + runAsUser: 5000 + fsGroup: 0 + containers: + allowPrivilegeEscalation: false + + +In the example above, the scheduler pod ``securityContext`` will be set to ``runAsUser: 5000`` and ``fsGroup: 0``. The scheduler container ``securityContext`` will be set to ``allowPrivilegeEscalation: false``. + +As one can see, the local setting will take precedence over the global setting when defined. The following explains the precedence rule for ``securityContexts`` options in this chart: + +.. code-block:: yaml + :caption: values.yaml + + uid: 40000 + gid: 0 + + securityContexts: + pod: + runAsUser: 50000 + fsGroup: 0 + + scheduler: + securityContexts: + pod: + runAsUser: 1001 + fsGroup: 0 + +This will generate the following scheduler deployment: + +.. code-block:: yaml + :caption: airflow-scheduler + + kind: Deployment + apiVersion: apps/v1 + metadata: + name: airflow-scheduler + spec: + template: + spec: + securityContext: # As the securityContexts was defined in ``scheduler``, its value will take priority + runAsUser: 1001 + fsGroup: 0 + +If we remove both the ``securityContexts`` and ``scheduler.securityContexts`` from the example above: + +.. code-block:: yaml + :caption: values.yaml + + uid: 40000 + gid: 0 + + securityContexts: {} + + scheduler: + securityContexts: {} + +it will generate the following scheduler deployment: + +.. code-block:: yaml + :caption: airflow-scheduler + + kind: Deployment + apiVersion: apps/v1 + metadata: + name: airflow-scheduler + spec: + template: + spec: + securityContext: + runAsUser: 40000 # As the securityContext was not defined in ``scheduler`` or ``podSecurity``, the value from uid will be used + fsGroup: 0 # As the securityContext was not defined in ``scheduler`` or ``podSecurity``, the value from gid will be used + initContainers: + - name: wait-for-airflow-migrations + ... + containers: + - name: scheduler + ... + +And finally if we set ``securityContexts``, but not ``scheduler.securityContexts``: + +.. code-block:: yaml + :caption: values.yaml + + uid: 40000 + gid: 0 + + securityContexts: + pod: + runAsUser: 50000 + fsGroup: 0 + + scheduler: + securityContexts: {} + +This will generate the following scheduler deployment: + +.. code-block:: yaml + :caption: airflow-scheduler + + kind: Deployment + apiVersion: apps/v1 + metadata: + name: airflow-scheduler + spec: + template: + spec: + securityContext: # As the securityContexts was not defined in ``scheduler``, the values from securityContexts will take priority + runAsUser: 50000 + fsGroup: 0 + initContainers: + - name: wait-for-airflow-migrations + ... + containers: + - name: scheduler + ... + +Built-in secrets and environment variables +------------------------------------------ + +The Helm Chart by default uses Kubernetes Secrets to store secrets that are needed by Airflow. +The contents of those secrets are by default turned into environment variables that are read by +Airflow. + +.. note:: + + Some of the environment variables have several variants to support older versions of Airflow. + +By default, the secret names are determined from the Release Name used when the Helm Chart, +but you can also use a different secret to set the variables or disable using secrets +entirely and rely on environment variables (specifically if you want to use ``_CMD`` or ``__SECRET`` variant +of the environment variable). + +However, Airflow supports other variants of setting secret configuration. You can specify a system +command to retrieve and automatically rotate the secret (by defining variable with ``_CMD`` suffix) or +to retrieve a variable from secret backed (by defining the variable with ``_SECRET`` suffix). + +If the ```` is set, it takes precedence over the ``_CMD`` and ``_SECRET`` variant, so +if you want to set one of the ``_CMD`` or ``_SECRET`` variants, you **must** disable the built in +variables retrieved from Kubernetes secrets, by setting ``.Values.enableBuiltInSecretEnvVars.`` +to ``false``. + +For example in order to use a command to retrieve the DB connection, you should (in your ``values.yaml`` +file) specify: + +.. code-block:: yaml + :caption: values.yaml + + extraEnv: + AIRFLOW_CONN_AIRFLOW_DB_CMD: "/usr/local/bin/retrieve_connection_url" + enableBuiltInSecretEnvVars: + AIRFLOW_CONN_AIRFLOW_DB: false + +Here is the full list of secrets that can be disabled and replaced by ``_CMD`` and ``_SECRET`` variants: + ++-------------------------------------------------------+------------------------------------------+--------------------------------------------------+ +| Default secret name if secret name not specified | Use a different Kubernetes Secret | Airflow Environment Variable | ++=======================================================+==========================================+==================================================+ +| ``-airflow-metadata`` | ``.Values.data.metadataSecretName`` | | ``AIRFLOW_CONN_AIRFLOW_DB`` | +| | | | ``AIRFLOW__DATABASE__SQL_ALCHEMY_CONN`` | ++-------------------------------------------------------+------------------------------------------+--------------------------------------------------+ +| ``-fernet-key`` | ``.Values.fernetKeySecretName`` | ``AIRFLOW__CORE__FERNET_KEY`` | ++-------------------------------------------------------+------------------------------------------+--------------------------------------------------+ +| ``-api-secret-key`` | ``.Values.apiSecretKeySecretName`` | ``AIRFLOW__API__SECRET_KEY`` | ++-------------------------------------------------------+------------------------------------------+--------------------------------------------------+ +| ``-jwt-secret`` | ``.Values.jwtSecretName`` | ``AIRFLOW__API_AUTH__JWT_SECRET`` | ++-------------------------------------------------------+------------------------------------------+--------------------------------------------------+ +| ``-webserver-secret-key`` | ``.Values.webserverSecretKeySecretName`` | ``AIRFLOW__WEBSERVER__SECRET_KEY`` | ++-------------------------------------------------------+------------------------------------------+--------------------------------------------------+ +| ``-airflow-result-backend`` | ``.Values.data.resultBackendSecretName`` | ``AIRFLOW__CELERY__RESULT_BACKEND`` | ++-------------------------------------------------------+------------------------------------------+--------------------------------------------------+ +| ``-airflow-broker-url`` | ``.Values.data.brokerUrlSecretName`` | ``AIRFLOW__CELERY__BROKER_URL`` | ++-------------------------------------------------------+------------------------------------------+--------------------------------------------------+ +| ``-elasticsearch`` | ``.Values.elasticsearch.secretName`` | ``AIRFLOW__ELASTICSEARCH__HOST`` | ++-------------------------------------------------------+------------------------------------------+--------------------------------------------------+ + +There are also a number of secrets, which names are also determined from the release name, that do not need to +be disabled. This is because either they do not follow the ``_CMD`` or ``_SECRET`` pattern, are variables +which do not start with ``AIRFLOW__``, or they do not have a corresponding variable. + +There is also ``AIRFLOW__CELERY__FLOWER_BASIC_AUTH``, that does not need to be disabled, +even if you want set the ``_CMD`` and ``_SECRET`` variant. This variable is not set by default. It is only set +when ``.Values.flower.secretName`` is set or when ``.Values.flower.user`` and ``.Values.flower.password`` +are set. If you do not set any of the ``.Values.flower.*`` variables, you can freely configure +flower Basic Auth using the ``_CMD`` or ``_SECRET`` variant without disabling the basic variant. + ++-------------------------------------------------------+------------------------------------------+------------------------------------------------+ +| Default secret name if secret name not specified | Use a different Kubernetes Secret | Airflow Environment Variable | ++=======================================================+==========================================+================================================+ +| ``-redis-password`` | ``.Values.redis.passwordSecretName`` | ``REDIS_PASSWORD`` | ++-------------------------------------------------------+------------------------------------------+------------------------------------------------+ +| ``-pgbouncer-config`` | ``.Values.pgbouncer.configSecretName`` | | ++-------------------------------------------------------+------------------------------------------+------------------------------------------------+ +| ``-pgbouncer-certificates`` | | | ++-------------------------------------------------------+------------------------------------------+------------------------------------------------+ +| ``-kerberos-keytab`` | | | ++-------------------------------------------------------+------------------------------------------+------------------------------------------------+ +| ``-flower`` | ``.Values.flower.secretName`` | ``AIRFLOW__CELERY__FLOWER_BASIC_AUTH`` | ++-------------------------------------------------------+------------------------------------------+------------------------------------------------+ + +A secret named ``-registry`` is also created when ``.Values.registry.connection`` is +defined and neither ``.Values.registry.secretName`` nor ``.Values.imagePullSecrets`` is set. However, +this behavior is deprecated in favor of explicitly defining ``.Values.imagePullSecrets``. + +You can read more about advanced ways of setting configuration variables in the +:doc:`apache-airflow:howto/set-config`. + +Service Account Token Volume Configuration +------------------------------------------ + +When using pod-launching executors (``CeleryExecutor``, ``CeleryKubernetesExecutor``, ``KubernetesExecutor``, ``LocalKubernetesExecutor``), +you can configure how Kubernetes service account tokens are mounted into pods. This provides enhanced security control +and compatibility with security policies like Kyverno. + +Background +^^^^^^^^^^ + +By default, Kubernetes automatically mounts service account tokens into pods via the ``automountServiceAccountToken`` setting. +However, for security reasons, you might want to disable automatic mounting and manually configure service account token volumes instead. + +This feature addresses Bug `#59099 `_ where ``scheduler.serviceAccount.automountServiceAccountToken: false`` was ignored +when using the KubernetesExecutor. The solution implements a defense-in-depth approach with both ServiceAccount-level +and Pod-level controls. + +Container-Specific Security +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The Service Account Token Volume is mounted **only** in containers that require Kubernetes API access, implementing the +**Principle of Least Privilege**: + +* **Scheduler Container**: Receives Service Account Token as it needs API access for pod management +* **Init Container "wait-for-airflow-migrations"**: No Service Account Token as it only performs database migrations +* **Sidecar Container "scheduler-log-groomer"**: No Service Account Token as it only performs log cleanup operations + +This container-specific approach ensures that: + +* **Database Migration Container**: Only accesses the database for schema updates as no Kubernetes API access required +* **Log Groomer Container**: Only performs filesystem operations for log cleanup as no API access required +* **Scheduler Container**: Requires API access for launching and managing pods with pod-launching executors + +**Security Benefits:** + +* **Explicit control**: Manual configuration makes token mounting intentional and visible +* **Policy compliance**: Compatible with security policies that restrict ``automountServiceAccountToken: true`` +* **Defense-in-depth**: Provides both ServiceAccount-level and Pod-level security controls +* **Custom expiration**: Allows setting shorter token lifetimes for enhanced security +* **Container isolation**: Only scheduler container receives API access, reducing attack surface +* **Principle of Least Privilege**: Each container receives only the minimum required permissions + +Configuration Options +^^^^^^^^^^^^^^^^^^^^^ + +The service account token volume configuration is available for the scheduler component and includes the following options: + +.. code-block:: yaml + :caption: values.yaml + + scheduler: + serviceAccount: + automountServiceAccountToken: false + serviceAccountTokenVolume: + enabled: true + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + volumeName: kube-api-access + expirationSeconds: 3600 + audience: ~ + +Security Implications +^^^^^^^^^^^^^^^^^^^^^ + +Manual token volumes should be used when: + +* Security policies require explicit control over service account token mounting +* Using security policy engines like Kyverno that restrict automatic token mounting +* Implementing defense-in-depth security strategies +* You need custom token expiration times or audiences +* Compliance frameworks mandate container-specific privilege assignment + +Use Cases and Examples +^^^^^^^^^^^^^^^^^^^^^^ + +For comprehensive configuration examples, security scenarios, and detailed use cases, +see :doc:`service-account-token-examples`. + +Supported Executors +^^^^^^^^^^^^^^^^^^^ + +The service account token volume configuration is only effective for pod-launching executors: + +* ``CeleryExecutor`` - when launching Celery worker pods +* ``CeleryKubernetesExecutor`` - for both Celery workers and Kubernetes task pods +* ``KubernetesExecutor`` - when launching task pods in Kubernetes +* ``LocalKubernetesExecutor`` - for Kubernetes task pods in local mode + +For other executors (``LocalExecutor``, ``SequentialExecutor``), this configuration has no effect +as they don't launch additional pods. + +Migration from Automatic to Manual Token Mounting +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To migrate from automatic to manual token mounting: + +1. Test the configuration in a non-production environment first +2. Update your ``values.yaml``: + + .. code-block:: yaml + :caption: values.yaml + + scheduler: + serviceAccount: + automountServiceAccountToken: false + serviceAccountTokenVolume: + enabled: true + +3. Deploy the changes using Helm upgrade +4. Verify that the scheduler can still launch pods successfully +5. Monitor for any authentication issues in the logs + +Troubleshooting +^^^^^^^^^^^^^^^ + +**Common Issues:** + +* **Authentication failures**: Ensure ``serviceAccountTokenVolume.enabled`` is set to ``true`` when ``automountServiceAccountToken`` is ``false`` +* **Permission denied**: Verify that the service account has the necessary RBAC permissions +* **Token expiration**: Check if ``expirationSeconds`` is too short for your workload patterns + +**Debugging:** + +Check the scheduler logs for authentication-related errors: + +.. code-block:: bash + + kubectl logs deployment/airflow-scheduler -n + +Verify the projected volume is mounted correctly: + +.. code-block:: bash + + kubectl describe pod -n + +Backward Compatibility +^^^^^^^^^^^^^^^^^^^^^^ + +This feature maintains full backward compatibility: + +* Existing deployments with ``automountServiceAccountToken: true`` continue to work unchanged +* The ``serviceAccountTokenVolume`` configuration is only applied when explicitly enabled +* Default values ensure no breaking changes for existing installations + +For more information about Kubernetes service account tokens and projected volumes, see the +`Kubernetes documentation on service account tokens `_. diff --git a/charts/airflow/docs/quick-start.rst b/charts/airflow/docs/quick-start.rst new file mode 100644 index 0000000..399e47d --- /dev/null +++ b/charts/airflow/docs/quick-start.rst @@ -0,0 +1,228 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Quick start with kind +===================== + +This article will show you how to install Airflow using Helm Chart on `Kind `__. + +Run Airflow on Kind cluster +--------------------------- + +Install kind and create a cluster +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We recommend testing with Kubernetes 1.30+, example: + +.. code-block:: bash + + kind create cluster --image kindest/node:v1.30.13 + +Confirm it's up: + +.. code-block:: bash + + kubectl cluster-info --context kind-kind + +Add Airflow Helm Stable Repo +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + helm repo add apache-airflow https://airflow.apache.org + helm repo update + +Create namespace +^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + export NAMESPACE=example-namespace + kubectl create namespace $NAMESPACE + +.. note:: + + Same exported ``NAMESPACE`` variable will be used in the next instructions. + +Install the chart +^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + export RELEASE_NAME=example-release + helm install $RELEASE_NAME apache-airflow/airflow --namespace $NAMESPACE + +.. note:: + + Same exported ``RELEASE_NAME`` variable will be used in the next instructions. + +Use the following code to install the chart with example Dags: + +.. code-block:: bash + + helm install $RELEASE_NAME apache-airflow/airflow \ + --namespace $NAMESPACE \ + --set-string "env[0].name=AIRFLOW__CORE__LOAD_EXAMPLES" \ + --set-string "env[0].value=True" + +It may take a few minutes. Confirm the pods are up: + +.. code-block:: bash + + kubectl get pods --namespace $NAMESPACE + helm list --namespace $NAMESPACE + +Run the following command to port-forward the Airflow UI to http://localhost:8080/ to confirm +Airflow is working. + +.. code-block:: bash + + kubectl port-forward svc/$RELEASE_NAME-api-server 8080:8080 --namespace $NAMESPACE + +.. _quick-start:extending-airflow-image: + +Extending Airflow Image +----------------------- + +The Apache Airflow community, releases Docker Images which are ``reference images`` for Apache Airflow. +However, when you try it out you want to add your own Dags, custom dependencies, packages, or even custom providers. + +.. note:: + + Creating custom images means that you need to maintain also a level of automation as you need to re-create the images + when either the packages you want to install or Airflow is upgraded. Do not forget about keeping these scripts. + Also keep in mind, that in cases when you run pure Python tasks, you can use the + `Python Virtualenv functions `_ + which will dynamically source and install python dependencies during runtime. Virtualenvs can also be cached. + +The best way to achieve it, is to build your own, custom image. + +Adding Dags to your image +^^^^^^^^^^^^^^^^^^^^^^^^^ + +1. Create a project: + + .. code-block:: bash + + mkdir my-airflow-project && cd my-airflow-project + mkdir dags # put dags here + cat < Dockerfile + FROM apache/airflow + COPY . . + EOM + +2. Then build the image: + + .. code-block:: bash + + docker build --pull --tag my-dags:0.0.1 . + +3. Load the image into kind: + + .. code-block:: bash + + kind load docker-image my-dags:0.0.1 + +4. Upgrade Helm deployment: + + .. code-block:: bash + + helm upgrade $RELEASE_NAME apache-airflow/airflow --namespace $NAMESPACE \ + --set images.airflow.repository=my-dags \ + --set images.airflow.tag=0.0.1 + +Adding ``apt`` packages to your image +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example below adds ``vim`` apt package. + +1. Create a project: + + .. code-block:: bash + + mkdir my-airflow-project && cd my-airflow-project + cat < Dockerfile + FROM apache/airflow + USER root + RUN apt-get update \ + && apt-get install -y --no-install-recommends vim \ + && apt-get autoremove -yqq --purge \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + USER airflow + EOM + +2. Then build the image: + + .. code-block:: bash + + docker build --pull --tag my-image:0.0.1 . + +3. Load the image into kind: + + .. code-block:: bash + + kind load docker-image my-image:0.0.1 + +4. Upgrade Helm deployment: + + .. code-block:: bash + + helm upgrade $RELEASE_NAME apache-airflow/airflow --namespace $NAMESPACE \ + --set images.airflow.repository=my-image \ + --set images.airflow.tag=0.0.1 + +Adding ``PyPI`` packages to your image +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example below adds ``lxml`` PyPI package. + +1. Create a project: + + .. code-block:: bash + + mkdir my-airflow-project && cd my-airflow-project + cat < Dockerfile + FROM apache/airflow + RUN pip install --no-cache-dir lxml + EOM + +2. Then build the image: + + .. code-block:: bash + + docker build --pull --tag my-image:0.0.1 . + +3. Load the image into kind: + + .. code-block:: bash + + kind load docker-image my-image:0.0.1 + +4. Upgrade Helm deployment: + + .. code-block:: bash + + helm upgrade $RELEASE_NAME apache-airflow/airflow --namespace $NAMESPACE \ + --set images.airflow.repository=my-image \ + --set images.airflow.tag=0.0.1 + +Further extending and customizing the image +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +See `Building the image `_ for more +details on how you can extend and customize the Airflow image. diff --git a/charts/airflow/docs/redirects.txt b/charts/airflow/docs/redirects.txt new file mode 100644 index 0000000..0689565 --- /dev/null +++ b/charts/airflow/docs/redirects.txt @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Release Notes +changelog.rst release_notes.rst +updating.rst release_notes.rst diff --git a/charts/airflow/docs/release_notes.rst b/charts/airflow/docs/release_notes.rst new file mode 100644 index 0000000..66ccaa3 --- /dev/null +++ b/charts/airflow/docs/release_notes.rst @@ -0,0 +1,23 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +Release Notes +============= + +.. include:: ../RELEASE_NOTES.rst diff --git a/charts/airflow/docs/service-account-token-examples.rst b/charts/airflow/docs/service-account-token-examples.rst new file mode 100644 index 0000000..de05502 --- /dev/null +++ b/charts/airflow/docs/service-account-token-examples.rst @@ -0,0 +1,426 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Service Account Token Volume Examples +===================================== + +This document provides comprehensive examples for configuring Service Account Token Volumes +in the Apache Airflow Helm Chart. These examples demonstrate various security scenarios and +use cases for pod-launching executors. + +Overview +-------- + +Service Account Token Volume configuration allows you to manually control how Kubernetes +service account tokens are mounted into pods launched by Airflow. This feature implements the +**Principle of Least Privilege** by providing tokens only to containers that require Kubernetes API access. + +**Container-Specific Security Model:** + +- **Scheduler Container**: Receives Service Account Token as it requires API access for pod management +- **Init Container**: No Service Account Token as it only performs database migrations +- **Sidecar Container**: No Service Account Token as it only performs log cleanup operations + +**This feature is particularly useful for**: + +- Compliance with security policies that restrict automatic token mounting +- Implementation of defense-in-depth security strategies +- Custom token expiration and audience configuration +- Compatibility with security policy engines like Kyverno +- Meeting compliance frameworks that mandate container-specific privilege assignment + +Basic Configuration Examples +---------------------------- + +Default Automatic Token Mounting +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This is the default behavior that continues to work without any changes: + +.. code-block:: yaml + :caption: values.yaml + + scheduler: + serviceAccount: + automountServiceAccountToken: true # Default value + +Manual Token Volume Configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Basic manual configuration that disables automatic mounting and enables manual token volume: + +.. code-block:: yaml + :caption: values.yaml + + scheduler: + serviceAccount: + automountServiceAccountToken: false + serviceAccountTokenVolume: + enabled: true + +This configuration: + +- Disables automatic service account token mounting at the ServiceAccount level +- Enables manual token volume mounting at the Pod level +- Uses default values for all other token volume settings + +Security-Focused Examples +------------------------- + +High-Security Environment +^^^^^^^^^^^^^^^^^^^^^^^^^ + +For environments requiring enhanced security with shorter token lifetimes: + +.. code-block:: yaml + :caption: values.yaml + + scheduler: + serviceAccount: + automountServiceAccountToken: false + serviceAccountTokenVolume: + enabled: true + expirationSeconds: 1800 # 30 minutes instead of default 1 hour + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + volumeName: secure-kube-access + +**Security Benefits:** + +- **Shorter token lifetime** reduces exposure window if compromised +- **Explicit control** over token mounting makes security intentional +- **Container isolation** as only scheduler container receives API access +- **Clear naming** for security auditing and compliance reporting + +Kyverno Policy Compliance +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Configuration that complies with Kyverno policies requiring ``automountServiceAccountToken: false`` +(`Restrict Auto-Mount of Service Account Tokens in Service Account`_ and `Restrict Auto-Mount of Service Account Tokens`_): + +.. _Restrict Auto-Mount of Service Account Tokens in Service Account: https://kyverno.io/policies/other/restrict-sa-automount-sa-token/restrict-sa-automount-sa-token/ +.. _Restrict Auto-Mount of Service Account Tokens: https://kyverno.io/policies/other/restrict-sa-automount-sa-token/restrict-sa-automount-sa-token/ + +.. code-block:: yaml + :caption: values.yaml + + scheduler: + serviceAccount: + automountServiceAccountToken: false # Required by Kyverno policy + serviceAccountTokenVolume: + enabled: true + expirationSeconds: 3600 + audience: "https://kubernetes.default.svc.cluster.local" + +Custom Mount Path Configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For applications that expect service account tokens at custom locations: + +.. code-block:: yaml + :caption: values.yaml + + scheduler: + serviceAccount: + automountServiceAccountToken: false + serviceAccountTokenVolume: + enabled: true + mountPath: /custom/sa-token + volumeName: custom-service-account-token + expirationSeconds: 7200 # 2 hours + +This configuration mounts the token at ``/custom/sa-token`` instead of the default location. + +Executor-Specific Examples +-------------------------- + +KubernetesExecutor Configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Optimal configuration for ``KubernetesExecutor`` with security focus: + +.. code-block:: yaml + :caption: values.yaml + + executor: KubernetesExecutor + + scheduler: + serviceAccount: + automountServiceAccountToken: false + serviceAccountTokenVolume: + enabled: true + expirationSeconds: 3600 + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + volumeName: k8s-executor-token + + # Ensure RBAC permissions for KubernetesExecutor + rbac: + create: true + +CeleryKubernetesExecutor Configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Configuration for hybrid executor that launches both Celery workers and Kubernetes task pods: + +.. code-block:: yaml + :caption: values.yaml + + executor: CeleryKubernetesExecutor + + scheduler: + serviceAccount: + automountServiceAccountToken: false + serviceAccountTokenVolume: + enabled: true + expirationSeconds: 5400 # 1.5 hours for longer-running tasks + volumeName: hybrid-executor-token + + # Redis configuration for Celery + redis: + enabled: true + +Multi-Environment Examples +-------------------------- + +Development Environment +^^^^^^^^^^^^^^^^^^^^^^^ + +Relaxed configuration for development with longer token lifetimes: + +.. code-block:: yaml + :caption: values-dev.yaml + + scheduler: + serviceAccount: + automountServiceAccountToken: false + serviceAccountTokenVolume: + enabled: true + expirationSeconds: 14400 # 4 hours for development convenience + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + +Production Environment +^^^^^^^^^^^^^^^^^^^^^^ + +Strict production configuration with enhanced security: + +.. code-block:: yaml + :caption: values-prod.yaml + + # Additional production security settings + securityContexts: + pod: + runAsNonRoot: true + runAsUser: 50000 + fsGroup: 0 + container: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + + scheduler: + serviceAccount: + automountServiceAccountToken: false + serviceAccountTokenVolume: + enabled: true + expirationSeconds: 1800 # 30 minutes for production security + audience: "https://kubernetes.default.svc.cluster.local" + volumeName: prod-airflow-token + +.. note:: + + Remember that it is a good practice to have the production configuration on the test environment to ensure + reliable testing before moving changes to production. + +Migration Examples +------------------ + +Gradual Migration from Automatic to Manual +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +1. Test manual configuration alongside automatic (for validation): + + .. code-block:: yaml + :caption: values-test.yaml + + scheduler: + serviceAccount: + automountServiceAccountToken: true # Keep automatic for now + serviceAccountTokenVolume: + enabled: false # Disable manual for testing + +2. Enable manual configuration while keeping automatic (transition phase): + + .. code-block:: yaml + :caption: values-transition.yaml + + scheduler: + serviceAccount: + automountServiceAccountToken: true # Still automatic + serviceAccountTokenVolume: + enabled: true # Test manual mounting + expirationSeconds: 3600 + +3. Complete migration to manual-only: + + .. code-block:: yaml + :caption: values-final.yaml + + scheduler: + serviceAccount: + automountServiceAccountToken: false # Disable automatic + serviceAccountTokenVolume: + enabled: true # Use manual only + expirationSeconds: 3600 + +Troubleshooting Examples +------------------------ + +Debug Configuration +^^^^^^^^^^^^^^^^^^^ + +Configuration with extended token lifetime for troubleshooting: + +.. code-block:: yaml + :caption: values-debug.yaml + + scheduler: + serviceAccount: + automountServiceAccountToken: false + serviceAccountTokenVolume: + enabled: true + expirationSeconds: 86400 # 24 hours for debugging + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + volumeName: debug-sa-token + + # Enable debug logging + config: + logging: + logging_level: DEBUG + +Validation Commands +^^^^^^^^^^^^^^^^^^^ + +Commands to validate the configuration is working correctly: + +.. code-block:: bash + + # Check if the service account token is mounted correctly + kubectl exec -it deployment/airflow-scheduler -- ls -la /var/run/secrets/kubernetes.io/serviceaccount/ + + # Verify token content and expiration + kubectl exec -it deployment/airflow-scheduler -- cat /var/run/secrets/kubernetes.io/serviceaccount/token | base64 -d + + # Check scheduler logs for authentication issues + kubectl logs deployment/airflow-scheduler | grep -i "auth\|token\|permission" + + # Describe the scheduler pod to see volume mounts + kubectl describe pod -l component=scheduler + +Advanced Configuration Examples +------------------------------- + +Custom Audience Configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For environments requiring specific token audiences: + +.. code-block:: yaml + :caption: values.yaml + + scheduler: + serviceAccount: + automountServiceAccountToken: false + serviceAccountTokenVolume: + enabled: true + audience: "https://my-custom-api-server.example.com" + expirationSeconds: 3600 + +Multi-Cluster Configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Configuration for multi-cluster deployments: + +.. code-block:: yaml + :caption: values-cluster-a.yaml + + scheduler: + serviceAccount: + automountServiceAccountToken: false + serviceAccountTokenVolume: + enabled: true + audience: "https://cluster-a.k8s.example.com" + volumeName: cluster-a-token + expirationSeconds: 3600 + +Integration with External Security Tools +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Configuration compatible with external security scanning and policy tools: + +.. code-block:: yaml + :caption: values-security-compliant.yaml + + # Additional security annotations + airflowPodAnnotations: + security.policy/scanned: "true" + security.policy/compliant: "service-account-token-manual" + + scheduler: + serviceAccount: + automountServiceAccountToken: false + serviceAccountTokenVolume: + enabled: true + expirationSeconds: 1800 # Short-lived tokens + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + volumeName: security-compliant-token + +Best Practices Summary +---------------------- + +**Container Security:** + +1. **Understand container roles**: Only the scheduler container requires Kubernetes API access +2. **Verify token isolation**: Ensure init and sidecar containers operate without service account tokens +3. **Implement least privilege**: Each container should receive only the minimum required permissions + +**Configuration Management:** + +1. **Always test** manual token configuration in non-production environments first +2. **Use shorter token lifetimes** around 1800-3600 seconds +3. **Set explicit audiences** when integrating with external systems +4. **Use descriptive volume names** for easier troubleshooting and security auditing + +**Security Monitoring:** + +1. **Monitor logs** for authentication issues after migration +2. **Document your configuration** for security auditing and compliance purposes +3. **Audit container permissions** regularly to ensure compliance with security policies +4. **Combine with other security measures** like Pod Security Standards and network policies + +**Migration Strategy:** + +1. **Plan gradual migration** from automatic to manual token mounting +2. **Validate functionality** of each container type after configuration changes +3. **Maintain backward compatibility** during transition periods + +**Why This Approach is More Secure:** + +1. **Reduced attack surface**: Containers without API access cannot be used to compromise the cluster +2. **Clear security boundaries**: Explicit definition of which containers have API privileges +3. **Compliance ready**: Meets requirements for container-specific privilege assignment +4. **Audit friendly**: Clear documentation of security controls for compliance reporting + +For more detailed information, see the :doc:`production-guide` section on Service Account Token Volume Configuration. diff --git a/charts/airflow/docs/setting-resources-for-containers.rst b/charts/airflow/docs/setting-resources-for-containers.rst new file mode 100644 index 0000000..1402d46 --- /dev/null +++ b/charts/airflow/docs/setting-resources-for-containers.rst @@ -0,0 +1,75 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Setting resources for containers +================================ + +It is possible to set `resources `__ for the containers managed by the chart. +You can define different resources for various Airflow containers. By default the resources are not set. + +.. note:: + + The Kubernetes scheduler can use resources to decide which node to place the pod on. Since a pod resource request/limit is the sum of + the resource requests/limits for each container in the pod, it is advised to specify resources for each container in the pod. + +Possible containers where resources can be configured include: + +* Main Airflow containers and their sidecars. You can add the resources for these containers through the following parameters: + + * ``workers.resources`` + * ``workers.celery.logGroomerSidecar.resources`` + * ``workers.kerberosSidecar.resources`` + * ``workers.kerberosInitContainer.resources`` + * ``scheduler.resources`` + * ``scheduler.logGroomerSidecar.resources`` + * ``dags.gitSync.resources`` + * ``apiServer.resources`` + * ``webserver.resources`` + * ``flower.resources`` + * ``dagProcessor.resources`` + * ``dagProcessor.logGroomerSidecar.resources`` + * ``triggerer.resources`` + * ``triggerer.logGroomerSidecar.resources`` + +* Containers used for Airflow Kubernetes jobs or cron jobs. You can add the resources for these containers through the following parameters: + + * ``cleanup.resources`` + * ``createUserJob.resources`` + * ``migrateDatabaseJob.resources`` + * ``databaseCleanup.resources`` + +* Other containers that can be deployed by the chart. You can add the resources for these containers through the following parameters: + + * ``statsd.resources`` + * ``pgbouncer.resources`` + * ``pgbouncer.metricsExporterSidecar.resources`` + * ``redis.resources`` + +For example, specifying resources for worker Kerberos sidecar: + +.. code-block:: yaml + :caption: values.yaml + + workers: + kerberosSidecar: + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi diff --git a/charts/airflow/docs/static/gh-jira-links.js b/charts/airflow/docs/static/gh-jira-links.js new file mode 100644 index 0000000..d731a93 --- /dev/null +++ b/charts/airflow/docs/static/gh-jira-links.js @@ -0,0 +1,34 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +document.addEventListener('DOMContentLoaded', function() { + var el = document.getElementById('release-notes'); + if (el !== null ) { + // [AIRFLOW-...] + el.innerHTML = el.innerHTML.replace( + /\[(AIRFLOW-[\d]+)\]/g, + `[$1]` + ); + // (#...) + el.innerHTML = el.innerHTML.replace( + /\(#([\d]+)\)/g, + `(#$1)` + ); + }; +}) diff --git a/charts/airflow/docs/using-additional-containers.rst b/charts/airflow/docs/using-additional-containers.rst new file mode 100644 index 0000000..af339d6 --- /dev/null +++ b/charts/airflow/docs/using-additional-containers.rst @@ -0,0 +1,68 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Using additional containers +=========================== + +Sidecar Containers +------------------ + +If you want to deploy your own sidecar container, you can add it through the ``extraContainers`` parameter. +You can define different containers for the scheduler, webserver/api-server, Kubernetes/Celery workers, triggerer, dag processor, flower, create user job and migrate database job pods. + +For example, sidecars that sync Dags from object storage: + +.. code-block:: yaml + :caption: values.yaml + + scheduler: + extraContainers: + - name: s3-sync + image: my-company/s3-sync:latest + imagePullPolicy: Always + + workers: + kubernetes: + extraContainers: + - name: s3-sync + image: my-company/s3-sync:latest + imagePullPolicy: Always + +.. note:: + + If you use ``workers.kubernetes.extraContainers`` (dedicated for ``KubernetesExecutor``), you are responsible for signaling + sidecars to exit when the main container finishes so Airflow can continue the worker shutdown process. + + +Init Containers +--------------- + +You can also deploy extra init containers through the ``extraInitContainers`` parameter. +You can define different containers for the scheduler, webserver/api-server, Celery/Kubernetes workers, triggerer, dag processor, create user job and migrate database job pods. + +For example, an init container that just says hello: + +.. code-block:: yaml + :caption: values.yaml + + scheduler: + extraInitContainers: + - name: hello + image: debian + args: + - echo + - hello diff --git a/charts/airflow/files/pod-template-file.kubernetes-helm-yaml b/charts/airflow/files/pod-template-file.kubernetes-helm-yaml new file mode 100644 index 0000000..26b1b85 --- /dev/null +++ b/charts/airflow/files/pod-template-file.kubernetes-helm-yaml @@ -0,0 +1,292 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} +--- +{{- $nodeSelector := or .Values.workers.kubernetes.nodeSelector .Values.workers.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.workers.kubernetes.affinity .Values.workers.affinity .Values.affinity }} +{{- $tolerations := or .Values.workers.kubernetes.tolerations .Values.workers.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.workers.kubernetes.topologySpreadConstraints .Values.workers.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $securityContext := include "airflowPodSecurityContext" (list .Values.workers.kubernetes .Values.workers .Values) }} +{{- $containerSecurityContextKerberosSidecar := include "containerSecurityContext" (list .Values.workers.kubernetes.kerberosSidecar .Values.workers.kerberosSidecar .Values) }} +{{- $containerLifecycleHooksKerberosSidecar := or .Values.workers.kubernetes.kerberosSidecar.containerLifecycleHooks .Values.workers.kerberosSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $containerSecurityContextKerberosInitContainer := include "containerSecurityContext" (list .Values.workers.kubernetes.kerberosInitContainer .Values.workers.kerberosInitContainer .Values) }} +{{- $containerLifecycleHooksKerberosInitContainer := or .Values.workers.kubernetes.kerberosInitContainer.containerLifecycleHooks .Values.workers.kerberosInitContainer.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.workers.kubernetes .Values.workers .Values) }} +{{- $containerLifecycleHooks := or .Values.workers.kubernetes.containerLifecycleHooks .Values.workers.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $safeToEvict := dict "cluster-autoscaler.kubernetes.io/safe-to-evict" (or .Values.workers.kubernetes.safeToEvict (and (not (has .Values.workers.kubernetes.safeToEvict (list true false))) .Values.workers.safeToEvict) | toString) }} +{{- $podAnnotations := mergeOverwrite (deepCopy .Values.airflowPodAnnotations) $safeToEvict (.Values.workers.kubernetes.podAnnotations | default .Values.workers.podAnnotations) }} +{{- $schedulerName := or .Values.workers.kubernetes.schedulerName .Values.workers.schedulerName .Values.schedulerName }} +apiVersion: v1 +kind: Pod +metadata: + name: placeholder-name + labels: + tier: airflow + component: worker + release: {{ .Release.Name }} + {{- if or .Values.labels .Values.workers.labels .Values.workers.kubernetes.labels }} + {{- mustMerge (.Values.workers.kubernetes.labels | default .Values.workers.labels) .Values.labels | toYaml | nindent 4 }} + {{- end }} + annotations: + {{- tpl (toYaml $podAnnotations) . | nindent 4 }} + {{- if or .Values.workers.kubernetes.kerberosInitContainer.enabled .Values.workers.kerberosInitContainer.enabled }} + checksum/kerberos-keytab: {{ include (print $.Template.BasePath "/secrets/kerberos-keytab-secret.yaml") . | sha256sum }} + {{- end }} +spec: + initContainers: + {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }} + {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 4 }} + {{- end }} + {{- if or .Values.workers.kubernetes.extraInitContainers .Values.workers.extraInitContainers }} + {{- tpl (toYaml (.Values.workers.kubernetes.extraInitContainers | default .Values.workers.extraInitContainers)) . | nindent 4 }} + {{- end }} + {{- if or .Values.workers.kubernetes.kerberosInitContainer.enabled .Values.workers.kerberosInitContainer.enabled }} + - name: kerberos-init + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContextKerberosInitContainer | nindent 8 }} + {{- if $containerLifecycleHooksKerberosInitContainer }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooksKerberosInitContainer) . | nindent 8 }} + {{- end }} + args: ["kerberos", "-o"] + resources: {{- toYaml (.Values.workers.kubernetes.kerberosInitContainer.resources | default .Values.workers.kerberosInitContainer.resources) | nindent 8 }} + volumeMounts: + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- include "airflow_config_mount" . | nindent 8 }} + - name: config + mountPath: {{ .Values.kerberos.configPath | quote }} + subPath: krb5.conf + readOnly: true + - name: kerberos-keytab + subPath: "kerberos.keytab" + mountPath: {{ .Values.kerberos.keytabPath | quote }} + readOnly: true + - name: kerberos-ccache + mountPath: {{ .Values.kerberos.ccacheMountPath | quote }} + readOnly: false + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 8 }} + {{- end }} + {{- if or .Values.workers.extraVolumeMounts .Values.workers.kubernetes.extraVolumeMounts }} + {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumeMounts | default .Values.workers.extraVolumeMounts)) . | nindent 8 }} + {{- end }} + {{- if semverCompare ">=3.0.0" .Values.airflowVersion }} + {{- if or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName }} + {{- include "airflow_api_server_config_mount" . | nindent 8 }} + {{- end }} + {{- else }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + {{- include "airflow_webserver_config_mount" . | nindent 8 }} + {{- end }} + {{- end }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 6 }} + env: + - name: KRB5_CONFIG + value: {{ .Values.kerberos.configPath | quote }} + - name: KRB5CCNAME + value: {{ include "kerberos_ccache_path" . | quote }} + {{- include "custom_airflow_environment" . | indent 6 }} + {{- include "standard_airflow_environment" . | indent 6 }} + {{- end }} + containers: + - envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 6 }} + env: + - name: AIRFLOW__CORE__EXECUTOR + value: {{ .Values.executor | quote }} + {{- if or .Values.workers.kubernetes.kerberosSidecar.enabled .Values.workers.kerberosSidecar.enabled .Values.workers.kubernetes.kerberosInitContainer.enabled .Values.workers.kerberosInitContainer.enabled }} + - name: KRB5_CONFIG + value: {{ .Values.kerberos.configPath | quote }} + - name: KRB5CCNAME + value: {{ include "kerberos_ccache_path" . | quote }} + {{- end }} + {{- include "standard_airflow_environment" . | indent 6}} + {{- include "custom_airflow_environment" . | indent 6 }} + {{- include "container_extra_envs" (list . (.Values.workers.kubernetes.env | default .Values.workers.env)) | indent 6 }} + image: {{ template "pod_template_image" . }} + imagePullPolicy: {{ .Values.images.pod_template.pullPolicy }} + securityContext: {{ $containerSecurityContext | nindent 8 }} + {{- if $containerLifecycleHooks }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 8 }} + {{- end }} + name: base + {{- if or .Values.workers.kubernetes.command .Values.workers.command }} + command: {{ tpl (toYaml (.Values.workers.kubernetes.command | default .Values.workers.command)) . | nindent 8 }} + {{- end }} + resources: {{- toYaml (.Values.workers.kubernetes.resources | default .Values.workers.resources) | nindent 8 }} + volumeMounts: + - mountPath: {{ template "airflow_logs" . }} + name: logs + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- include "airflow_config_mount" . | nindent 8 }} + {{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }} + {{- include "airflow_dags_mount" . | nindent 8 }} + {{- end }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 8 }} + {{- end }} + {{- if or .Values.workers.extraVolumeMounts .Values.workers.kubernetes.extraVolumeMounts }} + {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumeMounts | default .Values.workers.extraVolumeMounts)) . | nindent 8 }} + {{- end }} + {{- if .Values.kerberos.enabled }} + - name: kerberos-keytab + subPath: "kerberos.keytab" + mountPath: {{ .Values.kerberos.keytabPath | quote }} + readOnly: true + - name: config + mountPath: {{ .Values.kerberos.configPath | quote }} + subPath: krb5.conf + readOnly: true + - name: kerberos-ccache + mountPath: {{ .Values.kerberos.ccacheMountPath | quote }} + readOnly: true + {{- end }} + {{- if or .Values.workers.kubernetes.kerberosSidecar.enabled .Values.workers.kerberosSidecar.enabled }} + - name: worker-kerberos + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContextKerberosSidecar | nindent 8 }} + {{- if $containerLifecycleHooksKerberosSidecar }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooksKerberosSidecar) . | nindent 8 }} + {{- end }} + args: ["kerberos"] + resources: {{- toYaml (.Values.workers.kubernetes.kerberosSidecar.resources | default .Values.workers.kerberosSidecar.resources) | nindent 8 }} + volumeMounts: + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- include "airflow_config_mount" . | nindent 8 }} + - name: config + mountPath: {{ .Values.kerberos.configPath | quote }} + subPath: krb5.conf + readOnly: true + - name: kerberos-keytab + subPath: "kerberos.keytab" + mountPath: {{ .Values.kerberos.keytabPath | quote }} + readOnly: true + - name: kerberos-ccache + mountPath: {{ .Values.kerberos.ccacheMountPath | quote }} + readOnly: false + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 8 }} + {{- end }} + {{- if or .Values.workers.extraVolumeMounts .Values.workers.kubernetes.extraVolumeMounts }} + {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumeMounts | default .Values.workers.extraVolumeMounts)) . | nindent 8 }} + {{- end }} + {{- if semverCompare ">=3.0.0" .Values.airflowVersion }} + {{- if or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName }} + {{- include "airflow_api_server_config_mount" . | nindent 8 }} + {{- end }} + {{- else }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + {{- include "airflow_webserver_config_mount" . | nindent 8 }} + {{- end }} + {{- end }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 6 }} + env: + - name: KRB5_CONFIG + value: {{ .Values.kerberos.configPath | quote }} + - name: KRB5CCNAME + value: {{ include "kerberos_ccache_path" . | quote }} + {{- include "custom_airflow_environment" . | indent 6 }} + {{- include "standard_airflow_environment" . | indent 6 }} + {{- end }} + {{- if or .Values.workers.kubernetes.extraContainers .Values.workers.extraContainers }} + {{- tpl (toYaml (.Values.workers.kubernetes.extraContainers | default .Values.workers.extraContainers)) . | nindent 4 }} + {{- end }} + {{- if or .Values.workers.kubernetes.priorityClassName .Values.workers.priorityClassName }} + priorityClassName: {{ .Values.workers.kubernetes.priorityClassName | default .Values.workers.priorityClassName }} + {{- end }} + {{- if or .Values.workers.kubernetes.runtimeClassName .Values.workers.runtimeClassName }} + runtimeClassName: {{ .Values.workers.kubernetes.runtimeClassName | default .Values.workers.runtimeClassName }} + {{- end }} + imagePullSecrets: {{- include "image_pull_secrets" . | nindent 4 }} + {{- if or .Values.workers.kubernetes.hostAliases .Values.workers.hostAliases }} + hostAliases: {{- toYaml (.Values.workers.kubernetes.hostAliases | default .Values.workers.hostAliases) | nindent 4 }} + {{- end }} + restartPolicy: Never + securityContext: {{ $securityContext | nindent 4 }} + nodeSelector: {{- toYaml $nodeSelector | nindent 4 }} + affinity: {{- toYaml $affinity | nindent 4 }} + {{- if $schedulerName }} + schedulerName: {{ $schedulerName }} + {{- end }} + terminationGracePeriodSeconds: {{ .Values.workers.kubernetes.terminationGracePeriodSeconds | default .Values.workers.terminationGracePeriodSeconds }} + tolerations: {{- toYaml $tolerations | nindent 4 }} + topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 4 }} + {{- if .Values.workers.kubernetes.serviceAccount.create }} + serviceAccountName: {{ include "worker.kubernetes.serviceAccountName" . }} + {{- else }} + serviceAccountName: {{ include "worker.serviceAccountName" . }} + {{- end }} + volumes: + {{- if .Values.dags.persistence.enabled }} + - name: dags + persistentVolumeClaim: + claimName: {{ template "airflow_dags_volume_claim" . }} + {{- else if .Values.dags.gitSync.enabled }} + - name: dags + emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 6 }} + {{- end }} + {{- if .Values.logs.persistence.enabled }} + - name: logs + persistentVolumeClaim: + claimName: {{ template "airflow_logs_volume_claim" . }} + {{- else }} + - emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | nindent 6 }} + name: logs + {{- end }} + {{- if and .Values.dags.gitSync.enabled (or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey) }} + {{- include "git_sync_ssh_key_volume" . | nindent 2 }} + {{- end }} + - configMap: + name: {{ include "airflow_config" . }} + name: config + {{- if semverCompare ">=3.0.0" .Values.airflowVersion }} + {{- if and (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) (or .Values.workers.kubernetes.kerberosInitContainer.enabled .Values.workers.kerberosInitContainer.enabled .Values.workers.kubernetes.kerberosSidecar.enabled .Values.workers.kerberosSidecar.enabled) }} + - name: api-server-config + configMap: + name: {{ template "airflow_api_server_config_configmap_name" . }} + {{- end }} + {{- else }} + {{- if and (or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName) (or .Values.workers.kubernetes.kerberosInitContainer.enabled .Values.workers.kerberosInitContainer.enabled .Values.workers.kubernetes.kerberosSidecar.enabled .Values.workers.kerberosSidecar.enabled) }} + - name: webserver-config + configMap: + name: {{ template "airflow_webserver_config_configmap_name" . }} + {{- end }} + {{- end }} + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 2 }} + {{- end }} + {{- if .Values.kerberos.enabled }} + - name: kerberos-keytab + secret: + secretName: {{ include "kerberos_keytab_secret" . | quote }} + - name: kerberos-ccache + emptyDir: {} + {{- end }} + {{- if or .Values.workers.kubernetes.extraVolumes .Values.workers.extraVolumes }} + {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumes | default .Values.workers.extraVolumes)) . | nindent 2 }} + {{- end }} diff --git a/charts/airflow/files/statsd-mappings.yml b/charts/airflow/files/statsd-mappings.yml new file mode 100644 index 0000000..a6a216e --- /dev/null +++ b/charts/airflow/files/statsd-mappings.yml @@ -0,0 +1,142 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +# {{/* +# WARNING: Be aware that most changes to these mappings will break backwards compatibility! +# E.g. moving to labels will break existing dashboards. +# instead utilize `statsd.extraMappings` or `statsd.overrideMappings` in your environment, +# until we have version 2 of the helm chart. +# */}} +mappings: + # Map dot separated stats to labels + - match: airflow.dagrun.dependency-check.*.* + name: "airflow_dagrun_dependency_check" + labels: + dag_id: "$1" + + - match: airflow.operator_successes_(.*) + match_type: regex + name: "airflow_operator_successes" + labels: + operator: "$1" + + - match: airflow.operator_failures_(.*) + match_type: regex + name: "airflow_operator_failures" + labels: + operator: "$1" + + - match: airflow.scheduler_heartbeat + match_type: regex + name: "airflow_scheduler_heartbeat" + labels: + type: counter + + - match: airflow.dag_processor_heartbeat + match_type: regex + name: "airflow_dag_processor_heartbeat" + labels: + type: counter + + - match: airflow.dag.*.*.duration + name: "airflow_task_duration" + labels: + dag_id: "$1" + task_id: "$2" + + - match: airflow.dagrun.duration.success.* + name: "airflow_dagrun_duration" + labels: + dag_id: "$1" + + - match: airflow.dagrun.duration.failed.* + name: "airflow_dagrun_failed" + labels: + dag_id: "$1" + + - match: airflow.dagrun.schedule_delay.* + name: "airflow_dagrun_schedule_delay" + labels: + dag_id: "$1" + + - match: airflow.dag_processing.last_runtime.* + name: "airflow_dag_processing_last_runtime" + labels: + dag_file: "$1" + + - match: airflow.dag_processing.last_run.seconds_ago.* + name: "airflow_dag_processing_last_run_seconds_ago" + labels: + dag_file: "$1" + + - match: airflow.pool.open_slots.* + name: "airflow_pool_open_slots" + labels: + pool: "$1" + + - match: airflow.pool.used_slots.* + name: "airflow_pool_used_slots" + labels: + pool: "$1" + + - match: airflow.pool.starving_tasks.* + name: "airflow_pool_starving_tasks" + labels: + pool: "$1" + + - match: airflow.executor.open_slots.* + name: "airflow_executor_open_slots" + labels: + executor: "$1" + + - match: airflow.executor.queued_tasks.* + name: "airflow_executor_queued_tasks" + labels: + executor: "$1" + + - match: airflow.executor.running_tasks.* + name: "airflow_executor_running_tasks" + labels: + executor: "$1" + + - match: airflow.ti.running.*.*.* + name: "airflow_ti_running" + labels: + queue: "$1" + dag_id: "$2" + task_id: "$3" + + - match: airflow.ti.queued.*.*.* + name: "airflow_ti_queued" + labels: + queue: "$1" + dag_id: "$2" + task_id: "$3" + + - match: airflow.ti.scheduled.*.*.* + name: "airflow_ti_scheduled" + labels: + queue: "$1" + dag_id: "$2" + task_id: "$3" + + - match: airflow.ti.deferred.*.*.* + name: "airflow_ti_deferred" + labels: + queue: "$1" + dag_id: "$2" + task_id: "$3" diff --git a/charts/airflow/newsfragments/config.toml b/charts/airflow/newsfragments/config.toml new file mode 100644 index 0000000..b00560d --- /dev/null +++ b/charts/airflow/newsfragments/config.toml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +[tool.towncrier] +name = "Airflow Helm Chart" +filename = "RELEASE_NOTES.rst" +underlines = ["-", '^'] + +[[tool.towncrier.type]] +directory = "significant" +name = "Significant Changes" +showcontent = true + +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true + +[[tool.towncrier.type]] +directory = "improvement" +name = "Improvements" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Bug Fixes" +showcontent = true + +[[tool.towncrier.type]] +directory = "doc" +name = "Doc only Changes" +showcontent = true + +[[tool.towncrier.type]] +directory = "misc" +name = "Misc" +showcontent = true diff --git a/charts/airflow/pyproject.toml b/charts/airflow/pyproject.toml new file mode 100644 index 0000000..fc903a8 --- /dev/null +++ b/charts/airflow/pyproject.toml @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = [ + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", +] +build-backend = "hatchling.build" + +[project] +name = "apache-airflow-helm-chart" +description = "Programmatically author, schedule and monitor data pipelines" +requires-python = ">=3.10,!=3.15" +authors = [ + { name = "Apache Software Foundation", email = "dev@airflow.apache.org" }, +] +maintainers = [ + { name = "Apache Software Foundation", email="dev@airflow.apache.org" }, +] +keywords = [ "airflow", "orchestration", "workflow", "dag", "pipelines", "automation", "data" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Framework :: Apache Airflow", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: System :: Monitoring", + "Topic :: System :: Monitoring", +] + +version = "0.0.1" + +dependencies = [ + "apache-airflow-core", +] + +[tool.hatch.build.targets.sdist] +exclude = ["*"] + +[tool.hatch.build.targets.wheel] +bypass-selection = true + +[dependency-groups] +# To build docs run: +# +# uv run --group docs sphinx-build -T --color -b html . _build +# +# To check spelling: +# +# uv run --group docs sphinx-build -T --color -b spelling . _build +# +# To enable auto-refreshing build with server: +# +# uv run --group docs sphinx-autobuild -T --color -b html . _build +# +docs = [ + "apache-airflow-devel-common[docs]" +] + +packages = [] diff --git a/charts/airflow/reproducible_build.yaml b/charts/airflow/reproducible_build.yaml new file mode 100644 index 0000000..ef321db --- /dev/null +++ b/charts/airflow/reproducible_build.yaml @@ -0,0 +1,2 @@ +release-notes-hash: d05c9f69d0a418b377ccd2a4328c4bb8 +source-date-epoch: 1776802957 diff --git a/charts/airflow/templates/NOTES.txt b/charts/airflow/templates/NOTES.txt new file mode 100644 index 0000000..57d94ed --- /dev/null +++ b/charts/airflow/templates/NOTES.txt @@ -0,0 +1,1011 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + + +Thank you for installing Apache {{ title .Chart.Name }} {{ .Values.airflowVersion }}! + +Your release is named {{ .Release.Name }}. + +{{- if or .Values.ingress.web.enabled .Values.ingress.flower.enabled .Values.ingress.enabled }} +You can now access your service(s) by following defined Ingress urls: + +{{- if .Values.ingress.web.host }} + +DEPRECATION WARNING: + `ingress.web.host` has been renamed to `ingress.web.hosts` and is now an array. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.ingress.web.tls }} + +DEPRECATION WARNING: + `ingress.web.tls` has been renamed to `ingress.web.hosts[*].tls` and can be set per host. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.ingress.flower.host }} + +DEPRECATION WARNING: + `ingress.flower.host` has been renamed to `ingress.flower.hosts` and is now an array. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + + +{{- if .Values.ingress.flower.tls }} + +DEPRECATION WARNING: + `ingress.flower.tls` has been renamed to `ingress.flower.hosts[*].tls` and can be set per host. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.ingress.enabled }} + +DEPRECATION WARNING: + `ingress.enabled` has been deprecated. There are now separate flags to control the webserver and + flower individually, ``ingress.web.enabled`` and ``ingress.flower.enabled``. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if and .Values.webserver.enabled (or .Values.ingress.web.enabled .Values.ingress.enabled) }} +Airflow Webserver: +{{- range .Values.ingress.web.hosts | default (list .Values.ingress.web.host) }} + {{- $tlsEnabled := $.Values.ingress.web.tls.enabled -}} + {{- $hostname := $.Values.ingress.web.host -}} + {{- $path := $.Values.ingress.web.path -}} + {{- if . | kindIs "string" | not }} + {{- $hostname = .name -}} + {{- if .tls }} + {{- $tlsEnabled = .tls.enabled -}} + {{- end }} + {{- end }} + http{{ if $tlsEnabled }}s{{ end }}://{{ (tpl $hostname $) }}{{ $path }}{{ if (hasSuffix "/" $path) | not }}/{{ end }} +{{- end }} +{{- end }} +{{- if and (or .Values.ingress.flower.enabled .Values.ingress.enabled) (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }} +Flower Dashboard: +{{- range .Values.ingress.flower.hosts | default (list .Values.ingress.flower.host) }} + {{- $tlsEnabled := $.Values.ingress.flower.tls.enabled -}} + {{- $hostname := $.Values.ingress.flower.host -}} + {{- $path := $.Values.ingress.flower.path -}} + {{- if . | kindIs "string" | not }} + {{- $hostname = .name -}} + {{- if .tls }} + {{- $tlsEnabled = .tls.enabled -}} + {{- end }} + {{- end }} + http{{ if $tlsEnabled }}s{{ end }}://{{ (tpl $hostname $) }}{{ $path }}{{ if (hasSuffix "/" $path) | not }}/{{ end }} +{{- end }} +{{- end }} +{{- else }} +You can now access your dashboard(s) by executing the following command(s) and visiting the corresponding port at localhost in your browser: + +{{- if semverCompare "<3.0.0" .Values.airflowVersion }} +Airflow Webserver: kubectl port-forward svc/{{ include "airflow.fullname" . }}-webserver {{ .Values.ports.airflowUI }}:{{ .Values.ports.airflowUI }} --namespace {{ .Release.Namespace }} +{{- else }} +Airflow API Server: kubectl port-forward svc/{{ include "airflow.fullname" . }}-api-server {{ .Values.ports.airflowUI }}:{{ .Values.ports.airflowUI }} --namespace {{ .Release.Namespace }} +{{- end }} + +{{- if .Values.flower.enabled }} +{{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)}} +Flower Dashboard: kubectl port-forward svc/{{ include "airflow.fullname" . }}-flower {{ .Values.ports.flowerUI }}:{{ .Values.ports.flowerUI }} --namespace {{ .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{- if or .Values.ingress.apiServer.enabled .Values.ingress.enabled }} + +{{- if .Values.ingress.apiServer.host }} + +DEPRECATION WARNING: + `ingress.apiServer.host` has been renamed to `ingress.apiServer.hosts` and is now an array. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.ingress.apiServer.tls.enabled }} + +DEPRECATION WARNING: + `ingress.apiServer.tls` has been renamed to `ingress.apiServer.hosts[*].tls` and can be set per host. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- end }} + +{{- if and .Values.ingress.statsd.enabled .Values.ingress.statsd.host }} + +DEPRECATION WARNING: + `ingress.statsd.host` has been renamed to `ingress.statsd.hosts` and is now an array. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if and .Values.ingress.pgbouncer.enabled .Values.ingress.pgbouncer.host }} + +DEPRECATION WARNING: + `ingress.pgbouncer.host` has been renamed to `ingress.pgbouncer.hosts` and is now an array. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if eq (include "createUserJob.isEnabled" .) "true" }} +Default user (Airflow UI) Login credentials: + username: {{ if .Values.webserver.defaultUser }}{{ .Values.webserver.defaultUser.username }}{{ else }}{{ .Values.createUserJob.defaultUser.username }}{{ end }} + password: {{ if .Values.webserver.defaultUser }}{{ .Values.webserver.defaultUser.password }}{{ else }}{{ .Values.createUserJob.defaultUser.password }}{{ end }} +{{- end }} + +{{- if .Values.postgresql.enabled }} +Default Postgres connection credentials: + username: {{ .Values.data.metadataConnection.user }} + password: {{ .Values.data.metadataConnection.pass }} + port: {{ .Values.data.metadataConnection.port }} + +{{- end }} + +{{- if not .Values.fernetKeySecretName }} + +You can get Fernet Key value by running the following: + + echo Fernet Key: $(kubectl get secret --namespace {{ .Release.Namespace }} {{ include "airflow.fullname" . }}-fernet-key -o jsonpath="{.data.fernet-key}" | base64 --decode) + +{{- end }} + +{{- if or (contains "KubernetesExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }} +{{- if and (not .Values.logs.persistence.enabled) (eq (lower (tpl .Values.config.logging.remote_logging .)) "false") }} + +WARNING: + Kubernetes workers task logs may not persist unless you configure log persistence or remote logging! + Logging options can be found at: https://airflow.apache.org/docs/helm-chart/stable/manage-logs.html + (This warning can be ignored if logging is configured with environment variables or secrets backend) + +{{- end }} +{{- end }} + +{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret (not .Values.dags.gitSync.knownHosts)}} + +##################################################### +# WARNING: You should set dags.gitSync.knownHosts # +##################################################### + +You are using ssh authentication for your gitsync repo, however you currently have SSH known_hosts verification disabled, +making you susceptible to man-in-the-middle attacks! + +Information on how to set knownHosts can be found here: +https://airflow.apache.org/docs/helm-chart/stable/production-guide.html#knownhosts + +{{- end }} + +{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.readinessProbe }} + + DEPRECATION WARNING: + `dags.gitSync.readinessProbe` section has been removed as Git-Sync is not service-type object. + Please remove overwrite values of section, if defined, as support it will be removed in a future release. + +{{- end }} + +{{- if .Values.flower.extraNetworkPolicies }} + +DEPRECATION WARNING: + `flower.extraNetworkPolicies` has been renamed to `flower.networkPolicy.peers`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + + +{{- if .Values.webserver.extraNetworkPolicies }} + +DEPRECATION WARNING: + `webserver.extraNetworkPolicies` has been renamed to `webserver.networkPolicy.peers`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.securityContext }} + + DEPRECATION WARNING: + `securityContext` has been renamed to `securityContexts`, to be enabled on container and pod level. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.scheduler.securityContext }} + + DEPRECATION WARNING: + `scheduler.securityContext` has been renamed to `scheduler.securityContexts`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.webserver.securityContext }} + + DEPRECATION WARNING: + `webserver.securityContext` has been renamed to `webserver.securityContexts`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.statsd.securityContext }} + + DEPRECATION WARNING: + `statsd.securityContext` has been renamed to `statsd.securityContexts`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.replicas) 1 }} + + DEPRECATION WARNING: + `workers.replicas` has been renamed to `workers.celery.replicas`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.revisionHistoryLimit) }} + + DEPRECATION WARNING: + `workers.revisionHistoryLimit` has been renamed to `workers.celery.revisionHistoryLimit`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.command) }} + + DEPRECATION WARNING: + `workers.command` has been renamed to `workers.celery.command`/`workers.kubernetes.command`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (.Values.workers.args | toJson) (list "bash" "-c" "exec \\\nairflow celery worker\n{{- if and .Values.workers.queue (ne .Values.workers.queue \"default\") }}\n{{- \" -q \" }}{{ .Values.workers.queue }}\n{{- end }}" | toJson) }} + + DEPRECATION WARNING: + `workers.args` has been renamed to `workers.celery.args`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not .Values.workers.livenessProbe.enabled }} + + DEPRECATION WARNING: + `workers.livenessProbe.enabled` has been renamed to `workers.celery.livenessProbe.enabled`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.livenessProbe.initialDelaySeconds) 10 }} + + DEPRECATION WARNING: + `workers.livenessProbe.initialDelaySeconds` has been renamed to `workers.celery.livenessProbe.initialDelaySeconds`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.livenessProbe.timeoutSeconds) 20 }} + + DEPRECATION WARNING: + `workers.livenessProbe.timeoutSeconds` has been renamed to `workers.celery.livenessProbe.timeoutSeconds`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.livenessProbe.failureThreshold) 5 }} + + DEPRECATION WARNING: + `workers.livenessProbe.failureThreshold` has been renamed to `workers.celery.livenessProbe.failureThreshold`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.livenessProbe.periodSeconds) 60 }} + + DEPRECATION WARNING: + `workers.livenessProbe.periodSeconds` has been renamed to `workers.celery.livenessProbe.periodSeconds`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.livenessProbe.command) }} + + DEPRECATION WARNING: + `workers.livenessProbe.command` has been renamed to `workers.celery.livenessProbe.command`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.updateStrategy) }} + + DEPRECATION WARNING: + `workers.updateStrategy` has been renamed to `workers.celery.updateStrategy`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (toJson .Values.workers.strategy | quote) (toJson "{\"rollingUpdate\":{\"maxSurge\":\"100%\",\"maxUnavailable\":\"50%\"}}") }} + + DEPRECATION WARNING: + `workers.strategy` has been renamed to `workers.celery.strategy`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.podManagementPolicy) }} + + DEPRECATION WARNING: + `workers.podManagementPolicy` has been renamed to `workers.celery.podManagementPolicy`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.securityContext) }} + + DEPRECATION WARNING: + `workers.securityContext` has been renamed to `workers.celery.securityContexts`/`workers.kubernetes.securityContexts`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.securityContexts.pod) }} + + DEPRECATION WARNING: + `workers.securityContexts.pod` has been renamed to `workers.celery.securityContexts.pod`/`workers.kubernetes.securityContexts.pod`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.securityContexts.container) }} + + DEPRECATION WARNING: + `workers.securityContexts.container` has been renamed to `workers.celery.securityContexts.container`/`workers.kubernetes.securityContexts.container`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.containerLifecycleHooks) }} + + DEPRECATION WARNING: + `workers.containerLifecycleHooks` has been renamed to `workers.celery.containerLifecycleHooks`/`workers.kubernetes.containerLifecycleHooks`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.workers.podDisruptionBudget.enabled }} + + DEPRECATION WARNING: + `workers.podDisruptionBudget.enabled` has been renamed to `workers.celery.podDisruptionBudget.enabled`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.podDisruptionBudget.config.maxUnavailable) 1 }} + + DEPRECATION WARNING: + `workers.podDisruptionBudget.config.maxUnavailable` has been renamed to `workers.celery.podDisruptionBudget.config.maxUnavailable`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if hasKey .Values.workers.podDisruptionBudget.config "minAvailable" }} + + DEPRECATION WARNING: + `workers.podDisruptionBudget.config.minAvailable` has been renamed to `workers.celery.podDisruptionBudget.config.minAvailable`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not .Values.workers.serviceAccount.automountServiceAccountToken }} + + DEPRECATION WARNING: + `workers.serviceAccount.automountServiceAccountToken` has been renamed to `workers.celery.serviceAccount.automountServiceAccountToken`/`workers.kubernetes.serviceAccount.automountServiceAccountToken`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not .Values.workers.serviceAccount.create }} + + DEPRECATION WARNING: + `workers.serviceAccount.create` has been renamed to `workers.celery.serviceAccount.create`/`workers.kubernetes.serviceAccount.create`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.serviceAccount.name) }} + + DEPRECATION WARNING: + `workers.serviceAccount.name` has been renamed to `workers.celery.serviceAccount.name`/`workers.kubernetes.serviceAccount.name`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.serviceAccount.annotations) }} + + DEPRECATION WARNING: + `workers.serviceAccount.annotations` has been renamed to `workers.celery.serviceAccount.annotations`/`workers.kubernetes.serviceAccount.annotations`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.workers.keda.enabled }} + + DEPRECATION WARNING: + `workers.keda.enabled` has been renamed to `workers.celery.keda.enabled`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.keda.namespaceLabels) }} + + DEPRECATION WARNING: + `workers.keda.namespaceLabels` has been renamed to `workers.celery.keda.namespaceLabels`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.keda.pollingInterval) 5 }} + + DEPRECATION WARNING: + `workers.keda.pollingInterval` has been renamed to `workers.celery.keda.pollingInterval`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.keda.cooldownPeriod) 30 }} + + DEPRECATION WARNING: + `workers.keda.cooldownPeriod` has been renamed to `workers.celery.keda.cooldownPeriod`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.keda.minReplicaCount) 0 }} + + DEPRECATION WARNING: + `workers.keda.minReplicaCount` has been renamed to `workers.celery.keda.minReplicaCount`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.keda.maxReplicaCount) 10 }} + + DEPRECATION WARNING: + `workers.keda.maxReplicaCount` has been renamed to `workers.celery.keda.maxReplicaCount`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.keda.advanced) }} + + DEPRECATION WARNING: + `workers.keda.advanced` has been renamed to `workers.celery.keda.advanced`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (quote .Values.workers.keda.query) (quote "SELECT ceil(COUNT(*)::decimal / {{ .Values.config.celery.worker_concurrency }}) FROM task_instance WHERE (state='running' OR state='queued') AND queue IN ( {{- range $i, $q := splitList \",\" .Values.workers.queue -}} {{- if $i }},{{ end }}'{{ $q | trim }}' {{- end -}} ) {{- if contains \"CeleryKubernetesExecutor\" .Values.executor }} AND queue != '{{ .Values.config.celery_kubernetes_executor.kubernetes_queue }}' {{- else if contains \"KubernetesExecutor\" .Values.executor }} AND executor IS DISTINCT FROM 'KubernetesExecutor' {{- else if contains \"airflow.providers.edge3.executors.EdgeExecutor\" .Values.executor }} AND executor IS DISTINCT FROM 'EdgeExecutor' {{- end }}") }} + + DEPRECATION WARNING: + `workers.keda.query` has been renamed to `workers.celery.keda.query`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not .Values.workers.keda.usePgbouncer }} + + DEPRECATION WARNING: + `workers.keda.usePgbouncer` has been renamed to `workers.celery.keda.usePgbouncer`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.workers.hpa.enabled }} + + DEPRECATION WARNING: + `workers.hpa.enabled` has been renamed to `workers.celery.hpa.enabled`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.hpa.minReplicaCount) 0 }} + + DEPRECATION WARNING: + `workers.hpa.minReplicaCount` has been renamed to `workers.celery.hpa.minReplicaCount`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.hpa.maxReplicaCount) 5 }} + + DEPRECATION WARNING: + `workers.hpa.maxReplicaCount` has been renamed to `workers.celery.hpa.maxReplicaCount`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (toJson .Values.workers.hpa.metrics | quote) (toJson "[{\"resource\":{\"name\":\"cpu\",\"target\":{\"averageUtilization\":80,\"type\":\"Utilization\"}},\"type\":\"Resource\"}]") }} + + DEPRECATION WARNING: + `workers.hpa.metrics` has been renamed to `workers.celery.hpa.metrics`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.hpa.behavior) }} + + DEPRECATION WARNING: + `workers.hpa.behavior` has been renamed to `workers.celery.hpa.behavior`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not .Values.workers.persistence.enabled }} + + DEPRECATION WARNING: + `workers.persistence.enabled` has been renamed to `workers.celery.persistence.enabled`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.persistence.persistentVolumeClaimRetentionPolicy) }} + + DEPRECATION WARNING: + `workers.persistence.persistentVolumeClaimRetentionPolicy` has been renamed to `workers.celery.persistence.persistentVolumeClaimRetentionPolicy`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne .Values.workers.persistence.size "100Gi" }} + + DEPRECATION WARNING: + `workers.persistence.size` has been renamed to `workers.celery.persistence.size`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.persistence.storageClassName) }} + + DEPRECATION WARNING: + `workers.persistence.storageClassName` has been renamed to `workers.celery.persistence.storageClassName`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.workers.persistence.fixPermissions }} + + DEPRECATION WARNING: + `workers.persistence.fixPermissions` has been renamed to `workers.celery.persistence.fixPermissions`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.persistence.annotations) }} + + DEPRECATION WARNING: + `workers.persistence.annotations` has been renamed to `workers.celery.persistence.annotations`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.workers.persistence.securityContexts.container }} + + DEPRECATION WARNING: + `workers.persistence.securityContexts` has been renamed to `workers.celery.persistence.securityContexts`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.workers.kerberosSidecar.enabled }} + + DEPRECATION WARNING: + `workers.kerberosSidecar.enabled` has been renamed to `workers.celery.kerberosSidecar.enabled`/`workers.kubernetes.kerberosSidecar.enabled`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.workers.kerberosSidecar.resources }} + + DEPRECATION WARNING: + `workers.kerberosSidecar.resources` has been renamed to `workers.celery.kerberosSidecar.resources`/`workers.kubernetes.kerberosSidecar.resources`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.workers.kerberosSidecar.securityContexts.container }} + + DEPRECATION WARNING: + `workers.kerberosSidecar.securityContexts.container` has been renamed to `workers.celery.kerberosSidecar.securityContexts.container`/`workers.kubernetes.kerberosSidecar.securityContexts.container`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.workers.kerberosSidecar.containerLifecycleHooks }} + + DEPRECATION WARNING: + `workers.kerberosSidecar.containerLifecycleHooks` has been renamed to `workers.celery.kerberosSidecar.containerLifecycleHooks`/`workers.kubernetes.kerberosSidecar.containerLifecycleHooks`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.workers.kerberosInitContainer.enabled }} + + DEPRECATION WARNING: + `workers.kerberosInitContainer.enabled` has been renamed to `workers.celery.kerberosInitContainer.enabled`/`workers.kubernetes.kerberosInitContainer.enabled`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.workers.kerberosInitContainer.resources }} + + DEPRECATION WARNING: + `workers.kerberosInitContainer.resources` has been renamed to `workers.celery.kerberosInitContainer.resources`/`workers.kubernetes.kerberosInitContainer.resources`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.workers.kerberosInitContainer.securityContexts.container }} + + DEPRECATION WARNING: + `workers.kerberosInitContainer.securityContexts.container` has been renamed to `workers.celery.kerberosInitContainer.securityContexts.container`/`workers.kubernetes.kerberosInitContainer.securityContexts.container`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.workers.kerberosInitContainer.containerLifecycleHooks }} + + DEPRECATION WARNING: + `workers.kerberosInitContainer.containerLifecycleHooks` has been renamed to `workers.celery.kerberosInitContainer.containerLifecycleHooks`/`workers.kubernetes.kerberosInitContainer.containerLifecycleHooks`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.resources) }} + + DEPRECATION WARNING: + `workers.resources` has been renamed to `workers.celery.resources`/`workers.kubernetes.resources`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.terminationGracePeriodSeconds) 600 }} + + DEPRECATION WARNING: + `workers.terminationGracePeriodSeconds` has been renamed to `workers.celery.terminationGracePeriodSeconds`/`workers.kubernetes.terminationGracePeriodSeconds`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if .Values.workers.safeToEvict }} + + DEPRECATION WARNING: + `workers.safeToEvict` has been renamed to `workers.celery.safeToEvict`/`workers.kubernetes.safeToEvict`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.extraContainers) }} + + DEPRECATION WARNING: + `workers.extraContainers` has been renamed to `workers.celery.extraContainers`/`workers.kubernetes.extraContainers`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.extraInitContainers) }} + + DEPRECATION WARNING: + `workers.extraInitContainers` has been renamed to `workers.celery.extraInitContainers`/`workers.kubernetes.extraInitContainers`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.extraVolumes) }} + + DEPRECATION WARNING: + `workers.extraVolumes` has been renamed to `workers.celery.extraVolumes`/`workers.kubernetes.extraVolumes`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.extraVolumeMounts) }} + + DEPRECATION WARNING: + `workers.extraVolumeMounts` has been renamed to `workers.celery.extraVolumeMounts`/`workers.kubernetes.extraVolumeMounts`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.runtimeClassName) }} + + DEPRECATION WARNING: + `workers.runtimeClassName` has been renamed to `workers.celery.runtimeClassName`/`workers.kubernetes.runtimeClassName`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.priorityClassName) }} + + DEPRECATION WARNING: + `workers.priorityClassName` has been renamed to `workers.celery.priorityClassName`/`workers.kubernetes.priorityClassName`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.affinity) }} + + DEPRECATION WARNING: + `workers.affinity` has been renamed to `workers.celery.affinity`/`workers.kubernetes.affinity`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.tolerations) }} + + DEPRECATION WARNING: + `workers.tolerations` has been renamed to `workers.celery.tolerations`/`workers.kubernetes.tolerations`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.topologySpreadConstraints) }} + + DEPRECATION WARNING: + `workers.topologySpreadConstraints` has been renamed to `workers.celery.topologySpreadConstraints`/`workers.kubernetes.topologySpreadConstraints`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.nodeSelector) }} + + DEPRECATION WARNING: + `workers.nodeSelector` has been renamed to `workers.celery.nodeSelector`/`workers.kubernetes.nodeSelector`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.extraPorts) }} + + DEPRECATION WARNING: + `workers.extraPorts` has been renamed to `workers.celery.extraPorts`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.hostAliases) }} + + DEPRECATION WARNING: + `workers.hostAliases` has been renamed to `workers.celery.hostAliases`/`workers.kubernetes.hostAliases`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.annotations) }} + + DEPRECATION WARNING: + `workers.annotations` has been renamed to `workers.celery.annotations`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.podAnnotations) }} + + DEPRECATION WARNING: + `workers.podAnnotations` has been renamed to `workers.celery.podAnnotations`/`workers.kubernetes.podAnnotations`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.labels) }} + + DEPRECATION WARNING: + `workers.labels` has been renamed to `workers.celery.labels`/`workers.kubernetes.labels`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.volumeClaimTemplates) }} + + DEPRECATION WARNING: + `workers.volumeClaimTemplates` has been renamed to `workers.celery.volumeClaimTemplates`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.schedulerName) }} + + DEPRECATION WARNING: + `workers.schedulerName` has been renamed to `workers.celery.schedulerName`/`workers.kubernetes.schedulerName`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not .Values.workers.logGroomerSidecar.enabled }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.enabled` has been renamed to `workers.celery.logGroomerSidecar.enabled`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.logGroomerSidecar.command) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.command` has been renamed to `workers.celery.logGroomerSidecar.command`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (.Values.workers.logGroomerSidecar.args | toJson) (list "bash" "/clean-logs" | toJson) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.args` has been renamed to `workers.celery.logGroomerSidecar.args`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.logGroomerSidecar.retentionDays) 15 }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.retentionDays` has been renamed to `workers.celery.logGroomerSidecar.retentionDays`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.logGroomerSidecar.retentionMinutes) 0 }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.retentionMinutes` has been renamed to `workers.celery.logGroomerSidecar.retentionMinutes`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.logGroomerSidecar.frequencyMinutes) 15 }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.frequencyMinutes` has been renamed to `workers.celery.logGroomerSidecar.frequencyMinutes`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.logGroomerSidecar.maxSizeBytes) 0 }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.maxSizeBytes` has been renamed to `workers.celery.logGroomerSidecar.maxSizeBytes`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.logGroomerSidecar.maxSizePercent) 0 }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.maxSizePercent` has been renamed to `workers.celery.logGroomerSidecar.maxSizePercent`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.logGroomerSidecar.resources) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.resources` has been renamed to `workers.celery.logGroomerSidecar.resources`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.logGroomerSidecar.securityContexts.container) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.securityContexts.container` has been renamed to `workers.celery.logGroomerSidecar.securityContexts.container`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.logGroomerSidecar.env) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.env` has been renamed to `workers.celery.logGroomerSidecar.env`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.logGroomerSidecar.containerLifecycleHooks) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.containerLifecycleHooks` has been renamed to `workers.celery.logGroomerSidecar.containerLifecycleHooks`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not .Values.workers.waitForMigrations.enabled }} + + DEPRECATION WARNING: + `workers.waitForMigrations.enabled` has been renamed to `workers.celery.waitForMigrations.enabled`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.waitForMigrations.env) }} + + DEPRECATION WARNING: + `workers.waitForMigrations.env` has been renamed to `workers.celery.waitForMigrations.env`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.waitForMigrations.securityContexts.container) }} + + DEPRECATION WARNING: + `workers.waitForMigrations.securityContexts.container` has been renamed to `workers.celery.waitForMigrations.securityContexts.container`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.env) }} + + DEPRECATION WARNING: + `workers.env` has been renamed to `workers.celery.env`/`workers.kubernetes.env`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.webserver.defaultUser) }} + + DEPRECATION WARNING: + `webserver.defaultUser` has been renamed to `createUserJob.defaultUser`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not .Values.dags.gitSync.recommendedProbeSetting }} + + DEPRECATION WARNING: + Dags Git-Sync bevaiour with `dags.gitSync.recommendedProbeSetting` equal `false` is deprecated and will be removed in future. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (or .Values.webserverSecretKey .Values.webserverSecretKeySecretName .Values.apiSecretKey .Values.apiSecretKeySecretName) }} +{{ if (semverCompare ">=3.0.0" .Values.airflowVersion) }} +##################################################### +# WARNING: You should set a static API secret key # +##################################################### +{{ else }} +########################################################### +# WARNING: You should set a static webserver secret key # +########################################################### +{{ end }} +{{- $serverKind := ternary "API" "webserver" (semverCompare ">=3.0.0" .Values.airflowVersion) }} +You are using a dynamically generated {{ $serverKind }} secret key, which can lead to +unnecessary restarts of your Airflow components. + +Information on how to set a static {{ $serverKind }} secret key can be found here: +https://airflow.apache.org/docs/helm-chart/stable/production-guide.html#api-secret-key + +{{- end }} + +{{- if or .Values.postgresql.postgresqlUsername .Values.postgresql.postgresqlPassword }} + + {{ fail "postgresql.postgresqlUsername and postgresql.postgresqlPassword are no longer supported. If you wish to use the 'postgres' user, set its password with postgresql.auth.postgresPassword. If you wish to create a different user, do so with postgresql.auth.username and postgresql.auth.password." }} + +{{- end }} + +{{- if ne .Values.executor (tpl .Values.config.core.executor $) }} + {{ fail "Please configure the executor with `executor`, not `config.core.executor`." }} +{{- end }} diff --git a/charts/airflow/templates/_helpers.yaml b/charts/airflow/templates/_helpers.yaml new file mode 100644 index 0000000..53017f7 --- /dev/null +++ b/charts/airflow/templates/_helpers.yaml @@ -0,0 +1,1170 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airflow.fullname" -}} + {{- if not .Values.useStandardNaming }} + {{- .Release.Name }} + {{- else if .Values.fullnameOverride }} + {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} + {{- else }} + {{- $name := default .Chart.Name .Values.nameOverride }} + {{- if contains $name .Release.Name }} + {{- .Release.Name | trunc 63 | trimSuffix "-" }} + {{- else }} + {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} + {{- end }} + {{- end }} +{{- end }} + +{{- define "airflow.serviceAccountName" -}} + {{ if .Values.fullnameOverride }} + {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} + {{- else }} + {{- $name := default .Chart.Name .Values.nameOverride }} + {{- if contains $name .Release.Name }} + {{- .Release.Name | trunc 63 | trimSuffix "-" }} + {{- else }} + {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} + {{- end }} + {{- end }} +{{- end }} + +{{- define "airflow.tplDict" -}} + {{- $rendered := dict -}} + {{- range $key, $value := .values }} + {{- $_ := set $rendered $key (tpl (toString $value) $.context) -}} + {{- end }} + {{- toYaml $rendered -}} +{{- end }} + +{{/* Standard Airflow environment variables */}} +{{- define "standard_airflow_environment" }} + # Hard Coded Airflow Envs + - name: AIRFLOW_HOME + value: {{ .Values.airflowHome }} + {{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__CORE__FERNET_KEY }} + - name: AIRFLOW__CORE__FERNET_KEY + valueFrom: + secretKeyRef: + name: {{ template "fernet_key_secret" . }} + key: fernet-key + {{- end }} + {{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__DATABASE__SQL_ALCHEMY_CONN }} + - name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN + valueFrom: + secretKeyRef: + name: {{ template "airflow_metadata_secret" . }} + key: connection + {{- end }} + {{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW_CONN_AIRFLOW_DB }} + - name: AIRFLOW_CONN_AIRFLOW_DB + valueFrom: + secretKeyRef: + name: {{ template "airflow_metadata_secret" . }} + key: connection + {{- end }} + {{- $kedaEnabled := .Values.workers.keda.enabled }} + {{- $kedaUsePgBouncer := .Values.workers.keda.usePgbouncer }} + {{- if hasKey .Values.workers "celery" }} + {{- $kedaEnabled = or .Values.workers.celery.keda.enabled (and (not (has .Values.workers.celery.keda.enabled (list true false))) .Values.workers.keda.enabled) }} + {{- $kedaUsePgBouncer = or .Values.workers.celery.keda.usePgbouncer (and (not (has .Values.workers.celery.keda.usePgbouncer (list true false))) .Values.workers.keda.usePgbouncer) }} + {{- end }} + {{- if and $kedaEnabled (or (eq .Values.data.metadataConnection.protocol "mysql") (and .Values.pgbouncer.enabled (not $kedaUsePgBouncer))) }} + - name: KEDA_DB_CONN + valueFrom: + secretKeyRef: + name: {{ template "airflow_metadata_secret" . }} + key: kedaConnection + {{- end }} + {{- if and (semverCompare "<3.0.0" .Values.airflowVersion) .Values.enableBuiltInSecretEnvVars.AIRFLOW__WEBSERVER__SECRET_KEY }} + - name: AIRFLOW__WEBSERVER__SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ template "webserver_secret_key_secret" . }} + key: webserver-secret-key + {{- end }} + {{- if and (semverCompare ">=3.0.0" .Values.airflowVersion) .Values.enableBuiltInSecretEnvVars.AIRFLOW__API__SECRET_KEY }} + - name: AIRFLOW__API__SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ template "api_secret_key_secret" . }} + key: api-secret-key + {{- end }} + {{- if and .IncludeJwtSecret (semverCompare ">=3.0.0" .Values.airflowVersion) .Values.enableBuiltInSecretEnvVars.AIRFLOW__API_AUTH__JWT_SECRET }} + - name: AIRFLOW__API_AUTH__JWT_SECRET + valueFrom: + secretKeyRef: + name: {{ template "jwt_secret" . }} + key: jwt-secret + {{- end }} + {{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }} + {{- if and .Values.enableBuiltInSecretEnvVars.AIRFLOW__CELERY__RESULT_BACKEND (or .Values.data.resultBackendSecretName .Values.data.resultBackendConnection) }} + - name: AIRFLOW__CELERY__RESULT_BACKEND + valueFrom: + secretKeyRef: + name: {{ template "airflow_result_backend_secret" . }} + key: connection + {{- end }} + {{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__CELERY__BROKER_URL }} + - name: AIRFLOW__CELERY__BROKER_URL + valueFrom: + secretKeyRef: + name: {{ template "airflow_broker_url_secret" . }} + key: connection + {{- end }} + {{- end }} + {{- if and .Values.elasticsearch.enabled .Values.enableBuiltInSecretEnvVars.AIRFLOW__ELASTICSEARCH__HOST }} + - name: AIRFLOW__ELASTICSEARCH__HOST + valueFrom: + secretKeyRef: + name: {{ template "elasticsearch_secret" . }} + key: connection + {{- end }} + {{- if and .Values.opensearch.enabled .Values.enableBuiltInSecretEnvVars.AIRFLOW__OPENSEARCH__HOST }} + - name: AIRFLOW__OPENSEARCH__HOST + valueFrom: + secretKeyRef: + name: {{ template "opensearch_secret" . }} + key: connection + {{- end }} +{{- end }} + +{{/* User defined Airflow environment variables */}} +{{- define "custom_airflow_environment" }} + # Dynamically created environment variables + {{- range $i, $config := .Values.env }} + - name: {{ $config.name }} + value: {{ $config.value | quote }} + {{- end }} + # Dynamically created secret envs + {{- range $i, $config := .Values.secret }} + - name: {{ $config.envName }} + valueFrom: + secretKeyRef: + name: {{ $config.secretName }} + key: {{ default "value" $config.secretKey }} + {{- end }} + # Extra env + {{- $Global := . }} + {{- with .Values.extraEnv }} + {{- tpl . $Global | nindent 2 }} + {{- end }} +{{- end }} + +{{/* User defined Airflow environment from */}} +{{- define "custom_airflow_environment_from" }} + {{- $Global := . }} + {{- with .Values.extraEnvFrom }} + {{- tpl . $Global | nindent 2 }} + {{- end }} +{{- end }} + +{{/* User defined gitSync container environment from */}} +{{- define "custom_git_sync_environment_from" }} + {{- $Global := . }} + {{- with .Values.dags.gitSync.envFrom }} + {{- tpl . $Global | nindent 2 }} + {{- end }} +{{- end }} + +{{/* Git ssh key volume */}} +{{- define "git_sync_ssh_key_volume" }} +- name: git-sync-ssh-key + secret: + secretName: {{ template "git_sync_ssh_key" . }} + defaultMode: 288 +{{- end }} + +{{/* Git sync container */}} +{{- define "git_sync_container" }} +- name: {{ .Values.dags.gitSync.containerName }}{{ if .is_init }}-init{{ end }} + image: {{ template "git_sync_image" . }} + imagePullPolicy: {{ .Values.images.gitSync.pullPolicy }} + securityContext: {{- include "localContainerSecurityContext" .Values.dags.gitSync | nindent 4 }} + envFrom: {{- include "custom_git_sync_environment_from" . | default "\n []" | indent 2 }} + env: + - name: GIT_SYNC_REV + value: {{ .Values.dags.gitSync.rev | quote }} + - name: GITSYNC_REF + value: {{ .Values.dags.gitSync.ref | quote }} + - name: GIT_SYNC_BRANCH + value: {{ .Values.dags.gitSync.branch | quote }} + - name: GIT_SYNC_REPO + value: {{ .Values.dags.gitSync.repo | quote }} + - name: GITSYNC_REPO + value: {{ .Values.dags.gitSync.repo | quote }} + - name: GIT_SYNC_DEPTH + value: {{ .Values.dags.gitSync.depth | quote }} + - name: GITSYNC_DEPTH + value: {{ .Values.dags.gitSync.depth | quote }} + - name: GIT_SYNC_ROOT + value: "/git" + - name: GITSYNC_ROOT + value: "/git" + - name: GIT_SYNC_DEST + value: "repo" + - name: GITSYNC_LINK + value: "repo" + - name: GIT_SYNC_ADD_USER + value: "true" + - name: GITSYNC_ADD_USER + value: "true" + - name: GITSYNC_PERIOD + value: {{ .Values.dags.gitSync.period | quote }} + - name: GIT_SYNC_MAX_SYNC_FAILURES + value: {{ .Values.dags.gitSync.maxFailures | quote }} + - name: GITSYNC_MAX_FAILURES + value: {{ .Values.dags.gitSync.maxFailures | quote }} + {{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey }} + - name: GIT_SSH_KEY_FILE + value: "/etc/git-secret/ssh" + - name: GITSYNC_SSH_KEY_FILE + value: "/etc/git-secret/ssh" + - name: GIT_SYNC_SSH + value: "true" + - name: GITSYNC_SSH + value: "true" + {{- if .Values.dags.gitSync.knownHosts }} + - name: GIT_KNOWN_HOSTS + value: "true" + - name: GITSYNC_SSH_KNOWN_HOSTS + value: "true" + - name: GIT_SSH_KNOWN_HOSTS_FILE + value: "/etc/git-secret/known_hosts" + - name: GITSYNC_SSH_KNOWN_HOSTS_FILE + value: "/etc/git-secret/known_hosts" + {{- else }} + - name: GIT_KNOWN_HOSTS + value: "false" + - name: GITSYNC_SSH_KNOWN_HOSTS + value: "false" + {{- end }} + {{ else if .Values.dags.gitSync.credentialsSecret }} + - name: GIT_SYNC_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.dags.gitSync.credentialsSecret | quote }} + key: GIT_SYNC_USERNAME + - name: GITSYNC_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.dags.gitSync.credentialsSecret | quote }} + key: GITSYNC_USERNAME + - name: GIT_SYNC_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.dags.gitSync.credentialsSecret | quote }} + key: GIT_SYNC_PASSWORD + - name: GITSYNC_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.dags.gitSync.credentialsSecret | quote }} + key: GITSYNC_PASSWORD + {{- end }} + {{- if .Values.dags.gitSync.wait }} + - name: GIT_SYNC_WAIT + value: {{ .Values.dags.gitSync.wait | quote }} + {{- end }} + {{- if .is_init }} + - name: GIT_SYNC_ONE_TIME + value: "true" + - name: GITSYNC_ONE_TIME + value: "true" + {{- else }} + - name: GIT_SYNC_HTTP_BIND + value: ":{{ .Values.dags.gitSync.httpPort }}" + - name: GITSYNC_HTTP_BIND + value: ":{{ .Values.dags.gitSync.httpPort }}" + {{- end }} + {{- with .Values.dags.gitSync.env }} + {{- toYaml . | nindent 4 }} + {{- end }} + resources: {{ toYaml .Values.dags.gitSync.resources | nindent 4 }} + {{- if not .is_init }} + {{- if .Values.dags.gitSync.startupProbe.enabled }} + startupProbe: + httpGet: + path: / + port: {{ .Values.dags.gitSync.httpPort }} + timeoutSeconds: {{ .Values.dags.gitSync.startupProbe.timeoutSeconds }} + initialDelaySeconds: {{ .Values.dags.gitSync.startupProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.dags.gitSync.startupProbe.periodSeconds }} + failureThreshold: {{ .Values.dags.gitSync.startupProbe.failureThreshold }} + {{- end }} + {{- if and .Values.dags.gitSync.recommendedProbeSetting (hasKey .Values.dags.gitSync.livenessProbe "enabled") .Values.dags.gitSync.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: / + port: {{ .Values.dags.gitSync.httpPort }} + timeoutSeconds: {{ .Values.dags.gitSync.livenessProbe.timeoutSeconds | default 1 }} + initialDelaySeconds: {{ .Values.dags.gitSync.livenessProbe.initialDelaySeconds | default 0 }} + periodSeconds: {{ .Values.dags.gitSync.livenessProbe.periodSeconds | default 5 }} + failureThreshold: {{ .Values.dags.gitSync.livenessProbe.failureThreshold | default 10 }} + {{- else if .Values.dags.gitSync.livenessProbe }} + livenessProbe: {{ tpl (toYaml .Values.dags.gitSync.livenessProbe) . | nindent 4 }} + {{- end }} + {{- if and .Values.dags.gitSync.readinessProbe (not .Values.dags.gitSync.recommendedProbeSetting) }} + readinessProbe: {{ tpl (toYaml .Values.dags.gitSync.readinessProbe) . | nindent 4 }} + {{- end }} + {{- end }} + volumeMounts: + - name: dags + mountPath: /git + {{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey }} + - name: git-sync-ssh-key + mountPath: /etc/git-secret/ssh + readOnly: true + subPath: gitSshKey + {{- if .Values.dags.gitSync.knownHosts }} + - name: config + mountPath: /etc/git-secret/known_hosts + readOnly: true + subPath: known_hosts + {{- end }} + {{- end }} + {{- if .Values.dags.gitSync.extraVolumeMounts }} + {{- tpl (toYaml .Values.dags.gitSync.extraVolumeMounts) . | nindent 2 }} + {{- end }} + {{- if and .Values.dags.gitSync.containerLifecycleHooks (not .is_init) }} + lifecycle: {{- tpl (toYaml .Values.dags.gitSync.containerLifecycleHooks) . | nindent 4 }} + {{- end }} +{{- end }} + +{{/* This helper will change when customers deploy a new image */}} +{{- define "airflow_image" -}} + {{- $repository := .Values.images.airflow.repository | default .Values.defaultAirflowRepository -}} + {{- $tag := .Values.images.airflow.tag | default .Values.defaultAirflowTag -}} + {{- $digest := .Values.images.airflow.digest | default .Values.defaultAirflowDigest -}} + {{- if $digest }} + {{- printf "%s@%s" $repository $digest -}} + {{- else }} + {{- printf "%s:%s" $repository $tag -}} + {{- end }} +{{- end }} + +{{- define "pod_template_image" -}} + {{- printf "%s:%s" (.Values.images.pod_template.repository | default .Values.defaultAirflowRepository) (.Values.images.pod_template.tag | default .Values.defaultAirflowTag) }} +{{- end }} + +{{/* This helper is used for airflow containers that do not need the users code */}} +{{ define "default_airflow_image" -}} + {{- $repository := .Values.defaultAirflowRepository -}} + {{- $tag := .Values.defaultAirflowTag -}} + {{- $digest := .Values.defaultAirflowDigest -}} + {{- if $digest }} + {{- printf "%s@%s" $repository $digest -}} + {{- else }} + {{- printf "%s:%s" $repository $tag -}} + {{- end }} +{{- end }} + +{{ define "airflow_image_for_migrations" -}} + {{- if .Values.images.useDefaultImageForMigration }} + {{- template "default_airflow_image" . }} + {{- else }} + {{- template "airflow_image" . }} + {{- end }} +{{- end }} + +{{- define "flower_image" -}} + {{- printf "%s:%s" (.Values.images.flower.repository | default .Values.defaultAirflowRepository) (.Values.images.flower.tag | default .Values.defaultAirflowTag) }} +{{- end }} + +{{- define "statsd_image" -}} + {{- printf "%s:%s" .Values.images.statsd.repository .Values.images.statsd.tag }} +{{- end }} + +{{- define "redis_image" -}} + {{- printf "%s:%s" .Values.images.redis.repository .Values.images.redis.tag }} +{{- end }} + +{{- define "pgbouncer_image" -}} + {{- printf "%s:%s" .Values.images.pgbouncer.repository .Values.images.pgbouncer.tag }} +{{- end }} + +{{- define "pgbouncer_exporter_image" -}} + {{- printf "%s:%s" .Values.images.pgbouncerExporter.repository .Values.images.pgbouncerExporter.tag }} +{{- end }} + +{{- define "git_sync_image" -}} + {{- printf "%s:%s" .Values.images.gitSync.repository .Values.images.gitSync.tag }} +{{- end }} + +{{- define "fernet_key_secret" -}} + {{- default (printf "%s-fernet-key" (include "airflow.fullname" .)) .Values.fernetKeySecretName }} +{{- end }} + +{{- define "jwt_secret" -}} + {{- default (printf "%s-jwt-secret" (include "airflow.fullname" .)) .Values.jwtSecretName }} +{{- end }} + +{{- define "webserver_secret_key_secret" -}} + {{- default (printf "%s-webserver-secret-key" (include "airflow.fullname" .)) .Values.webserverSecretKeySecretName }} +{{- end }} + +{{- define "api_secret_key_secret" -}} + {{- default (printf "%s-api-secret-key" (include "airflow.fullname" .)) .Values.apiSecretKeySecretName }} +{{- end }} + +{{- define "redis_password_secret" -}} + {{- default (printf "%s-redis-password" (include "airflow.fullname" .)) .Values.redis.passwordSecretName }} +{{- end }} + +{{- define "airflow_broker_url_secret" -}} + {{- default (printf "%s-broker-url" (include "airflow.fullname" .)) .Values.data.brokerUrlSecretName }} +{{- end }} + +{{- define "airflow_metadata_secret" -}} + {{- default (printf "%s-metadata" (include "airflow.fullname" .)) .Values.data.metadataSecretName }} +{{- end }} + +{{- define "airflow_result_backend_secret" -}} + {{- default (printf "%s-result-backend" (include "airflow.fullname" .)) .Values.data.resultBackendSecretName }} +{{- end }} + +{{- define "airflow_pod_template_file" -}} + {{- printf "%s/pod_templates" .Values.airflowHome }} +{{- end }} + +{{- define "pgbouncer_config_secret" -}} + {{- default (printf "%s-pgbouncer-config" (include "airflow.fullname" .)) .Values.pgbouncer.configSecretName }} +{{- end }} + +{{- define "pgbouncer_certificates_secret" -}} + {{- printf "%s-pgbouncer-certificates" (include "airflow.fullname" .) }} +{{- end }} + +{{- define "pgbouncer_stats_secret" -}} + {{- default (printf "%s-pgbouncer-stats" (include "airflow.fullname" .)) .Values.pgbouncer.metricsExporterSidecar.statsSecretName }} +{{- end }} + +{{- define "image_pull_secrets" -}} + {{- $secrets := default (list .Values.registry.secretName) .Values.imagePullSecrets -}} + {{- $secrets = ($secrets | compact | uniq) -}} + {{- if and (not $secrets) .Values.registry.connection -}} + {{- $secrets = append $secrets (printf "%s-registry" (include "airflow.fullname" .)) -}} + {{- end -}} + {{- $out := list -}} + {{- range $n := $secrets }} + {{- if kindIs "string" $n }} + {{- $n = dict "name" $n -}} + {{- end -}} + {{- $out = append $out (dict "name" $n.name) -}} + {{- end -}} + {{- toYaml $out -}} +{{- end }} + +{{- define "elasticsearch_secret" -}} + {{- default (printf "%s-elasticsearch" (include "airflow.fullname" .)) .Values.elasticsearch.secretName }} +{{- end }} + +{{- define "opensearch_secret" -}} + {{- default (printf "%s-opensearch" (include "airflow.fullname" .)) .Values.opensearch.secretName }} +{{- end }} + +{{- define "flower_secret" -}} + {{- default (printf "%s-flower" (include "airflow.fullname" .)) .Values.flower.secretName }} +{{- end }} + +{{- define "kerberos_keytab_secret" -}} + {{- printf "%s-kerberos-keytab" (include "airflow.fullname" .) }} +{{- end }} + +{{- define "kerberos_ccache_path" -}} + {{- printf "%s/%s" .Values.kerberos.ccacheMountPath .Values.kerberos.ccacheFileName }} +{{- end }} + +{{/* Create the name of the git sync ssh secret to use */}} +{{- define "git_sync_ssh_key" -}} + {{- default (printf "%s-ssh-secret" (include "airflow.fullname" .)) .Values.dags.gitSync.sshKeySecret }} +{{- end }} + +{{- define "celery_executor_namespace" -}} + {{- print "airflow.providers.celery.executors.celery_executor.app" -}} +{{- end }} + +{{- define "pgbouncer_config" -}} +{{ $resultBackendConnection := .Values.data.resultBackendConnection | default .Values.data.metadataConnection }} +{{ $pgMetadataHost := .Values.data.metadataConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }} +{{ $pgResultBackendHost := $resultBackendConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }} +[databases] +{{ .Release.Name }}-metadata = host={{ $pgMetadataHost }} dbname={{ tpl .Values.data.metadataConnection.db . }} port={{ .Values.data.metadataConnection.port }} pool_size={{ .Values.pgbouncer.metadataPoolSize }} {{ .Values.pgbouncer.extraIniMetadata | default "" }} +{{ .Release.Name }}-result-backend = host={{ $pgResultBackendHost }} dbname={{ tpl $resultBackendConnection.db . }} port={{ $resultBackendConnection.port }} pool_size={{ .Values.pgbouncer.resultBackendPoolSize }} {{ .Values.pgbouncer.extraIniResultBackend | default "" }} + +[pgbouncer] +pool_mode = transaction +listen_port = {{ .Values.ports.pgbouncer }} +listen_addr = * +auth_type = {{ .Values.pgbouncer.auth_type }} +auth_file = {{ .Values.pgbouncer.auth_file }} +stats_users = {{ tpl .Values.data.metadataConnection.user . }} +ignore_startup_parameters = extra_float_digits +max_client_conn = {{ .Values.pgbouncer.maxClientConn }} +verbose = {{ .Values.pgbouncer.verbose }} +log_disconnections = {{ .Values.pgbouncer.logDisconnections }} +log_connections = {{ .Values.pgbouncer.logConnections }} + +server_tls_sslmode = {{ .Values.pgbouncer.sslmode }} +server_tls_ciphers = {{ .Values.pgbouncer.ciphers }} + +{{- if .Values.pgbouncer.ssl.ca }} +server_tls_ca_file = /etc/pgbouncer/root.crt +{{- end }} +{{- if .Values.pgbouncer.ssl.cert }} +server_tls_cert_file = /etc/pgbouncer/server.crt +{{- end }} +{{- if .Values.pgbouncer.ssl.key }} +server_tls_key_file = /etc/pgbouncer/server.key +{{- end }} + +{{- if .Values.pgbouncer.extraIni }} +{{ .Values.pgbouncer.extraIni }} +{{- end }} +{{- end }} + +{{ define "pgbouncer_users" }} +{{- $resultBackendConnection := .Values.data.resultBackendConnection | default .Values.data.metadataConnection }} +{{ tpl .Values.data.metadataConnection.user . | quote }} {{ .Values.data.metadataConnection.pass | quote }} +{{ tpl $resultBackendConnection.user . | quote }} {{ $resultBackendConnection.pass | quote }} +{{- end }} + +{{- define "airflow_logs" -}} + {{- printf "%s/logs" .Values.airflowHome | quote }} +{{- end }} + +{{- define "airflow_logs_no_quote" -}} + {{- printf "%s/logs" .Values.airflowHome }} +{{- end }} + +{{- define "airflow_logs_volume_claim" -}} + {{- if .Values.logs.persistence.existingClaim }} + {{- .Values.logs.persistence.existingClaim }} + {{- else }} + {{- printf "%s-logs" .Release.Name }} + {{- end }} +{{- end }} + +{{- define "airflow_dags" -}} + {{- if .Values.dags.mountPath }} + {{- if .Values.dags.gitSync.enabled }} + {{- printf "%s/repo/%s" .Values.dags.mountPath .Values.dags.gitSync.subPath }} + {{- else }} + {{- printf "%s" .Values.dags.mountPath }} + {{- end }} + {{- else }} + {{- if .Values.dags.gitSync.enabled }} + {{- printf "%s/dags/repo/%s" .Values.airflowHome .Values.dags.gitSync.subPath }} + {{- else }} + {{- printf "%s/dags" .Values.airflowHome }} + {{- end }} + {{- end }} +{{- end }} + +{{- define "airflow_dags_volume_claim" -}} + {{- if .Values.dags.persistence.existingClaim }} + {{- .Values.dags.persistence.existingClaim }} + {{- else }} + {{- printf "%s-dags" .Release.Name }} + {{- end }} +{{- end }} + +{{- define "airflow_dags_mount" -}} +- name: dags + {{- if .Values.dags.mountPath }} + mountPath: {{ .Values.dags.mountPath }} + {{- else }} + mountPath: {{ printf "%s/dags" .Values.airflowHome }} + {{- end }} + {{- if .Values.dags.persistence.subPath }} + subPath: {{ .Values.dags.persistence.subPath }} + {{- end }} + readOnly: {{ .Values.dags.gitSync.enabled | ternary "True" "False" }} +{{- end }} + +{{- define "airflow_config_path" -}} + {{- printf "%s/airflow.cfg" .Values.airflowHome | quote }} +{{- end }} + +{{- define "airflow_webserver_config_path" -}} + {{- printf "%s/webserver_config.py" .Values.airflowHome | quote }} +{{- end }} + +{{- define "airflow_webserver_config_configmap_name" -}} + {{- if .Values.webserver.webserverConfigConfigMapName }} + {{- tpl .Values.webserver.webserverConfigConfigMapName . }} + {{- else }} + {{- printf "%s-webserver-config" (include "airflow.fullname" .) }} + {{- end }} +{{- end }} + +{{- define "airflow_webserver_config_mount" -}} +- name: webserver-config + mountPath: {{ template "airflow_webserver_config_path" . }} + subPath: webserver_config.py + readOnly: True +{{- end }} + +{{- define "airflow_api_server_config_configmap_name" -}} + {{- if .Values.apiServer.apiServerConfigConfigMapName }} + {{- tpl .Values.apiServer.apiServerConfigConfigMapName . }} + {{- else }} + {{- printf "%s-api-server-config" (include "airflow.fullname" .) }} + {{- end }} +{{- end }} + +{{- define "airflow_api_server_config_mount" -}} +- name: api-server-config + mountPath: {{ template "airflow_webserver_config_path" . }} + subPath: webserver_config.py + readOnly: True +{{- end }} + +{{- define "airflow_local_setting_path" -}} + {{- printf "%s/config/airflow_local_settings.py" .Values.airflowHome | quote }} +{{- end }} + +{{- define "airflow_config" -}} + {{- printf "%s-config" (include "airflow.fullname" .) }} +{{- end }} + +{{- define "airflow_config_mount" -}} +- name: config + mountPath: {{ template "airflow_config_path" . }} + subPath: airflow.cfg + readOnly: true + {{- if .Values.airflowLocalSettings }} +- name: config + mountPath: {{ template "airflow_local_setting_path" . }} + subPath: airflow_local_settings.py + readOnly: true + {{- end }} +{{- end }} + +{{/* Helper for service account name generation */}} +{{- define "_serviceAccountNameGen" -}} + {{- if .sa.create }} + {{- default (printf "%s-%s" (include "airflow.serviceAccountName" .) (default .key .nameSuffix)) .sa.name | quote }} + {{- else }} + {{- default "default" .sa.name | quote }} + {{- end }} +{{- end }} + +{{/* Helper to generate service account name respecting .Values.$section.serviceAccount or .Values.$section.$subSection.serviceAccount flags */}} +{{- define "_serviceAccountName" -}} + {{- if .subKey }} + {{- $sa := get (get (get .Values .key) .subKey) "serviceAccount" -}} + {{- include "_serviceAccountNameGen" (merge (dict "sa" $sa "key" .key "nameSuffix" .nameSuffix) .) }} + {{- else }} + {{- $sa := get (get .Values .key) "serviceAccount" }} + {{- include "_serviceAccountNameGen" (merge (dict "sa" $sa "key" .key "nameSuffix" .nameSuffix) .) }} + {{- end }} +{{- end }} + +{{/* Create the name of the webserver service account to use */}} +{{- define "webserver.serviceAccountName" -}} + {{- include "_serviceAccountName" (merge (dict "key" "webserver") .) -}} +{{- end }} + +{{/* Create the name of the API server service account to use */}} +{{- define "apiServer.serviceAccountName" -}} + {{- include "_serviceAccountName" (merge (dict "key" "apiServer" "nameSuffix" "api-server" ) .) -}} +{{- end }} + +{{/* Create the name of the redis service account to use */}} +{{- define "redis.serviceAccountName" -}} + {{- include "_serviceAccountName" (merge (dict "key" "redis") .) -}} +{{- end }} + +{{/* Create the name of the flower service account to use */}} +{{- define "flower.serviceAccountName" -}} + {{- include "_serviceAccountName" (merge (dict "key" "flower") .) -}} +{{- end }} + +{{/* Create the name of the scheduler service account to use */}} +{{- define "scheduler.serviceAccountName" -}} + {{- include "_serviceAccountName" (merge (dict "key" "scheduler") .) -}} +{{- end }} + +{{/* Create the name of the StatsD service account to use */}} +{{- define "statsd.serviceAccountName" -}} + {{- include "_serviceAccountName" (merge (dict "key" "statsd") .) -}} +{{- end }} + +{{/* Create the name of the create user job service account to use */}} +{{- define "createUserJob.serviceAccountName" -}} + {{- include "_serviceAccountName" (merge (dict "key" "createUserJob" "nameSuffix" "create-user-job") .) -}} +{{- end }} + +{{/* Create the name of the migrate database job service account to use */}} +{{- define "migrateDatabaseJob.serviceAccountName" -}} + {{- include "_serviceAccountName" (merge (dict "key" "migrateDatabaseJob" "nameSuffix" "migrate-database-job") .) -}} +{{- end }} + +{{/* Create the name of the worker service account to use */}} +{{- define "worker.serviceAccountName" -}} + {{- if and (hasKey .Values.workers "name") (ne .Values.workers.name "default") }} + {{- include "_serviceAccountName" (merge (dict "key" "workers" "nameSuffix" (printf "%s-%s" "worker" .Values.workers.name)) .) -}} + {{- else }} + {{- include "_serviceAccountName" (merge (dict "key" "workers" "nameSuffix" "worker") .) -}} + {{- end }} +{{- end }} + +{{/* Create the name of the worker kubernetes service account to use */}} +{{- define "worker.kubernetes.serviceAccountName" -}} + {{- include "_serviceAccountName" (merge (dict "key" "workers" "subKey" "kubernetes" "nameSuffix" "worker-kubernetes") .) -}} +{{- end }} + +{{/* Create the name of the triggerer service account to use */}} +{{- define "triggerer.serviceAccountName" -}} + {{- include "_serviceAccountName" (merge (dict "key" "triggerer") .) -}} +{{- end }} + +{{/* Determine trigger capacity, taking Airflow 2 and 3 config option differences into account */}} +{{- define "triggerer.capacity" -}} + {{- $triggerer_section := .Values.config.triggerer | default dict }} + {{- $triggerer_section.capacity | default $triggerer_section.default_capacity | default 1000 | int -}} +{{- end -}} + +{{/* Create the name of the dag processor service account to use */}} +{{- define "dagProcessor.serviceAccountName" -}} + {{- include "_serviceAccountName" (merge (dict "key" "dagProcessor" "nameSuffix" "dag-processor") .) -}} +{{- end }} + +{{/* Create the name of the pgbouncer service account to use */}} +{{- define "pgbouncer.serviceAccountName" -}} + {{- include "_serviceAccountName" (merge (dict "key" "pgbouncer") .) -}} +{{- end }} + +{{/* Create the name of the cleanup service account to use */}} +{{- define "cleanup.serviceAccountName" -}} + {{- include "_serviceAccountName" (merge (dict "key" "cleanup") .) -}} +{{- end }} + +{{/* Create the name of the database cleanup service account to use */}} +{{- define "databaseCleanup.serviceAccountName" -}} + {{- include "_serviceAccountName" (merge (dict "key" "databaseCleanup" "nameSuffix" "database-cleanup") .) -}} +{{- end }} + +{{- define "wait-for-migrations-command" }} + - airflow + - db + - check-migrations + - --migration-wait-timeout={{ .Values.images.migrationsWaitTimeout }} +{{- end }} + +{{- define "scheduler_liveness_check_command" }} + - sh + - -c + - | + CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \ + airflow jobs check --job-type SchedulerJob --local +{{- end }} + + +{{- define "scheduler_startup_check_command" }} + - sh + - -c + - | + CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \ + airflow jobs check --job-type SchedulerJob --local +{{- end }} + +{{- define "triggerer_liveness_check_command" }} + - sh + - -c + - | + CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \ + airflow jobs check --job-type TriggererJob --local +{{- end }} + +{{- define "dag_processor_liveness_check_command" }} + - sh + - -c + - | + CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \ + airflow jobs check --local --job-type DagProcessorJob +{{- end }} + +{{- define "registry_docker_config" }} + {{- $user := .Values.registry.connection.user }} + {{- $pass := .Values.registry.connection.pass }} + + {{- $config := dict "auths" }} + {{- $auth := dict }} + {{- $data := dict }} + {{- $_ := set $data "username" $user }} + {{- $_ := set $data "password" $pass }} + {{- $_ := set $data "email" .Values.registry.connection.email }} + {{- $_ := set $data "auth" (printf "%v:%v" $user $pass | b64enc) }} + {{- $_ := set $auth .Values.registry.connection.host $data }} + {{- $_ := set $config "auths" $auth }} + {{ $config | toJson | print }} +{{- end }} + +{{/* +Set the default value for pod securityContext +If no value is passed for securityContexts.pod or .securityContexts.pod or legacy securityContext and .securityContext, defaults to global uid and gid. + + +-----------------------------+ +------------------------+ +----------------------+ +-----------------+ +-------------------------+ + | .securityContexts.pod | -> | .securityContext | -> | securityContexts.pod | -> | securityContext | -> | Values.uid + Values.gid | + +-----------------------------+ +------------------------+ +----------------------+ +-----------------+ +-------------------------+ + +Values are not accumulated meaning that if runAsUser is set to 10 in .securityContexts.pod, +any extra values set to securityContext or uid+gid will be ignored. + +The template can be called like so: + include "airflowPodSecurityContext" (list .Values.webserver .Values) + +Where `.Values` is the global variables scope and `.Values.webserver` the local variables scope for the webserver template. +Priority of values are from left to right, meaning if first value is not empty, the rest will not be evaluated. +*/}} +{{- define "airflowPodSecurityContext" }} + {{- $ := last . }} + {{- $result := dict }} + {{- range . }} + {{- if and (hasKey . "securityContexts") (hasKey .securityContexts "pod") .securityContexts.pod }} + {{- $result = .securityContexts.pod }} + {{- break }} + {{- end }} + {{- if and (hasKey . "securityContext") .securityContext }} + {{- $result = .securityContext }} + {{- break }} + {{- end }} + {{- end }} + {{- if $result }} + {{- toYaml $result | print }} + {{- else }} +runAsUser: {{ $.uid }} +fsGroup: {{ $.gid }} + {{- end }} +{{- end }} + +{{/* +Set the default value for pod securityContext +If no value is passed for .securityContexts.pod or .securityContext, defaults to UID in the local node. + + +-----------------------------+ +------------------------+ +------------+ + | .securityContexts.pod | -> | .securityContext | -> | .uid | + +-----------------------------+ +------------------------+ +------------+ + +The template can be called like so: + include "localPodSecurityContext" (list . .Values.schedule) + +It is important to pass the local variables scope to this template as it is used to determine the local node value for uid. +*/}} +{{- define "localPodSecurityContext" -}} + {{- if .securityContexts.pod -}} + {{ toYaml .securityContexts.pod | print }} + {{- else if .securityContext -}} + {{ toYaml .securityContext | print }} + {{- else -}} +runAsUser: {{ .uid }} + {{- end -}} +{{- end -}} + +{{/* +Set the default value for container securityContext +If no value is passed for .securityContexts.container or .securityContext, defaults to UID in the local node. + + +-----------------------------------+ +------------------------+ +------------+ + | .securityContexts.container | -> | .securityContext | -> | .uid | + +-----------------------------------+ +------------------------+ +------------+ + +The template can be called like so: + include "localContainerSecurityContext" .Values.statsd + +It is important to pass the local variables scope to this template as it is used to determine the local node value for uid. +*/}} +{{- define "localContainerSecurityContext" -}} + {{- if .securityContexts.container -}} + {{ toYaml .securityContexts.container | print }} + {{- else if .securityContext -}} + {{ toYaml .securityContext | print }} + {{- else -}} +runAsUser: {{ .uid }} + {{- end -}} +{{- end -}} + +{{/* +Set the default value for workers chown for persistent storage +If no value is passed for securityContexts.pod or .securityContexts.pod or legacy securityContext and .securityContext, defaults to global uid and gid. +The template looks for `runAsUser` and `fsGroup` specifically, any other parameter will be ignored. + + +-----------------------------+ +----------------------------------------------------+ +------------------+ +-------------------------+ + | .securityContexts.pod | -> | securityContexts.pod | .securityContexts.pod | -> | securityContexts | -> | Values.uid + Values.gid | + +-----------------------------+ +----------------------------------------------------+ +------------------+ +-------------------------+ + +Values are not accumulated meaning that if runAsUser is set to 10 in .securityContexts.pod, +any extra values set to securityContexts or uid+gid will be ignored. + +The template can be called like so: + include "airflowPodSecurityContextsIds" (list . .Values.webserver) + +Where `.` is the global variables scope and `.Values.workers` the local variables scope for the workers template. +*/}} +{{- define "airflowPodSecurityContextsIds" -}} + {{- $ := index . 0 -}} + {{- with index . 1 }} + {{- if .securityContexts.pod -}} + {{ pluck "runAsUser" .securityContexts.pod | first | default $.Values.uid }}:{{ pluck "fsGroup" .securityContexts.pod | first | default $.Values.gid }} + {{- else if $.Values.securityContext -}} + {{ pluck "runAsUser" $.Values.securityContext | first | default $.Values.uid }}:{{ pluck "fsGroup" $.Values.securityContext | first | default $.Values.gid }} + {{- else if $.Values.securityContexts.pod -}} + {{ pluck "runAsUser" $.Values.securityContexts.pod | first | default $.Values.uid }}:{{ pluck "fsGroup" $.Values.securityContexts.pod | first | default $.Values.gid }} + {{- else -}} +{{ $.Values.uid }}:{{ $.Values.gid }} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Set the default value for container securityContext +If no value is passed for securityContexts.container or .securityContexts.container, defaults to deny privileges escalation and dropping all POSIX capabilities. + + +-----------------------------------+ +-----------------------------+ +------------------------------------------------------------+ + | .securityContexts.container | -> | securityContexts.containers | -> | allowPrivilegesEscalation: false, capabilities.drop: [ALL] | + +-----------------------------------+ +-----------------------------+ +------------------------------------------------------------+ + +The template can be called like so: + include "containerSecurityContext" (list .Values.webserver .Values) + +Where `.Values` is the global variables scope and `.Values.webserver` the local variables scope for the webserver template. +Priority of values are from left to right, meaning if first value is not empty, the rest will not be evaluated. +*/}} +{{- define "containerSecurityContext" -}} + {{- $ := last . }} + {{- $result := dict }} + {{- range . }} + {{- if and (hasKey . "securityContexts") (hasKey .securityContexts "container") .securityContexts.container }} + {{- $result = .securityContexts.container }} + {{- break }} + {{- end }} + {{- end }} + {{- if $result }} + {{- toYaml $result | print }} + {{- else if and (hasKey $ "securityContexts") (hasKey $.securityContexts "containers") $.securityContexts.containers }} + {{- toYaml $.securityContexts.containers | print }} + {{- else -}} +allowPrivilegeEscalation: false +capabilities: + drop: + - ALL + {{- end }} +{{- end -}} + +{{/* +Set the default value for external container securityContext(redis and statsd). +If no value is passed for .securityContexts.container, defaults to deny privileges escalation and dropping all POSIX capabilities. + + +-----------------------------------+ +------------------------------------------------------------+ + | .securityContexts.container | -> | allowPrivilegesEscalation: false, capabilities.drop: [ALL] | + +-----------------------------------+ +------------------------------------------------------------+ + +The template can be called like so: + include "externalContainerSecurityContext" .Values.statsd +*/}} +{{- define "externalContainerSecurityContext" -}} + {{- if .securityContexts.container -}} + {{ toYaml .securityContexts.container | print }} + {{- else -}} +allowPrivilegeEscalation: false +capabilities: + drop: + - ALL + {{- end -}} +{{- end -}} + +{{- define "container_extra_envs" -}} + {{- $ := index . 0 -}} + {{- $env := index . 1 -}} + {{- range $i, $config := $env }} + - name: {{ $config.name }} + {{- if $config.value }} + value: {{ $config.value | quote }} + {{- else if $config.valueFrom }} + valueFrom: + {{- if $config.valueFrom.secretKeyRef }} + secretKeyRef: + name: {{ $config.valueFrom.secretKeyRef.name }} + key: {{ $config.valueFrom.secretKeyRef.key }} + {{- else if $config.valueFrom.configMapKeyRef }} + configMapKeyRef: + name: {{ $config.valueFrom.configMapKeyRef.name }} + key: {{ $config.valueFrom.configMapKeyRef.key }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} + +{{/* +Convert a Kubernetes CPU limit (e.g., "500m", "1.5", "2", "750m") into an integer number of CPU cores. +*/}} +{{- define "cpu_count" -}} + {{- $v := toString . -}} + {{- if hasSuffix "m" $v -}} + {{- /* millicores path: e.g. 500m, 1500m */ -}} + {{- $m := float64 (trimSuffix "m" $v) -}} + {{- int (ceil (divf $m 1000)) -}} + {{- else -}} + {{- /* plain cores: e.g. 0.5, 1, 1.5 */ -}} + {{- int (ceil (float64 $v)) -}} + {{- end -}} +{{- end }} + +{{/* +Check if the current executor launches pods +This helper function returns true if the executor can launch pods (Kubernetes-based executors) +*/}} +{{- define "airflow.podLaunchingExecutor" -}} + {{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) (contains "KubernetesExecutor" .Values.executor) (contains "LocalKubernetesExecutor" .Values.executor) -}} + {{- print "true" -}} + {{- end -}} +{{- end -}} + +{{/* +Get revisionHistoryLimit with nil-aware fallback. +Unlike `or`, this properly handles 0 as a valid value. +Pass a list of values to check in order of priority. + +Usage: + include "airflow.revisionHistoryLimit" (list .Values.scheduler.revisionHistoryLimit .Values.revisionHistoryLimit) +*/}} +{{- define "airflow.revisionHistoryLimit" -}} + {{- $result := "" -}} + {{- range . -}} + {{- if and (not (kindIs "invalid" .)) -}} + {{- $result = . -}} + {{- break -}} + {{- end -}} + {{- end -}} + {{- $result -}} +{{- end -}} + +{{/* +Determine if the create-user job should be enabled. +When webserver.defaultUser is set (deprecated), it takes precedence to preserve +backwards compatibility. Otherwise, fall back to createUserJob.enabled. +*/}} +{{- define "createUserJob.isEnabled" -}} + {{- if .Values.webserver.defaultUser -}} + {{- .Values.webserver.defaultUser.enabled -}} + {{- else -}} + {{- .Values.createUserJob.enabled -}} + {{- end -}} +{{- end -}} + +{{/* +Convert dagBundleConfigList YAML list to JSON string for dag_bundle_config_list. +This helper function converts the structured YAML format to the JSON string +format required by Airflow's dag_bundle_config_list configuration. + +Usage: + config: + dag_processor: + dag_bundle_config_list: '{{ include "dag_bundle_config_list" . }}' +*/}} +{{- define "dag_bundle_config_list" -}} + {{- if .Values.dagProcessor.dagBundleConfigList -}} + {{- $bundles := list -}} + {{- range .Values.dagProcessor.dagBundleConfigList -}} + {{- $bundle := dict "name" .name "classpath" .classpath "kwargs" (.kwargs | default dict) -}} + {{- $bundles = append $bundles $bundle -}} + {{- end -}} + {{- $bundles | toJson -}} + {{- else -}} + {{- "[]" -}} + {{- end -}} +{{- end }} + +{{/* +Custom merge function which enables full map overwrite and `or` logic for boolean overwrite. +It takes 4 arguments where: + 1. Input map (source for merge) + 2. Input map with values which will overwrite values from first map + 3. Root section name (used for `or` boolean logic) + 4. List of section names for which logic for boolean parameters should be `or` instead of overwrite + +Usage: + {{ include "workersMergeValues" (list .Values.workers .Values.workers.celery "" (list "kerberosInitContainer")) }} +*/}} +{{- define "workersMergeValues" -}} + {{- $inputMap := index . 0 -}} + {{- $overwriteMap := index . 1 -}} + {{- $sectionName := index . 2 -}} + {{- $orBoolean := index . 3 -}} + {{- $outputMap := dict -}} + + {{- $fullOverwrite := list "annotations" "podAnnotations" "persistentVolumeClaimRetentionPolicy" "pod" "container" "securityContext" "containerLifecycleHooks" "config" "advanced" "behavior" "resources" "nodeSelector" "affinity" "labels" -}} + + {{- range $key, $val := $inputMap -}} + {{/* Below if holds logic for full overwrite map logic */}} + {{- if and (hasKey $overwriteMap $key) (has $key $fullOverwrite) -}} + {{- $_ := set $outputMap $key (get $overwriteMap $key) -}} + {{/* Below if holds base logic for merge values */}} + {{- else if and (hasKey $overwriteMap $key) (kindIs "map" $val) -}} + {{- $nested := include "workersMergeValues" (list $val (get $overwriteMap $key) $key $orBoolean) | fromYaml -}} + {{- if gt (len $nested) 0 -}} + {{- $_ := set $outputMap $key $nested -}} + {{- end -}} + {{/* Below if contains check for `slice` to fully overwrite list parameters */}} + {{- else if and (hasKey $overwriteMap $key) (not (and (kindIs "slice" (get $overwriteMap $key)) (eq (len (get $overwriteMap $key)) 0))) -}} + {{/* Below if holds `or` logic for boolean fields in a map */}} + {{- if and (kindIs "bool" $val) (has $sectionName $orBoolean) -}} + {{- $_ := set $outputMap $key (or $val (get $overwriteMap $key)) -}} + {{- else -}} + {{- $_ := set $outputMap $key (get $overwriteMap $key) -}} + {{- end -}} + {{- else -}} + {{- $_ := set $outputMap $key $val -}} + {{- end -}} + {{- end -}} + {{/* Below if makes sure that all fields in overwrite map input parameter will be copied to source map (if they don't exist) */}} + {{- range $key, $val := $overwriteMap -}} + {{- if not (hasKey $inputMap $key) -}} + {{- $_ := set $outputMap $key $val -}} + {{- end -}} + {{- end -}} + {{- toYaml $outputMap -}} +{{- end -}} + +{{/* +Remove all nil fields in a map and return new dict object. + +Usage: + {{ include "removeNilFields" . }} +*/}} +{{- define "removeNilFields" -}} + {{- $newValues := dict -}} + {{- range $key, $val := . -}} + {{- if kindIs "map" $val -}} + {{- $nested := include "removeNilFields" $val | fromYaml -}} + {{- if gt (len $nested) 0 -}} + {{- $_ := set $newValues $key $nested -}} + {{- end -}} + {{- else if not (eq $val nil) -}} + {{- $_ := set $newValues $key $val -}} + {{- end -}} + {{- end -}} + {{- toYaml $newValues -}} +{{- end -}} diff --git a/charts/airflow/templates/api-server/api-server-deployment.yaml b/charts/airflow/templates/api-server/api-server-deployment.yaml new file mode 100644 index 0000000..a79b77c --- /dev/null +++ b/charts/airflow/templates/api-server/api-server-deployment.yaml @@ -0,0 +1,249 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow API Server Deployment +################################# +{{- if and .Values.apiServer.enabled (semverCompare ">=3.0.0" .Values.airflowVersion) }} +{{- $nodeSelector := or .Values.apiServer.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.apiServer.affinity .Values.affinity }} +{{- $tolerations := or .Values.apiServer.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.apiServer.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $revisionHistoryLimit := include "airflow.revisionHistoryLimit" (list .Values.apiServer.revisionHistoryLimit .Values.revisionHistoryLimit) }} +{{- $securityContext := include "airflowPodSecurityContext" (list .Values.apiServer .Values) }} +{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.apiServer .Values) }} +{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list .Values.apiServer.waitForMigrations .Values) }} +{{- $containerLifecycleHooks := or .Values.apiServer.containerLifecycleHooks .Values.containerLifecycleHooks }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "airflow.fullname" . }}-api-server + labels: + tier: airflow + component: api-server + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.apiServer.annotations }} + annotations: {{- toYaml .Values.apiServer.annotations | nindent 4 }} + {{- end }} +spec: + {{- if not .Values.apiServer.hpa.enabled }} + replicas: {{ .Values.apiServer.replicas }} + {{- end }} + {{- if ne $revisionHistoryLimit "" }} + revisionHistoryLimit: {{ $revisionHistoryLimit }} + {{- end }} + strategy: + {{- if .Values.apiServer.strategy }} + {{- toYaml .Values.apiServer.strategy | nindent 4 }} + {{- else }} + # Here we define the rolling update strategy + # - maxSurge define how many pod we can add at a time + # - maxUnavailable define how many pod can be unavailable + # during the rolling update + # Setting maxUnavailable to 0 would make sure we have the appropriate + # capacity during the rolling update. + # You can also use percentage based value instead of integer. + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + {{- end }} + selector: + matchLabels: + tier: airflow + component: api-server + release: {{ .Release.Name }} + template: + metadata: + labels: + tier: airflow + component: api-server + release: {{ .Release.Name }} + {{- if or .Values.labels .Values.apiServer.labels }} + {{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent 8 }} + {{- end }} + annotations: + checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }} + checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }} + checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }} + checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }} + checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }} + {{- if not .Values.jwtSecretName }} + checksum/jwt-secret: {{ include (print $.Template.BasePath "/secrets/jwt-secret.yaml") . | sha256sum }} + {{- end }} + {{- if .Values.airflowPodAnnotations }} + {{- tpl (toYaml .Values.airflowPodAnnotations) . | nindent 8 }} + {{- end }} + {{- if .Values.apiServer.podAnnotations }} + {{- tpl (toYaml .Values.apiServer.podAnnotations) . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.apiServer.hostAliases }} + hostAliases: {{- toYaml .Values.apiServer.hostAliases | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "apiServer.serviceAccountName" . }} + {{- if .Values.apiServer.priorityClassName }} + priorityClassName: {{ .Values.apiServer.priorityClassName }} + {{- end }} + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} + affinity: + {{- if $affinity }} + {{- toYaml $affinity | nindent 8 }} + {{- else }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + component: api-server + topologyKey: kubernetes.io/hostname + weight: 100 + {{- end }} + tolerations: {{- toYaml $tolerations | nindent 8 }} + topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }} + restartPolicy: Always + securityContext: {{ $securityContext | nindent 8 }} + imagePullSecrets: {{- include "image_pull_secrets" . | nindent 8 }} + initContainers: + {{- if .Values.apiServer.waitForMigrations.enabled }} + - name: wait-for-airflow-migrations + resources: {{- toYaml .Values.apiServer.resources | nindent 12 }} + image: {{ template "airflow_image_for_migrations" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }} + volumeMounts: + {{- include "airflow_config_mount" . | nindent 12 }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.apiServer.extraVolumeMounts }} + {{- tpl (toYaml .Values.apiServer.extraVolumeMounts) . | nindent 12 }} + {{- end }} + args: {{- include "wait-for-migrations-command" . | indent 10 }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- if .Values.apiServer.waitForMigrations.env }} + {{- tpl (toYaml .Values.apiServer.waitForMigrations.env) $ | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.apiServer.extraInitContainers }} + {{- tpl (toYaml .Values.apiServer.extraInitContainers) . | nindent 8 }} + {{- end }} + containers: + - name: api-server + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContext | nindent 12 }} + {{- if $containerLifecycleHooks }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }} + {{- end }} + {{- if .Values.apiServer.command }} + command: {{ tpl (toYaml .Values.apiServer.command) . | nindent 12 }} + {{- end }} + {{- if .Values.apiServer.args }} + args: {{- tpl (toYaml .Values.apiServer.args) . | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.apiServer.resources | nindent 12 }} + volumeMounts: + {{- include "airflow_config_mount" . | nindent 12 }} + {{- if or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName }} + {{- include "airflow_api_server_config_mount" . | nindent 12 }} + {{- end }} + {{- if .Values.logs.persistence.enabled }} + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- end }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.apiServer.extraVolumeMounts }} + {{- tpl (toYaml .Values.apiServer.extraVolumeMounts) . | nindent 12 }} + {{- end }} + ports: + - name: api-server + containerPort: {{ .Values.ports.apiServer }} + livenessProbe: + httpGet: + path: /api/v2/monitor/health + port: {{ .Values.ports.apiServer }} + scheme: {{ .Values.apiServer.livenessProbe.scheme | default "http" }} + initialDelaySeconds: {{ .Values.apiServer.livenessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.apiServer.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.apiServer.livenessProbe.failureThreshold }} + periodSeconds: {{ .Values.apiServer.livenessProbe.periodSeconds }} + readinessProbe: + httpGet: + path: /api/v2/monitor/health + port: {{ .Values.ports.apiServer }} + scheme: {{ .Values.apiServer.readinessProbe.scheme | default "http" }} + initialDelaySeconds: {{ .Values.apiServer.readinessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.apiServer.readinessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.apiServer.readinessProbe.failureThreshold }} + periodSeconds: {{ .Values.apiServer.readinessProbe.periodSeconds }} + startupProbe: + httpGet: + path: /api/v2/monitor/health + port: {{ .Values.ports.apiServer }} + scheme: {{ .Values.apiServer.startupProbe.scheme | default "http" }} + initialDelaySeconds: {{ .Values.apiServer.startupProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.apiServer.startupProbe.timeoutSeconds }} + failureThreshold: {{ .Values.apiServer.startupProbe.failureThreshold }} + periodSeconds: {{ .Values.apiServer.startupProbe.periodSeconds }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" true) .) | indent 10 }} + {{- include "container_extra_envs" (list . .Values.apiServer.env) | indent 10 }} + {{- if .Values.apiServer.extraContainers }} + {{- tpl (toYaml .Values.apiServer.extraContainers) . | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ template "airflow_config" . }} + {{- if or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName }} + - name: api-server-config + configMap: + name: {{ template "airflow_api_server_config_configmap_name" . }} + {{- end }} + {{- if .Values.logs.persistence.enabled }} + - name: logs + persistentVolumeClaim: + claimName: {{ template "airflow_logs_volume_claim" . }} + {{- end }} + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 8 }} + {{- end }} + {{- if .Values.apiServer.extraVolumes }} + {{- tpl (toYaml .Values.apiServer.extraVolumes) . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/api-server/api-server-hpa.yaml b/charts/airflow/templates/api-server/api-server-hpa.yaml new file mode 100644 index 0000000..16988bc --- /dev/null +++ b/charts/airflow/templates/api-server/api-server-hpa.yaml @@ -0,0 +1,49 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Api-Server HPA +################################# +{{- if and .Values.apiServer.enabled .Values.apiServer.hpa.enabled (semverCompare ">=3.0.0" .Values.airflowVersion) }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "airflow.fullname" . }}-api-server + labels: + tier: airflow + component: api-server-horizontalpodautoscaler + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + deploymentName: {{ .Release.Name }}-api-server + {{- if or .Values.labels .Values.apiServer.labels }} + {{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "airflow.fullname" . }}-api-server + minReplicas: {{ .Values.apiServer.hpa.minReplicaCount }} + maxReplicas: {{ .Values.apiServer.hpa.maxReplicaCount }} + metrics: {{- toYaml .Values.apiServer.hpa.metrics | nindent 4 }} + {{- with .Values.apiServer.hpa.behavior }} + behavior: {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/api-server/api-server-ingress.yaml b/charts/airflow/templates/api-server/api-server-ingress.yaml new file mode 100644 index 0000000..9e530d7 --- /dev/null +++ b/charts/airflow/templates/api-server/api-server-ingress.yaml @@ -0,0 +1,109 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow API Server Ingress +################################# +{{- if and .Values.apiServer.enabled (semverCompare ">=3.0.0" .Values.airflowVersion) (or .Values.ingress.apiServer.enabled .Values.ingress.enabled) }} +{{- $fullname := include "airflow.fullname" . }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullname }}-ingress + labels: + tier: airflow + component: airflow-ingress + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.apiServer.labels }} + {{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.ingress.apiServer.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.apiServer.hosts (.Values.ingress.apiServer.hosts | first | kindIs "string" | not) }} + {{- $anyTlsHosts := false -}} + {{- range .Values.ingress.apiServer.hosts }} + {{- if and .tls .tls.enabled }} + {{- $anyTlsHosts = true -}} + {{- end }} + {{- end }} + {{- if $anyTlsHosts }} + tls: + {{- range .Values.ingress.apiServer.hosts }} + {{- if and .tls .tls.enabled }} + - hosts: + - {{ tpl .name $ | quote }} + secretName: {{ .tls.secretName }} + {{- end }} + {{- end }} + {{- end }} + {{- else if .Values.ingress.apiServer.tls.enabled }} + tls: + - hosts: + {{- range .Values.ingress.apiServer.hosts | default (list .Values.ingress.apiServer.host) }} + - {{ tpl . $ | quote }} + {{- end }} + secretName: {{ .Values.ingress.apiServer.tls.secretName }} + {{- end }} + rules: + {{- range .Values.ingress.apiServer.hosts | default (list .Values.ingress.apiServer.host) }} + - http: + paths: + {{- range $.Values.ingress.apiServer.precedingPaths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ $fullname }}-api-server + port: + name: {{ .servicePort }} + {{- end }} + - backend: + service: + name: {{ $fullname }}-api-server + port: + name: api-server + {{- if $.Values.ingress.apiServer.path }} + path: {{ $.Values.ingress.apiServer.path }} + pathType: {{ $.Values.ingress.apiServer.pathType }} + {{- end }} + {{- range $.Values.ingress.apiServer.succeedingPaths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ $fullname }}-api-server + port: + name: {{ .servicePort }} + {{- end }} + {{- $hostname := . -}} + {{- if . | kindIs "string" | not }} + {{- $hostname = .name -}} + {{- end }} + {{- if $hostname }} + host: {{ tpl $hostname $ | quote }} + {{- end }} + {{- end }} + {{- if .Values.ingress.apiServer.ingressClassName }} + ingressClassName: {{ .Values.ingress.apiServer.ingressClassName }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/api-server/api-server-networkpolicy.yaml b/charts/airflow/templates/api-server/api-server-networkpolicy.yaml new file mode 100644 index 0000000..4459a85 --- /dev/null +++ b/charts/airflow/templates/api-server/api-server-networkpolicy.yaml @@ -0,0 +1,56 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow API Server NetworkPolicy +################################# +{{- if and .Values.apiServer.enabled .Values.networkPolicies.enabled (semverCompare ">=3.0.0" .Values.airflowVersion) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "airflow.fullname" . }}-api-server-policy + labels: + tier: airflow + component: airflow-api-server-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.apiServer.labels }} + {{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: api-server + release: {{ .Release.Name }} + policyTypes: + - Ingress + {{- if .Values.apiServer.networkPolicy.ingress.from }} + ingress: + - from: {{- toYaml .Values.apiServer.networkPolicy.ingress.from | nindent 6 }} + ports: + {{ range .Values.apiServer.networkPolicy.ingress.ports }} + - + {{- range $key, $val := . }} + {{ $key }}: {{ tpl (toString $val) $ }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/api-server/api-server-poddisruptionbudget.yaml b/charts/airflow/templates/api-server/api-server-poddisruptionbudget.yaml new file mode 100644 index 0000000..f6dfb85 --- /dev/null +++ b/charts/airflow/templates/api-server/api-server-poddisruptionbudget.yaml @@ -0,0 +1,44 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow API Server PodDisruptionBudget +################################# +{{- if and .Values.apiServer.enabled .Values.apiServer.podDisruptionBudget.enabled (semverCompare ">=3.0.0" .Values.airflowVersion) }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "airflow.fullname" . }}-api-server-pdb + labels: + tier: airflow + component: api-server + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.apiServer.labels }} + {{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + tier: airflow + component: api-server + release: {{ .Release.Name }} + {{- toYaml .Values.apiServer.podDisruptionBudget.config | nindent 2 }} +{{- end }} diff --git a/charts/airflow/templates/api-server/api-server-service.yaml b/charts/airflow/templates/api-server/api-server-service.yaml new file mode 100644 index 0000000..a402321 --- /dev/null +++ b/charts/airflow/templates/api-server/api-server-service.yaml @@ -0,0 +1,59 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow API Server Service +################################# +{{- if and .Values.apiServer.enabled (semverCompare ">=3.0.0" .Values.airflowVersion) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "airflow.fullname" . }}-api-server + labels: + tier: airflow + component: api-server + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.apiServer.labels }} + {{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.apiServer.service.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.apiServer.service.type }} + selector: + tier: airflow + component: api-server + release: {{ .Release.Name }} + ports: + {{ range .Values.apiServer.service.ports }} + - + {{- range $key, $val := . }} + {{ $key }}: {{ tpl (toString $val) $ }} + {{- end }} + {{- end }} + {{- if .Values.apiServer.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.apiServer.service.loadBalancerIP }} + {{- end }} + {{- if .Values.apiServer.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.apiServer.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/api-server/api-server-serviceaccount.yaml b/charts/airflow/templates/api-server/api-server-serviceaccount.yaml new file mode 100644 index 0000000..f9bbaf8 --- /dev/null +++ b/charts/airflow/templates/api-server/api-server-serviceaccount.yaml @@ -0,0 +1,42 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +###################################### +## Airflow API Server ServiceAccount +###################################### +{{- if and (semverCompare ">=3.0.0" .Values.airflowVersion) .Values.apiServer.enabled .Values.apiServer.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.apiServer.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ include "apiServer.serviceAccountName" . }} + labels: + tier: airflow + component: api-server + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.apiServer.labels }} + {{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.apiServer.serviceAccount.annotations }} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/check-values.yaml b/charts/airflow/templates/check-values.yaml new file mode 100644 index 0000000..c89fc5a --- /dev/null +++ b/charts/airflow/templates/check-values.yaml @@ -0,0 +1,97 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +{{- /* +The sole purpose of this YAML file is it to check the values file is consistent for some complex combinations. +*/ -}} + +{{- /* +############################## + Version checks +############################# +*/ -}} + {{- if semverCompare "<2.11.0" .Values.airflowVersion }} + {{ required "This chart only supports Apache Airflow version 2.11.0 and above." nil }} + {{- end }} +{{- /* +############################## + Redis related checks +############################# +*/ -}} + + {{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }} + {{- if .Values.redis.enabled }} + + {{- if .Values.redis.passwordSecretName }} + {{- $existedBrokerUrlCmd := false }} + {{- range .Values.env }} + {{- if eq .name "AIRFLOW__CELERY__BROKER_URL_CMD" }} + {{- $existedBrokerUrlCmd = true }} + {{- break -}} + {{- end }} + {{- end }} + + {{- if not (or .Values.data.brokerUrlSecretName $existedBrokerUrlCmd) }} + {{ required "When using the internal redis of the chart and setting the value redis.passwordSecretName, you must also set the value data.brokerUrlSecretName or AIRFLOW__CELERY__BROKER_URL_CMD in env." nil }} + {{- end }} + {{- end }} + + {{- if and .Values.redis.passwordSecretName .Values.redis.password }} + {{ required "You must not set both values redis.passwordSecretName and redis.password" nil }} + {{- end }} + + {{- else }} + + {{- if not (or .Values.data.brokerUrlSecretName .Values.data.brokerUrl) }} + {{ required "You must set one of the values data.brokerUrlSecretName or data.brokerUrl when using a Celery based executor with redis.enabled set to false (we need the url to the redis instance)." nil }} + {{- end }} + + {{- end }} + + {{- if and .Values.data.brokerUrlSecretName .Values.data.brokerUrl }} + {{ required "You must not set both values data.brokerUrlSecretName and data.brokerUrl" nil }} + {{- end }} + + {{- end }} + + {{- if and .Values.elasticsearch.enabled .Values.opensearch.enabled }} + {{ required "You must not set both values elasticsearch.enabled and opensearch.enabled" nil }} + {{- end }} + + {{- if .Values.elasticsearch.enabled }} + {{- if and .Values.elasticsearch.secretName .Values.elasticsearch.connection }} + {{ required "You must not set both values elasticsearch.secretName and elasticsearch.connection" nil }} + {{- end }} + + {{- if not (or .Values.elasticsearch.secretName .Values.elasticsearch.connection) }} + {{ required "You must set one of the values elasticsearch.secretName or elasticsearch.connection when using a Elasticsearch" nil }} + {{- end }} + + {{- end }} + + {{- if .Values.opensearch.enabled }} + {{- if and .Values.opensearch.secretName .Values.opensearch.connection }} + {{ required "You must not set both values opensearch.secretName and opensearch.connection" nil }} + {{- end }} + + {{- if not (or .Values.opensearch.secretName .Values.opensearch.connection) }} + {{ required "You must set one of the values opensearch.secretName or opensearch.connection when using OpenSearch" nil }} + {{- end }} + + {{- end }} diff --git a/charts/airflow/templates/cleanup/cleanup-cronjob.yaml b/charts/airflow/templates/cleanup/cleanup-cronjob.yaml new file mode 100644 index 0000000..ae202fd --- /dev/null +++ b/charts/airflow/templates/cleanup/cleanup-cronjob.yaml @@ -0,0 +1,117 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Cleanup Pods CronJob +################################# +{{- if and .Values.cleanup.enabled (contains "KubernetesExecutor" .Values.executor) }} +{{- $nodeSelector := or .Values.cleanup.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.cleanup.affinity .Values.affinity }} +{{- $tolerations := or .Values.cleanup.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.cleanup.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $securityContext := include "airflowPodSecurityContext" (list .Values.cleanup .Values) }} +{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.cleanup .Values) }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "airflow.fullname" . }}-cleanup + labels: + tier: airflow + component: airflow-cleanup-pods + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.cleanup.jobAnnotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + schedule: "{{ tpl .Values.cleanup.schedule . }}" + # The cron job does not allow concurrent runs; if it is time for a new job run and the previous job run hasn't finished yet, the cron job skips the new job run + concurrencyPolicy: Forbid + {{- if not (eq .Values.cleanup.failedJobsHistoryLimit nil) }} + failedJobsHistoryLimit: {{ .Values.cleanup.failedJobsHistoryLimit }} + {{- end }} + {{- if not (eq .Values.cleanup.successfulJobsHistoryLimit nil) }} + successfulJobsHistoryLimit: {{ .Values.cleanup.successfulJobsHistoryLimit }} + {{- end }} + jobTemplate: + spec: + backoffLimit: 1 + template: + metadata: + labels: + tier: airflow + component: airflow-cleanup-pods + release: {{ .Release.Name }} + {{- if or .Values.labels .Values.cleanup.labels }} + {{- mustMerge .Values.cleanup.labels .Values.labels | toYaml | nindent 12 }} + {{- end }} + annotations: + {{- if .Values.airflowPodAnnotations }} + {{- tpl (toYaml .Values.airflowPodAnnotations) . | nindent 12 }} + {{- end }} + {{- if .Values.cleanup.podAnnotations }} + {{- tpl (toYaml .Values.cleanup.podAnnotations) . | nindent 12 }} + {{- end }} + spec: + restartPolicy: Never + {{- if .Values.cleanup.priorityClassName }} + priorityClassName: {{ .Values.cleanup.priorityClassName }} + {{- end }} + nodeSelector: {{- toYaml $nodeSelector | nindent 12 }} + affinity: {{- toYaml $affinity | nindent 12 }} + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + tolerations: {{- toYaml $tolerations | nindent 12 }} + topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 12 }} + serviceAccountName: {{ include "cleanup.serviceAccountName" . }} + imagePullSecrets: {{- include "image_pull_secrets" . | nindent 12 }} + securityContext: {{ $securityContext | nindent 12 }} + containers: + - name: airflow-cleanup-pods + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContext | nindent 16 }} + {{- if .Values.cleanup.command }} + command: {{ tpl (toYaml .Values.cleanup.command) . | nindent 16 }} + {{- end }} + {{- if .Values.cleanup.args }} + args: {{ tpl (toYaml .Values.cleanup.args) . | nindent 16 }} + {{- end }} + env: + {{- include "standard_airflow_environment" . | indent 12 }} + {{- include "container_extra_envs" (list . .Values.cleanup.env) | indent 12 }} + volumeMounts: + {{- include "airflow_config_mount" . | nindent 16 }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 16 }} + {{- end }} + resources: {{- toYaml .Values.cleanup.resources | nindent 16 }} + volumes: + - name: config + configMap: + name: {{ template "airflow_config" . }} + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 12 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/cleanup/cleanup-serviceaccount.yaml b/charts/airflow/templates/cleanup/cleanup-serviceaccount.yaml new file mode 100644 index 0000000..d3c2222 --- /dev/null +++ b/charts/airflow/templates/cleanup/cleanup-serviceaccount.yaml @@ -0,0 +1,41 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Cleanup ServiceAccount +################################# +{{- if and .Values.cleanup.enabled .Values.cleanup.serviceAccount.create (contains "KubernetesExecutor" .Values.executor) }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.cleanup.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ include "cleanup.serviceAccountName" . }} + labels: + tier: airflow + component: airflow-cleanup-pods + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.cleanup.labels }} + {{- mustMerge .Values.cleanup.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.cleanup.serviceAccount.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/configmaps/api-server-configmap.yaml b/charts/airflow/templates/configmaps/api-server-configmap.yaml new file mode 100644 index 0000000..69d5dd4 --- /dev/null +++ b/charts/airflow/templates/configmaps/api-server-configmap.yaml @@ -0,0 +1,44 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow ConfigMap +################################# +{{- if and (semverCompare ">=3.0.0" .Values.airflowVersion) .Values.apiServer.apiServerConfig (not .Values.apiServer.apiServerConfigConfigMapName) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "airflow_api_server_config_configmap_name" . }} + labels: + tier: airflow + component: config + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.apiServer.configMapAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + webserver_config.py: |- + {{- tpl .Values.apiServer.apiServerConfig . | nindent 4 }} +{{- end }} diff --git a/charts/airflow/templates/configmaps/configmap.yaml b/charts/airflow/templates/configmaps/configmap.yaml new file mode 100644 index 0000000..a762fee --- /dev/null +++ b/charts/airflow/templates/configmaps/configmap.yaml @@ -0,0 +1,83 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow ConfigMap +################################# +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "airflow_config" . }} + labels: + tier: airflow + component: config + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end -}} + {{- if .Values.airflowConfigAnnotations }} + annotations: {{- toYaml .Values.airflowConfigAnnotations | nindent 4 }} + {{- end }} +{{- $Global := . }} +data: + {{- $config := deepCopy .Values.config | merge (dict "core" dict) }} + {{/*- Set a default for core.execution_api_server_url pointing to the api-server service if it's not set -*/}} + {{- if semverCompare ">=3.0.0" .Values.airflowVersion -}} + {{- $basePath := "" -}} + {{- if not (hasKey $config.core "execution_api_server_url") -}} + {{- if and $config.api $config.api.base_url -}} + {{- with urlParse $config.api.base_url }}{{ $basePath = (trimSuffix "/" .path) }}{{ end }} + {{- end -}} + {{- $_ := set $config.core "execution_api_server_url" (printf "http://%s-api-server:%d%s/execution/" (include "airflow.fullname" .) (int .Values.ports.apiServer) $basePath) -}} + {{- end -}} + {{- end -}} + # These are system-specified config overrides. + airflow.cfg: |- + {{- range $section, $settings := $config }} + [{{ $section }}] + {{- range $key, $val := $settings }} + {{ $key }} = {{ tpl ($val | toString) $Global }} + {{- end }} + {{ end }} + + {{- if .Values.airflowLocalSettings }} + airflow_local_settings.py: |- + {{- tpl .Values.airflowLocalSettings . | nindent 4 }} + {{- end }} + + {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.knownHosts }} + known_hosts: |- + {{- .Values.dags.gitSync.knownHosts | nindent 4 }} + {{- end }} + + {{- if or (contains "LocalKubernetesExecutor" $.Values.executor) (contains "KubernetesExecutor" $.Values.executor) (contains "CeleryKubernetesExecutor" $.Values.executor) }} + pod_template_file.yaml: |- + {{- if .Values.podTemplate }} + {{- tpl .Values.podTemplate . | nindent 4 }} + {{- else }} + {{- tpl (.Files.Get "files/pod-template-file.kubernetes-helm-yaml") . | nindent 4 }} + {{- end }} + {{- end }} + + {{- if .Values.kerberos.enabled }} + krb5.conf: |- + {{- tpl .Values.kerberos.config . | nindent 4 }} + {{- end }} diff --git a/charts/airflow/templates/configmaps/extra-configmaps.yaml b/charts/airflow/templates/configmaps/extra-configmaps.yaml new file mode 100644 index 0000000..2e02552 --- /dev/null +++ b/charts/airflow/templates/configmaps/extra-configmaps.yaml @@ -0,0 +1,56 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +#################################################### +## Extra ConfigMaps provisioned via the chart values +#################################################### +{{- $Global := . }} +{{- range $configMapName, $configMapContent := .Values.extraConfigMaps }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ tpl $configMapName $Global | quote }} + labels: + tier: airflow + release: {{ $Global.Release.Name }} + chart: "{{ $Global.Chart.Name }}-{{ $Global.Chart.Version }}" + heritage: {{ $Global.Release.Service }} + {{- with $Global.Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if $configMapContent.labels }} + {{- toYaml $configMapContent.labels | nindent 4 }} + {{- end }} + {{- $annotations := dict }} + {{- if or $configMapContent.useHelmHooks (not (hasKey $configMapContent "useHelmHooks")) }} + {{- $_ := set $annotations "helm.sh/hook" "pre-install,pre-upgrade" }} + {{- $_ := set $annotations "helm.sh/hook-weight" "0" }} + {{- $_ := set $annotations "helm.sh/hook-delete-policy" "before-hook-creation" }} + {{- end }} + {{- with $annotations := merge $annotations ($configMapContent.annotations | default dict) }} + annotations: {{- $annotations | toYaml | nindent 4 }} + {{- end }} +{{- if $configMapContent.data }} +data: + {{- with $configMapContent.data }} + {{- tpl . $Global | nindent 2 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/airflow/templates/configmaps/statsd-configmap.yaml b/charts/airflow/templates/configmaps/statsd-configmap.yaml new file mode 100644 index 0000000..69799cb --- /dev/null +++ b/charts/airflow/templates/configmaps/statsd-configmap.yaml @@ -0,0 +1,56 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow StatsD ConfigMap +################################# +{{- if .Values.statsd.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "airflow.fullname" . }}-statsd + labels: + tier: airflow + component: config + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.statsd.configMapAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + mappings.yml: |- + {{- if .Values.statsd.overrideMappings }} + mappings: + {{- toYaml .Values.statsd.overrideMappings | nindent 6 }} + {{- else }} + {{- tpl (.Files.Get "files/statsd-mappings.yml") . | nindent 4 }} + {{- if .Values.statsd.extraMappings }} + {{- toYaml .Values.statsd.extraMappings | nindent 6 }} + {{- end }} + {{- end }} + {{- if .Values.statsd.cache.ttl }} + defaults: + ttl: {{ .Values.statsd.cache.ttl }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/configmaps/webserver-configmap.yaml b/charts/airflow/templates/configmaps/webserver-configmap.yaml new file mode 100644 index 0000000..c1b25ba --- /dev/null +++ b/charts/airflow/templates/configmaps/webserver-configmap.yaml @@ -0,0 +1,44 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow ConfigMap +################################# +{{- if and .Values.webserver.webserverConfig (not .Values.webserver.webserverConfigConfigMapName) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "airflow_webserver_config_configmap_name" . }} + labels: + tier: airflow + component: config + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.webserver.configMapAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + webserver_config.py: |- + {{- tpl .Values.webserver.webserverConfig . | nindent 4 }} +{{- end }} diff --git a/charts/airflow/templates/dag-processor/dag-processor-deployment.yaml b/charts/airflow/templates/dag-processor/dag-processor-deployment.yaml new file mode 100644 index 0000000..f228631 --- /dev/null +++ b/charts/airflow/templates/dag-processor/dag-processor-deployment.yaml @@ -0,0 +1,291 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Dag Processor Deployment +################################# +{{- $enabled := .Values.dagProcessor.enabled }} +{{- if eq $enabled nil}} + {{ $enabled = ternary true false (semverCompare ">=3.0.0" .Values.airflowVersion) }} +{{- end }} +{{- if $enabled }} +{{- $nodeSelector := or .Values.dagProcessor.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.dagProcessor.affinity .Values.affinity }} +{{- $tolerations := or .Values.dagProcessor.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.dagProcessor.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $revisionHistoryLimit := include "airflow.revisionHistoryLimit" (list .Values.dagProcessor.revisionHistoryLimit .Values.revisionHistoryLimit) }} +{{- $securityContext := include "airflowPodSecurityContext" (list .Values.dagProcessor .Values) }} +{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.dagProcessor .Values) }} +{{- $containerSecurityContextLogGroomerSidecar := include "containerSecurityContext" (list .Values.dagProcessor.logGroomerSidecar .Values) }} +{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list .Values.dagProcessor.waitForMigrations .Values) }} +{{- $containerLifecycleHooks := or .Values.dagProcessor.containerLifecycleHooks .Values.containerLifecycleHooks }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "airflow.fullname" . }}-dag-processor + labels: + tier: airflow + component: dag-processor + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.dagProcessor.annotations }} + annotations: {{- toYaml .Values.dagProcessor.annotations | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.dagProcessor.replicas }} + {{- if ne $revisionHistoryLimit "" }} + revisionHistoryLimit: {{ $revisionHistoryLimit }} + {{- end }} + selector: + matchLabels: + tier: airflow + component: dag-processor + release: {{ .Release.Name }} + {{- if .Values.dagProcessor.strategy }} + strategy: {{- toYaml .Values.dagProcessor.strategy | nindent 4 }} + {{- end }} + template: + metadata: + labels: + tier: airflow + component: dag-processor + release: {{ .Release.Name }} + {{- if or .Values.labels .Values.dagProcessor.labels }} + {{- mustMerge .Values.dagProcessor.labels .Values.labels | toYaml | nindent 8 }} + {{- end }} + annotations: + checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }} + checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }} + checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }} + checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }} + checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }} + {{- if .Values.dagProcessor.safeToEvict }} + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + {{- end }} + {{- if .Values.airflowPodAnnotations }} + {{- tpl (toYaml .Values.airflowPodAnnotations) . | nindent 8 }} + {{- end }} + {{- if .Values.dagProcessor.podAnnotations }} + {{- tpl (toYaml .Values.dagProcessor.podAnnotations) . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.dagProcessor.priorityClassName }} + priorityClassName: {{ .Values.dagProcessor.priorityClassName }} + {{- end }} + nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} + affinity: + {{- if $affinity }} + {{- toYaml $affinity | nindent 8 }} + {{- else }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + component: dag-processor + topologyKey: kubernetes.io/hostname + weight: 100 + {{- end }} + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + tolerations: {{- toYaml $tolerations | nindent 8 }} + topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.dagProcessor.terminationGracePeriodSeconds }} + restartPolicy: Always + serviceAccountName: {{ include "dagProcessor.serviceAccountName" . }} + securityContext: {{ $securityContext | nindent 8 }} + imagePullSecrets: {{ include "image_pull_secrets" . | nindent 8 }} + initContainers: + {{- if .Values.dagProcessor.waitForMigrations.enabled }} + - name: wait-for-airflow-migrations + resources: {{- toYaml .Values.dagProcessor.resources | nindent 12 }} + image: {{ template "airflow_image_for_migrations" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }} + volumeMounts: + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.dagProcessor.extraVolumeMounts }} + {{- tpl (toYaml .Values.dagProcessor.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- include "airflow_config_mount" . | nindent 12 }} + args: {{- include "wait-for-migrations-command" . | indent 10 }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- if .Values.dagProcessor.waitForMigrations.env }} + {{- tpl (toYaml .Values.dagProcessor.waitForMigrations.env) $ | nindent 12 }} + {{- end }} + {{- end }} + {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }} + {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }} + {{- end }} + {{- if .Values.dagProcessor.extraInitContainers }} + {{- tpl (toYaml .Values.dagProcessor.extraInitContainers) . | nindent 8 }} + {{- end }} + containers: + - name: dag-processor + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContext | nindent 12 }} + {{- if $containerLifecycleHooks }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }} + {{- end }} + {{- if .Values.dagProcessor.command }} + command: {{ tpl (toYaml .Values.dagProcessor.command) . | nindent 12 }} + {{- end }} + {{- if .Values.dagProcessor.args }} + args: {{ tpl (toYaml .Values.dagProcessor.args) . | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.dagProcessor.resources | nindent 12 }} + volumeMounts: + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.dagProcessor.extraVolumeMounts }} + {{- tpl (toYaml .Values.dagProcessor.extraVolumeMounts) . | nindent 12 }} + {{- end }} + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- include "airflow_config_mount" . | nindent 12 }} + {{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }} + {{- include "airflow_dags_mount" . | nindent 12 }} + {{- end }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- include "container_extra_envs" (list . .Values.dagProcessor.env) | indent 10 }} + livenessProbe: + initialDelaySeconds: {{ .Values.dagProcessor.livenessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.dagProcessor.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.dagProcessor.livenessProbe.failureThreshold }} + periodSeconds: {{ .Values.dagProcessor.livenessProbe.periodSeconds }} + exec: + command: + {{- if .Values.dagProcessor.livenessProbe.command }} + {{- toYaml .Values.dagProcessor.livenessProbe.command | nindent 16 }} + {{- else }} + {{- include "dag_processor_liveness_check_command" . | indent 14 }} + {{- end }} + {{- if .Values.dags.gitSync.enabled }} + {{- include "git_sync_container" . | indent 8 }} + {{- end }} + {{- if .Values.dagProcessor.logGroomerSidecar.enabled }} + - name: dag-processor-log-groomer + resources: {{- toYaml .Values.dagProcessor.logGroomerSidecar.resources | nindent 12 }} + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContextLogGroomerSidecar | nindent 12 }} + {{- if .Values.dagProcessor.logGroomerSidecar.command }} + command: {{ tpl (toYaml .Values.dagProcessor.logGroomerSidecar.command) . | nindent 12 }} + {{- end }} + {{- if .Values.dagProcessor.logGroomerSidecar.args }} + args: {{- tpl (toYaml .Values.dagProcessor.logGroomerSidecar.args) . | nindent 12 }} + {{- end }} + env: + {{- if .Values.dagProcessor.logGroomerSidecar.retentionDays }} + - name: AIRFLOW__LOG_RETENTION_DAYS + value: "{{ .Values.dagProcessor.logGroomerSidecar.retentionDays }}" + {{- end }} + {{- if .Values.dagProcessor.logGroomerSidecar.retentionMinutes }} + - name: AIRFLOW__LOG_RETENTION_MINUTES + value: "{{ .Values.dagProcessor.logGroomerSidecar.retentionMinutes }}" + {{- end }} + {{- if .Values.dagProcessor.logGroomerSidecar.frequencyMinutes }} + - name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES + value: "{{ .Values.dagProcessor.logGroomerSidecar.frequencyMinutes }}" + {{- end }} + {{- if .Values.dagProcessor.logGroomerSidecar.maxSizeBytes }} + - name: AIRFLOW__LOG_MAX_SIZE_BYTES + value: "{{ .Values.dagProcessor.logGroomerSidecar.maxSizeBytes | int64 }}" + {{- end }} + {{- if .Values.dagProcessor.logGroomerSidecar.maxSizePercent }} + - name: AIRFLOW__LOG_MAX_SIZE_PERCENT + value: "{{ .Values.dagProcessor.logGroomerSidecar.maxSizePercent }}" + {{- end }} + - name: AIRFLOW_HOME + value: "{{ .Values.airflowHome }}" + {{- if .Values.dagProcessor.logGroomerSidecar.env }} + {{- tpl (toYaml .Values.dagProcessor.logGroomerSidecar.env) $ | nindent 12 }} + {{- end }} + volumeMounts: + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.dagProcessor.extraVolumeMounts }} + {{- tpl (toYaml .Values.dagProcessor.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + {{- include "airflow_webserver_config_mount" . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.dagProcessor.extraContainers }} + {{- tpl (toYaml .Values.dagProcessor.extraContainers) . | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ template "airflow_config" . }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + - name: webserver-config + configMap: + name: {{ template "airflow_webserver_config_configmap_name" . }} + {{- end }} + {{- if .Values.dags.persistence.enabled }} + - name: dags + persistentVolumeClaim: + claimName: {{ template "airflow_dags_volume_claim" . }} + {{- else if .Values.dags.gitSync.enabled }} + - name: dags + emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }} + {{- end }} + {{- if and .Values.dags.gitSync.enabled (or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey) }} + {{- include "git_sync_ssh_key_volume" . | indent 8 }} + {{- end }} + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 8 }} + {{- end }} + {{- if .Values.dagProcessor.extraVolumes }} + {{- tpl (toYaml .Values.dagProcessor.extraVolumes) . | nindent 8 }} + {{- end }} + {{- if .Values.logs.persistence.enabled }} + - name: logs + persistentVolumeClaim: + claimName: {{ template "airflow_logs_volume_claim" . }} + {{- else }} + - name: logs + emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | nindent 12 }} +{{- end }} +{{- end }} diff --git a/charts/airflow/templates/dag-processor/dag-processor-poddisruptionbudget.yaml b/charts/airflow/templates/dag-processor/dag-processor-poddisruptionbudget.yaml new file mode 100644 index 0000000..996c0e7 --- /dev/null +++ b/charts/airflow/templates/dag-processor/dag-processor-poddisruptionbudget.yaml @@ -0,0 +1,48 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Dag Processor PodDisruptionBudget +################################# +{{- $enabled := .Values.dagProcessor.enabled }} +{{- if eq $enabled nil}} + {{ $enabled = ternary true false (semverCompare ">=3.0.0" .Values.airflowVersion) }} +{{- end }} +{{- if and $enabled .Values.dagProcessor.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "airflow.fullname" . }}-dag-processor-pdb + labels: + tier: airflow + component: dag-processor + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.dagProcessor.labels }} + {{- mustMerge .Values.dagProcessor.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + tier: airflow + component: dag-processor + release: {{ .Release.Name }} + {{- toYaml .Values.dagProcessor.podDisruptionBudget.config | nindent 2 }} +{{- end }} diff --git a/charts/airflow/templates/dag-processor/dag-processor-serviceaccount.yaml b/charts/airflow/templates/dag-processor/dag-processor-serviceaccount.yaml new file mode 100644 index 0000000..89b71eb --- /dev/null +++ b/charts/airflow/templates/dag-processor/dag-processor-serviceaccount.yaml @@ -0,0 +1,46 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +####################################### +## Airflow Dag Processor ServiceAccount +####################################### +{{- $enabled := .Values.dagProcessor.enabled }} +{{- if eq $enabled nil}} + {{ $enabled = ternary true false (semverCompare ">=3.0.0" .Values.airflowVersion) }} +{{- end }} +{{- if and .Values.dagProcessor.serviceAccount.create $enabled }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.dagProcessor.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ include "dagProcessor.serviceAccountName" . }} + labels: + tier: airflow + component: dag-processor + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.dagProcessor.labels }} + {{- mustMerge .Values.dagProcessor.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.dagProcessor.serviceAccount.annotations}} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/dags-persistent-volume-claim.yaml b/charts/airflow/templates/dags-persistent-volume-claim.yaml new file mode 100644 index 0000000..198330b --- /dev/null +++ b/charts/airflow/templates/dags-persistent-volume-claim.yaml @@ -0,0 +1,52 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +###################################### +## Airflow DAGs PersistentVolumeClaim +###################################### +{{- if and (not .Values.dags.persistence.existingClaim) .Values.dags.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ template "airflow_dags_volume_claim" . }} + labels: + tier: airflow + component: dags-pvc + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.dags.persistence.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: [{{ .Values.dags.persistence.accessMode | quote }}] + resources: + requests: + storage: {{ .Values.dags.persistence.size | quote }} + {{- if .Values.dags.persistence.storageClassName }} + {{- if (eq "-" .Values.dags.persistence.storageClassName) }} + storageClassName: "" + {{- else }} + storageClassName: {{ tpl .Values.dags.persistence.storageClassName . | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/database-cleanup/database-cleanup-cronjob.yaml b/charts/airflow/templates/database-cleanup/database-cleanup-cronjob.yaml new file mode 100644 index 0000000..ae506f7 --- /dev/null +++ b/charts/airflow/templates/database-cleanup/database-cleanup-cronjob.yaml @@ -0,0 +1,125 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Database Cleanup CronJob +################################# +{{- if and .Values.databaseCleanup.enabled (or .Values.databaseCleanup.command .Values.databaseCleanup.args) }} +{{- $nodeSelector := or .Values.databaseCleanup.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.databaseCleanup.affinity .Values.affinity }} +{{- $tolerations := or .Values.databaseCleanup.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.databaseCleanup.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $securityContext := include "airflowPodSecurityContext" (list .Values.databaseCleanup .Values) }} +{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.databaseCleanup .Values) }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "airflow.fullname" . }}-database-cleanup + labels: + tier: airflow + component: database-cleanup + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.databaseCleanup.jobAnnotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + schedule: "{{ tpl .Values.databaseCleanup.schedule . }}" + # The cron job does not allow concurrent runs; if it is time for a new job run and the previous job run hasn't finished yet, the cron job skips the new job run + concurrencyPolicy: Forbid + {{- if not (eq .Values.databaseCleanup.failedJobsHistoryLimit nil) }} + failedJobsHistoryLimit: {{ .Values.databaseCleanup.failedJobsHistoryLimit }} + {{- end }} + {{- if not (eq .Values.databaseCleanup.successfulJobsHistoryLimit nil) }} + successfulJobsHistoryLimit: {{ .Values.databaseCleanup.successfulJobsHistoryLimit }} + {{- end }} + jobTemplate: + spec: + backoffLimit: 1 + {{- if not (kindIs "invalid" .Values.databaseCleanup.ttlSecondsAfterFinished) }} + ttlSecondsAfterFinished: {{ .Values.databaseCleanup.ttlSecondsAfterFinished }} + {{- end }} + template: + metadata: + labels: + tier: airflow + component: database-cleanup + release: {{ .Release.Name }} + {{- if or .Values.labels .Values.databaseCleanup.labels }} + {{- mustMerge .Values.databaseCleanup.labels .Values.labels | toYaml | nindent 12 }} + {{- end }} + annotations: + {{- if .Values.airflowPodAnnotations }} + {{- tpl (toYaml .Values.airflowPodAnnotations) . | nindent 12 }} + {{- end }} + {{- if .Values.databaseCleanup.podAnnotations }} + {{- tpl (toYaml .Values.databaseCleanup.podAnnotations) . | nindent 12 }} + {{- end }} + spec: + restartPolicy: Never + {{- if .Values.databaseCleanup.priorityClassName }} + priorityClassName: {{ .Values.databaseCleanup.priorityClassName }} + {{- end }} + nodeSelector: {{- toYaml $nodeSelector | nindent 12 }} + affinity: {{- toYaml $affinity | nindent 12 }} + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + tolerations: {{- toYaml $tolerations | nindent 12 }} + topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 12 }} + serviceAccountName: {{ include "databaseCleanup.serviceAccountName" . }} + imagePullSecrets: {{- include "image_pull_secrets" . | nindent 12 }} + securityContext: {{ $securityContext | nindent 12 }} + containers: + - name: database-cleanup + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContext | nindent 16 }} + {{- if .Values.databaseCleanup.command }} + command: {{ tpl (toYaml .Values.databaseCleanup.command) . | nindent 16 }} + {{- end }} + {{- if .Values.databaseCleanup.args }} + args: {{ tpl (toYaml .Values.databaseCleanup.args) . | nindent 16 }} + {{- end }} + {{- if .Values.databaseCleanup.applyCustomEnv }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 14 }} + env: {{- include "custom_airflow_environment" . | indent 14 }} + {{- else }} + env: + {{- end }} + {{- include "standard_airflow_environment" . | indent 14 }} + {{- include "container_extra_envs" (list . .Values.databaseCleanup.env) | indent 14 }} + volumeMounts: + {{- include "airflow_config_mount" . | nindent 16 }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 16 }} + {{- end }} + resources: {{- toYaml .Values.databaseCleanup.resources | nindent 16 }} + volumes: + - name: config + configMap: + name: {{ template "airflow_config" . }} + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 12 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/database-cleanup/database-cleanup-serviceaccount.yaml b/charts/airflow/templates/database-cleanup/database-cleanup-serviceaccount.yaml new file mode 100644 index 0000000..80262a4 --- /dev/null +++ b/charts/airflow/templates/database-cleanup/database-cleanup-serviceaccount.yaml @@ -0,0 +1,41 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Database Cleanup ServiceAccount +################################# +{{- if and .Values.databaseCleanup.serviceAccount.create .Values.databaseCleanup.enabled }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.databaseCleanup.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ include "databaseCleanup.serviceAccountName" . }} + labels: + tier: airflow + component: database-cleanup + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.databaseCleanup.labels }} + {{- mustMerge .Values.databaseCleanup.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.databaseCleanup.serviceAccount.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/flower/flower-deployment.yaml b/charts/airflow/templates/flower/flower-deployment.yaml new file mode 100644 index 0000000..4f5ee79 --- /dev/null +++ b/charts/airflow/templates/flower/flower-deployment.yaml @@ -0,0 +1,179 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Flower Deployment +################################# +{{- if and .Values.flower.enabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }} +{{- $nodeSelector := or .Values.flower.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.flower.affinity .Values.affinity }} +{{- $tolerations := or .Values.flower.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.flower.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $revisionHistoryLimit := include "airflow.revisionHistoryLimit" (list .Values.flower.revisionHistoryLimit .Values.revisionHistoryLimit) }} +{{- $securityContext := include "airflowPodSecurityContext" (list .Values.flower .Values) }} +{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.flower .Values) }} +{{- $containerLifecycleHooks := or .Values.flower.containerLifecycleHooks .Values.containerLifecycleHooks }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "airflow.fullname" . }}-flower + labels: + tier: airflow + component: flower + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.flower.annotations }} + annotations: {{- toYaml .Values.flower.annotations | nindent 4 }} + {{- end }} +spec: + replicas: 1 + {{- if ne $revisionHistoryLimit "" }} + revisionHistoryLimit: {{ $revisionHistoryLimit }} + {{- end }} + selector: + matchLabels: + tier: airflow + component: flower + release: {{ .Release.Name }} + template: + metadata: + labels: + tier: airflow + component: flower + release: {{ .Release.Name }} + {{- if or .Values.labels .Values.flower.labels }} + {{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 8 }} + {{- end }} + annotations: + checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }} + checksum/flower-secret: {{ include (print $.Template.BasePath "/secrets/flower-secret.yaml") . | sha256sum }} + {{- if or .Values.airflowPodAnnotations .Values.flower.podAnnotations }} + {{- tpl (mustMerge .Values.flower.podAnnotations .Values.airflowPodAnnotations | toYaml) . | nindent 8 }} + {{- end }} + spec: + nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} + affinity: {{- toYaml $affinity | nindent 8 }} + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + tolerations: {{- toYaml $tolerations | nindent 8 }} + topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }} + serviceAccountName: {{ include "flower.serviceAccountName" . }} + {{- if .Values.flower.priorityClassName }} + priorityClassName: {{ .Values.flower.priorityClassName }} + {{- end }} + restartPolicy: Always + securityContext: {{ $securityContext | nindent 8 }} + imagePullSecrets: {{- include "image_pull_secrets" . | nindent 8 }} + containers: + - name: flower + image: {{ template "flower_image" . }} + imagePullPolicy: {{ .Values.images.flower.pullPolicy }} + securityContext: {{ $containerSecurityContext | nindent 12 }} + {{- if $containerLifecycleHooks }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }} + {{- end }} + {{- if .Values.flower.command }} + command: {{ tpl (toYaml .Values.flower.command) . | nindent 12 }} + {{- end }} + {{- if .Values.flower.args }} + args: {{ tpl (toYaml .Values.flower.args) . | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.flower.resources | nindent 12 }} + volumeMounts: + {{- include "airflow_config_mount" . | nindent 12 }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.flower.extraVolumeMounts }} + {{- tpl (toYaml .Values.flower.extraVolumeMounts) . | nindent 12 }} + {{- end }} + ports: + - name: flower-ui + containerPort: {{ .Values.ports.flowerUI }} + livenessProbe: + failureThreshold: {{ .Values.flower.livenessProbe.failureThreshold }} + exec: + command: + - curl + {{- if or .Values.flower.secretName (and .Values.flower.username .Values.flower.password) }} + - "--user" + - $AIRFLOW__CELERY__FLOWER_BASIC_AUTH + {{- end }} + - {{ printf "localhost:%s" (.Values.ports.flowerUI | toString) }} + initialDelaySeconds: {{ .Values.flower.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.flower.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.flower.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.flower.readinessProbe.failureThreshold }} + exec: + command: + - curl + {{- if or .Values.flower.secretName (and .Values.flower.username .Values.flower.password) }} + - "--user" + - $AIRFLOW__CELERY__FLOWER_BASIC_AUTH + {{- end }} + - {{ printf "localhost:%s" (.Values.ports.flowerUI | toString) }} + initialDelaySeconds: {{ .Values.flower.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.flower.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.flower.readinessProbe.timeoutSeconds }} + startupProbe: + failureThreshold: {{ .Values.flower.startupProbe.failureThreshold }} + exec: + command: + - curl + {{- if or .Values.flower.secretName (and .Values.flower.username .Values.flower.password) }} + - "--user" + - $AIRFLOW__CELERY__FLOWER_BASIC_AUTH + {{- end }} + - {{ printf "localhost:%s" (.Values.ports.flowerUI | toString) }} + periodSeconds: {{ .Values.flower.startupProbe.periodSeconds }} + initialDelaySeconds: {{ .Values.flower.startupProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.flower.startupProbe.timeoutSeconds }} + envFrom: + {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + {{- if or .Values.flower.secretName (and .Values.flower.username .Values.flower.password) }} + - name: AIRFLOW__CELERY__FLOWER_BASIC_AUTH + valueFrom: + secretKeyRef: + name: {{ template "flower_secret" . }} + key: basicAuth + {{- end }} + {{- include "standard_airflow_environment" . | indent 10 }} + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "container_extra_envs" (list . .Values.flower.env) | indent 10 }} + {{- if .Values.flower.extraContainers }} + {{- tpl (toYaml .Values.flower.extraContainers) . | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ template "airflow_config" . }} + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 8 }} + {{- end }} + {{- if .Values.flower.extraVolumes }} + {{- tpl (toYaml .Values.flower.extraVolumes) . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/flower/flower-ingress.yaml b/charts/airflow/templates/flower/flower-ingress.yaml new file mode 100644 index 0000000..11ba85a --- /dev/null +++ b/charts/airflow/templates/flower/flower-ingress.yaml @@ -0,0 +1,93 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Flower Ingress +################################# +{{- if .Values.flower.enabled }} +{{- if and (or .Values.ingress.flower.enabled .Values.ingress.enabled) (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }} +{{- $fullname := include "airflow.fullname" . }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullname }}-flower-ingress + labels: + tier: airflow + component: flower-ingress + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.flower.labels }} + {{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.ingress.flower.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.flower.hosts (.Values.ingress.flower.hosts | first | kindIs "string" | not) }} + {{- $anyTlsHosts := false -}} + {{- range .Values.ingress.flower.hosts }} + {{- if and .tls .tls.enabled }} + {{- $anyTlsHosts = true -}} + {{- end }} + {{- end }} + {{- if $anyTlsHosts }} + tls: + {{- range .Values.ingress.flower.hosts }} + {{- if and .tls .tls.enabled }} + - hosts: + - {{ tpl .name $ | quote }} + secretName: {{ .tls.secretName }} + {{- end }} + {{- end }} + {{- end }} + {{- else if .Values.ingress.flower.tls.enabled }} + tls: + - hosts: + {{- range .Values.ingress.flower.hosts | default (list .Values.ingress.flower.host) }} + - {{ tpl . $ | quote }} + {{- end }} + secretName: {{ .Values.ingress.flower.tls.secretName }} + {{- end }} + rules: + {{- range .Values.ingress.flower.hosts | default (list .Values.ingress.flower.host) }} + - http: + paths: + - backend: + service: + name: {{ $fullname }}-flower + port: + name: flower-ui + {{- if $.Values.ingress.flower.path }} + path: {{ $.Values.ingress.flower.path }} + pathType: {{ $.Values.ingress.flower.pathType }} + {{- end }} + {{- $hostname := . -}} + {{- if . | kindIs "string" | not }} + {{- $hostname = .name -}} + {{- end }} + {{- if $hostname }} + host: {{ tpl $hostname $ | quote }} + {{- end }} + {{- end }} + {{- if .Values.ingress.flower.ingressClassName }} + ingressClassName: {{ .Values.ingress.flower.ingressClassName }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/airflow/templates/flower/flower-networkpolicy.yaml b/charts/airflow/templates/flower/flower-networkpolicy.yaml new file mode 100644 index 0000000..118480a --- /dev/null +++ b/charts/airflow/templates/flower/flower-networkpolicy.yaml @@ -0,0 +1,57 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Flower NetworkPolicy +################################# +{{- if and .Values.flower.enabled .Values.networkPolicies.enabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }} +{{- $from := or .Values.flower.networkPolicy.ingress.from .Values.flower.extraNetworkPolicies }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "airflow.fullname" . }}-flower-policy + labels: + tier: airflow + component: airflow-flower-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.flower.labels }} + {{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: flower + release: {{ .Release.Name }} + policyTypes: + - Ingress + {{- if $from }} + ingress: + - from: {{- toYaml $from | nindent 6 }} + ports: + {{ range .Values.flower.networkPolicy.ingress.ports }} + - + {{- range $key, $val := . }} + {{ $key }}: {{ tpl (toString $val) $ }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/flower/flower-service.yaml b/charts/airflow/templates/flower/flower-service.yaml new file mode 100644 index 0000000..9539677 --- /dev/null +++ b/charts/airflow/templates/flower/flower-service.yaml @@ -0,0 +1,59 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Flower Service Component +################################# +{{- if and .Values.flower.enabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "airflow.fullname" . }}-flower + labels: + tier: airflow + component: flower + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.flower.labels }} + {{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.flower.service.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.flower.service.type }} + selector: + tier: airflow + component: flower + release: {{ .Release.Name }} + ports: + {{ range .Values.flower.service.ports }} + - + {{- range $key, $val := . }} + {{ $key }}: {{ tpl (toString $val) $ }} + {{- end }} + {{- end }} + {{- if .Values.flower.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.flower.service.loadBalancerIP }} + {{- end }} + {{- if .Values.flower.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.flower.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/flower/flower-serviceaccount.yaml b/charts/airflow/templates/flower/flower-serviceaccount.yaml new file mode 100644 index 0000000..87aacdb --- /dev/null +++ b/charts/airflow/templates/flower/flower-serviceaccount.yaml @@ -0,0 +1,41 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +###################################### +## Airflow Flower ServiceAccount +###################################### +{{- if and .Values.flower.enabled .Values.flower.serviceAccount.create (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.flower.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ include "flower.serviceAccountName" . }} + labels: + tier: airflow + component: flower + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.flower.labels }} + {{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.flower.serviceAccount.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/jobs/create-user-job-serviceaccount.yaml b/charts/airflow/templates/jobs/create-user-job-serviceaccount.yaml new file mode 100644 index 0000000..be619b0 --- /dev/null +++ b/charts/airflow/templates/jobs/create-user-job-serviceaccount.yaml @@ -0,0 +1,41 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +######################################### +## Airflow Create User Job ServiceAccount +######################################### +{{- if .Values.createUserJob.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.createUserJob.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ include "createUserJob.serviceAccountName" . }} + labels: + tier: airflow + component: create-user-job + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.createUserJob.labels }} + {{- mustMerge .Values.createUserJob.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.createUserJob.serviceAccount.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/jobs/create-user-job.yaml b/charts/airflow/templates/jobs/create-user-job.yaml new file mode 100644 index 0000000..1f4151f --- /dev/null +++ b/charts/airflow/templates/jobs/create-user-job.yaml @@ -0,0 +1,138 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +########################## +## Airflow Create User Job +########################## +{{- if eq (include "createUserJob.isEnabled" .) "true" }} +{{- $nodeSelector := or .Values.createUserJob.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.createUserJob.affinity .Values.affinity }} +{{- $tolerations := or .Values.createUserJob.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.createUserJob.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $securityContext := include "airflowPodSecurityContext" (list .Values.createUserJob .Values) }} +{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.createUserJob .Values) }} +{{- $containerLifecycleHooks := or .Values.createUserJob.containerLifecycleHooks .Values.containerLifecycleHooks }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "airflow.fullname" . }}-create-user + labels: + tier: airflow + component: create-user-job + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- $annotations := dict }} + {{- if .Values.createUserJob.useHelmHooks }} + {{- $_ := set $annotations "helm.sh/hook" "post-install,post-upgrade" }} + {{- $_ := set $annotations "helm.sh/hook-weight" "2" }} + {{- $_ := set $annotations "helm.sh/hook-delete-policy" "before-hook-creation,hook-succeeded" }} + {{- end }} + {{- with $annotations := merge $annotations .Values.createUserJob.jobAnnotations }} + annotations: {{- $annotations | toYaml | nindent 4 }} + {{- end }} +spec: + {{- if not (kindIs "invalid" .Values.createUserJob.ttlSecondsAfterFinished) }} + ttlSecondsAfterFinished: {{ .Values.createUserJob.ttlSecondsAfterFinished }} + {{- end }} + template: + metadata: + labels: + tier: airflow + component: create-user-job + release: {{ .Release.Name }} + {{- if or .Values.labels .Values.createUserJob.labels }} + {{- mustMerge .Values.createUserJob.labels .Values.labels | toYaml | nindent 8 }} + {{- end }} + {{- if or .Values.airflowPodAnnotations .Values.createUserJob.annotations }} + annotations: + {{- if .Values.airflowPodAnnotations }} + {{- tpl (toYaml .Values.airflowPodAnnotations) . | nindent 8 }} + {{- end }} + {{- if .Values.createUserJob.annotations }} + {{- tpl (toYaml .Values.createUserJob.annotations) . | nindent 8 }} + {{- end }} + {{- end }} + spec: + securityContext: {{ $securityContext | nindent 8 }} + restartPolicy: {{ .Values.createUserJob.restartPolicy }} + {{- if .Values.createUserJob.priorityClassName }} + priorityClassName: {{ .Values.createUserJob.priorityClassName }} + {{- end }} + nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} + affinity: {{- toYaml $affinity | nindent 8 }} + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + tolerations: {{- toYaml $tolerations | nindent 8 }} + topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }} + serviceAccountName: {{ include "createUserJob.serviceAccountName" . }} + imagePullSecrets: {{- include "image_pull_secrets" . | nindent 8 }} + {{- if .Values.createUserJob.extraInitContainers }} + initContainers: + {{- tpl (toYaml .Values.createUserJob.extraInitContainers) . | nindent 8 }} + {{- end }} + containers: + - name: create-user + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContext | nindent 12 }} + {{- if $containerLifecycleHooks }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }} + {{- end }} + {{- if .Values.createUserJob.command }} + command: {{ tpl (toYaml .Values.createUserJob.command) . | nindent 12 }} + {{- end }} + {{- if .Values.createUserJob.args }} + args: {{ tpl (toYaml .Values.createUserJob.args) . | nindent 12 }} + {{- end }} + {{- if .Values.createUserJob.applyCustomEnv }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: {{- include "custom_airflow_environment" . | indent 10 }} + {{- else }} + env: + {{- end }} + {{- include "standard_airflow_environment" . | indent 10 }} + {{- include "container_extra_envs" (list . .Values.createUserJob.env) | indent 10 }} + resources: {{- toYaml .Values.createUserJob.resources | nindent 12 }} + volumeMounts: + {{- include "airflow_config_mount" . | nindent 12 }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.createUserJob.extraVolumeMounts }} + {{- tpl (toYaml .Values.createUserJob.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- if .Values.createUserJob.extraContainers }} + {{- tpl (toYaml .Values.createUserJob.extraContainers) . | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ template "airflow_config" . }} + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 8 }} + {{- end }} + {{- if .Values.createUserJob.extraVolumes }} + {{- tpl (toYaml .Values.createUserJob.extraVolumes) . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/jobs/migrate-database-job-serviceaccount.yaml b/charts/airflow/templates/jobs/migrate-database-job-serviceaccount.yaml new file mode 100644 index 0000000..e5ee856 --- /dev/null +++ b/charts/airflow/templates/jobs/migrate-database-job-serviceaccount.yaml @@ -0,0 +1,41 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +############################################# +## Airflow Migrate Database Job ServiceAccount +############################################## +{{- if .Values.migrateDatabaseJob.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.migrateDatabaseJob.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ include "migrateDatabaseJob.serviceAccountName" . }} + labels: + tier: airflow + component: run-airflow-migrations + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.migrateDatabaseJob.labels }} + {{- mustMerge .Values.migrateDatabaseJob.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.migrateDatabaseJob.serviceAccount.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/jobs/migrate-database-job.yaml b/charts/airflow/templates/jobs/migrate-database-job.yaml new file mode 100644 index 0000000..362d5f4 --- /dev/null +++ b/charts/airflow/templates/jobs/migrate-database-job.yaml @@ -0,0 +1,142 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Run Migrations +################################# +{{- if .Values.migrateDatabaseJob.enabled }} +{{- $nodeSelector := or .Values.migrateDatabaseJob.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.migrateDatabaseJob.affinity .Values.affinity }} +{{- $tolerations := or .Values.migrateDatabaseJob.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.migrateDatabaseJob.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $securityContext := include "airflowPodSecurityContext" (list .Values.migrateDatabaseJob .Values) }} +{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.migrateDatabaseJob .Values) }} +{{- $containerLifecycleHooks := or .Values.migrateDatabaseJob.containerLifecycleHooks .Values.containerLifecycleHooks }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "airflow.fullname" . }}-run-airflow-migrations + labels: + tier: airflow + component: run-airflow-migrations + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- $annotations := dict }} + {{- if .Values.migrateDatabaseJob.useHelmHooks }} + {{- $_ := set $annotations "helm.sh/hook" "post-install,post-upgrade" }} + {{- $_ := set $annotations "helm.sh/hook-weight" "1" }} + {{- $_ := set $annotations "helm.sh/hook-delete-policy" "before-hook-creation,hook-succeeded" }} + {{- end }} + {{- with $annotations := merge $annotations .Values.migrateDatabaseJob.jobAnnotations }} + annotations: {{- $annotations | toYaml | nindent 4 }} + {{- end }} +spec: + {{- if not (kindIs "invalid" .Values.migrateDatabaseJob.ttlSecondsAfterFinished) }} + ttlSecondsAfterFinished: {{ .Values.migrateDatabaseJob.ttlSecondsAfterFinished }} + {{- end }} + template: + metadata: + labels: + tier: airflow + component: run-airflow-migrations + release: {{ .Release.Name }} + {{- if or .Values.labels .Values.migrateDatabaseJob.labels }} + {{- mustMerge .Values.migrateDatabaseJob.labels .Values.labels | toYaml | nindent 8 }} + {{- end }} + {{- if or .Values.airflowPodAnnotations .Values.migrateDatabaseJob.annotations }} + annotations: + {{- if .Values.airflowPodAnnotations }} + {{- tpl (toYaml .Values.airflowPodAnnotations) . | nindent 8 }} + {{- end }} + {{- if .Values.migrateDatabaseJob.annotations }} + {{- tpl (toYaml .Values.migrateDatabaseJob.annotations) . | nindent 8 }} + {{- end }} + {{- end }} + spec: + securityContext: {{ $securityContext | nindent 8 }} + restartPolicy: {{ .Values.migrateDatabaseJob.restartPolicy }} + {{- if .Values.migrateDatabaseJob.priorityClassName }} + priorityClassName: {{ .Values.migrateDatabaseJob.priorityClassName }} + {{- end }} + nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} + affinity: {{- toYaml $affinity | nindent 8 }} + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + tolerations: {{- toYaml $tolerations | nindent 8 }} + topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }} + serviceAccountName: {{ include "migrateDatabaseJob.serviceAccountName" . }} + imagePullSecrets: {{- include "image_pull_secrets" . | nindent 8 }} + {{- if .Values.migrateDatabaseJob.extraInitContainers }} + initContainers: + {{- tpl (toYaml .Values.migrateDatabaseJob.extraInitContainers) . | nindent 8 }} + {{- end }} + containers: + - name: run-airflow-migrations + image: {{ template "airflow_image_for_migrations" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContext | nindent 12 }} + {{- if $containerLifecycleHooks }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }} + {{- end }} + {{- if .Values.migrateDatabaseJob.command }} + command: {{- tpl (toYaml .Values.migrateDatabaseJob.command) . | nindent 12 }} + {{- end }} + {{- if .Values.migrateDatabaseJob.args }} + args: {{- tpl (toYaml .Values.migrateDatabaseJob.args) . | nindent 12 }} + {{- end }} + {{- if .Values.migrateDatabaseJob.applyCustomEnv }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: {{- include "custom_airflow_environment" . | indent 10 }} + {{- else }} + env: + {{- end }} + - name: PYTHONUNBUFFERED + value: "1" + {{- include "standard_airflow_environment" . | indent 10 }} + {{- if .Values.migrateDatabaseJob.env }} + {{- tpl (toYaml .Values.migrateDatabaseJob.env) $ | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.migrateDatabaseJob.resources | nindent 12 }} + volumeMounts: + {{- include "airflow_config_mount" . | nindent 12 }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.migrateDatabaseJob.extraVolumeMounts }} + {{- tpl (toYaml .Values.migrateDatabaseJob.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- if .Values.migrateDatabaseJob.extraContainers }} + {{- tpl (toYaml .Values.migrateDatabaseJob.extraContainers) . | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ template "airflow_config" . }} + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 8 }} + {{- end }} + {{- if .Values.migrateDatabaseJob.extraVolumes }} + {{- tpl (toYaml .Values.migrateDatabaseJob.extraVolumes) . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/limitrange.yaml b/charts/airflow/templates/limitrange.yaml new file mode 100644 index 0000000..98d0ae8 --- /dev/null +++ b/charts/airflow/templates/limitrange.yaml @@ -0,0 +1,39 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Namespace LimitRange +################################# +{{- if .Values.limits }} +apiVersion: v1 +kind: LimitRange +metadata: + name: {{ include "airflow.fullname" . }}-limit-range + labels: + tier: resources + component: limitrange + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + limits: {{- toYaml .Values.limits | nindent 4 }} +{{- end }} diff --git a/charts/airflow/templates/logs-persistent-volume-claim.yaml b/charts/airflow/templates/logs-persistent-volume-claim.yaml new file mode 100644 index 0000000..00028a3 --- /dev/null +++ b/charts/airflow/templates/logs-persistent-volume-claim.yaml @@ -0,0 +1,52 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +###################################### +## Airflow LOGs PersistentVolumeClaim +###################################### +{{- if and (not .Values.logs.persistence.existingClaim) .Values.logs.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ template "airflow_logs_volume_claim" . }} + labels: + tier: airflow + component: logs-pvc + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.logs.persistence.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: ["ReadWriteMany"] + resources: + requests: + storage: {{ .Values.logs.persistence.size | quote }} + {{- if .Values.logs.persistence.storageClassName }} + {{- if (eq "-" .Values.logs.persistence.storageClassName) }} + storageClassName: "" + {{- else }} + storageClassName: {{ tpl .Values.logs.persistence.storageClassName . | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/pgbouncer/pgbouncer-deployment.yaml b/charts/airflow/templates/pgbouncer/pgbouncer-deployment.yaml new file mode 100644 index 0000000..9d6550a --- /dev/null +++ b/charts/airflow/templates/pgbouncer/pgbouncer-deployment.yaml @@ -0,0 +1,221 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Pgbouncer Deployment +################################# +{{- if .Values.pgbouncer.enabled }} +{{- $nodeSelector := or .Values.pgbouncer.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.pgbouncer.affinity .Values.affinity }} +{{- $tolerations := or .Values.pgbouncer.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.pgbouncer.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $revisionHistoryLimit := include "airflow.revisionHistoryLimit" (list .Values.pgbouncer.revisionHistoryLimit .Values.revisionHistoryLimit) }} +{{- $securityContext := include "localPodSecurityContext" .Values.pgbouncer }} +{{- $containerSecurityContext := include "externalContainerSecurityContext" .Values.pgbouncer }} +{{- $containerSecurityContextMetricsExporter := include "externalContainerSecurityContext" .Values.pgbouncer.metricsExporterSidecar }} +{{- $containerLifecycleHooks := .Values.pgbouncer.containerLifecycleHooks }} +{{- $containerLifecycleHooksMetricsExporter := .Values.pgbouncer.metricsExporterSidecar.containerLifecycleHooks }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "airflow.fullname" . }}-pgbouncer + labels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.pgbouncer.annotations }} + annotations: {{- toYaml .Values.pgbouncer.annotations | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.pgbouncer.replicas | default "1" }} + {{- if ne $revisionHistoryLimit "" }} + revisionHistoryLimit: {{ $revisionHistoryLimit }} + {{- end }} + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + template: + metadata: + labels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + {{- if or .Values.labels .Values.pgbouncer.labels }} + {{- mustMerge .Values.pgbouncer.labels .Values.labels | toYaml | nindent 8 }} + {{- end }} + annotations: + checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }} + checksum/pgbouncer-certificates-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-certificates-secret.yaml") . | sha256sum }} + {{- if .Values.pgbouncer.podAnnotations }} + {{- tpl (toYaml .Values.pgbouncer.podAnnotations) . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.pgbouncer.priorityClassName }} + priorityClassName: {{ .Values.pgbouncer.priorityClassName }} + {{- end }} + nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} + affinity: {{- toYaml $affinity | nindent 8 }} + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + tolerations: {{- toYaml $tolerations | nindent 8 }} + topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }} + serviceAccountName: {{ include "pgbouncer.serviceAccountName" . }} + securityContext: {{ $securityContext | nindent 8 }} + restartPolicy: Always + imagePullSecrets: {{- include "image_pull_secrets" . | nindent 8 }} + containers: + - name: pgbouncer + image: {{ template "pgbouncer_image" . }} + imagePullPolicy: {{ .Values.images.pgbouncer.pullPolicy }} + securityContext: {{ $containerSecurityContext | nindent 12 }} + {{- if .Values.pgbouncer.command }} + command: {{ tpl (toYaml .Values.pgbouncer.command) . | nindent 12 }} + {{- end }} + {{- if .Values.pgbouncer.args }} + args: {{ tpl (toYaml .Values.pgbouncer.args) . | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.pgbouncer.resources | nindent 12 }} + {{- with .Values.pgbouncer.env }} + env: {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: pgbouncer + containerPort: {{ .Values.ports.pgbouncer }} + livenessProbe: + tcpSocket: + port: {{ .Values.ports.pgbouncer }} + readinessProbe: + tcpSocket: + port: {{ .Values.ports.pgbouncer }} + {{- if or .Values.pgbouncer.mountConfigSecret .Values.pgbouncer.ssl.ca .Values.pgbouncer.ssl.cert .Values.pgbouncer.ssl.key .Values.volumeMounts .Values.pgbouncer.extraVolumeMounts }} + volumeMounts: + {{- if .Values.pgbouncer.mountConfigSecret }} + - name: pgbouncer-config + subPath: pgbouncer.ini + mountPath: /etc/pgbouncer/pgbouncer.ini + readOnly: true + - name: pgbouncer-config + subPath: users.txt + mountPath: /etc/pgbouncer/users.txt + readOnly: true + {{- end}} + {{- if .Values.pgbouncer.ssl.ca }} + - name: pgbouncer-certificates + subPath: root.crt + mountPath: /etc/pgbouncer/root.crt + readOnly: true + {{- end }} + {{- if .Values.pgbouncer.ssl.cert }} + - name: pgbouncer-certificates + subPath: server.crt + mountPath: /etc/pgbouncer/server.crt + readOnly: true + {{- end }} + {{- if .Values.pgbouncer.ssl.key }} + - name: pgbouncer-certificates + subPath: server.key + mountPath: /etc/pgbouncer/server.key + readOnly: true + {{- end }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.pgbouncer.extraVolumeMounts }} + {{- tpl (toYaml .Values.pgbouncer.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- end}} + {{- if $containerLifecycleHooks }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }} + {{- end }} + - name: metrics-exporter + resources: {{- toYaml .Values.pgbouncer.metricsExporterSidecar.resources | nindent 12 }} + image: {{ template "pgbouncer_exporter_image" . }} + imagePullPolicy: {{ .Values.images.pgbouncerExporter.pullPolicy }} + securityContext: {{ $containerSecurityContextMetricsExporter | nindent 12 }} + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ template "pgbouncer_stats_secret" . }} + {{- if and .Values.pgbouncer.metricsExporterSidecar.statsSecretName .Values.pgbouncer.metricsExporterSidecar.statsSecretKey }} + key: {{ .Values.pgbouncer.metricsExporterSidecar.statsSecretKey }} + {{- else }} + key: "connection" + {{- end }} + ports: + - name: metrics + containerPort: {{ .Values.ports.pgbouncerScrape }} + livenessProbe: + exec: + command: + - pgbouncer_exporter + - health + initialDelaySeconds: {{ .Values.pgbouncer.metricsExporterSidecar.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.pgbouncer.metricsExporterSidecar.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.pgbouncer.metricsExporterSidecar.livenessProbe.timeoutSeconds }} + readinessProbe: + exec: + command: + - pgbouncer_exporter + - health + initialDelaySeconds: {{ .Values.pgbouncer.metricsExporterSidecar.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.pgbouncer.metricsExporterSidecar.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.pgbouncer.metricsExporterSidecar.readinessProbe.timeoutSeconds }} + {{- if $containerLifecycleHooksMetricsExporter }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooksMetricsExporter) . | nindent 12 }} + {{- end }} + {{- if .Values.pgbouncer.metricsExporterSidecar.extraVolumeMounts }} + volumeMounts: + {{- tpl (toYaml .Values.pgbouncer.metricsExporterSidecar.extraVolumeMounts) . | nindent 12 }} + {{- end}} + {{- if .Values.pgbouncer.extraContainers }} + {{- tpl (toYaml .Values.pgbouncer.extraContainers) . | nindent 8 }} + {{- end }} + {{- if or .Values.pgbouncer.mountConfigSecret .Values.pgbouncer.ssl.ca .Values.pgbouncer.ssl.cert .Values.pgbouncer.ssl.key .Values.volumes .Values.pgbouncer.extraVolumes }} + volumes: + {{- if .Values.pgbouncer.mountConfigSecret }} + - name: pgbouncer-config + secret: + secretName: {{ template "pgbouncer_config_secret" . }} + {{- end}} + {{- if or .Values.pgbouncer.ssl.ca .Values.pgbouncer.ssl.cert .Values.pgbouncer.ssl.key }} + - name: pgbouncer-certificates + secret: + secretName: {{ template "pgbouncer_certificates_secret" . }} + {{- end }} + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 8 }} + {{- end }} + {{- if .Values.pgbouncer.extraVolumes }} + {{- tpl (toYaml .Values.pgbouncer.extraVolumes) . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/pgbouncer/pgbouncer-ingress.yaml b/charts/airflow/templates/pgbouncer/pgbouncer-ingress.yaml new file mode 100644 index 0000000..0ad1f88 --- /dev/null +++ b/charts/airflow/templates/pgbouncer/pgbouncer-ingress.yaml @@ -0,0 +1,84 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Pgbouncer Ingress +################################# +{{- if and .Values.pgbouncer.enabled .Values.ingress.pgbouncer.enabled }} +{{- $fullname := include "airflow.fullname" . }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullname }}-pgbouncer-ingress + labels: + tier: airflow + component: pgbouncer-ingress + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.pgbouncer.labels }} + {{- mustMerge .Values.pgbouncer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.ingress.pgbouncer.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.pgbouncer.hosts (.Values.ingress.pgbouncer.hosts | first | kindIs "string" | not) }} + {{- $anyTlsHosts := false -}} + {{- range .Values.ingress.pgbouncer.hosts }} + {{- if and .tls .tls.enabled }} + {{- $anyTlsHosts = true -}} + {{- end }} + {{- end }} + {{- if $anyTlsHosts }} + tls: + {{- range .Values.ingress.pgbouncer.hosts }} + {{- if and .tls .tls.enabled }} + - hosts: + - {{ tpl .name $ | quote }} + secretName: {{ .tls.secretName }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.pgbouncer.hosts | default (list .Values.ingress.pgbouncer.host) }} + - http: + paths: + - backend: + service: + name: {{ $fullname }}-pgbouncer + port: + name: pgb-metrics + {{- if $.Values.ingress.pgbouncer.path }} + path: {{ $.Values.ingress.pgbouncer.path }} + pathType: {{ $.Values.ingress.pgbouncer.pathType }} + {{- end }} + {{- $hostname := . -}} + {{- if . | kindIs "string" | not }} + {{- $hostname = .name -}} + {{- end }} + {{- if $hostname }} + host: {{ tpl $hostname $ | quote }} + {{- end }} + {{- end }} + {{- if .Values.ingress.pgbouncer.ingressClassName }} + ingressClassName: {{ .Values.ingress.pgbouncer.ingressClassName }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/pgbouncer/pgbouncer-networkpolicy.yaml b/charts/airflow/templates/pgbouncer/pgbouncer-networkpolicy.yaml new file mode 100644 index 0000000..0c6a4dc --- /dev/null +++ b/charts/airflow/templates/pgbouncer/pgbouncer-networkpolicy.yaml @@ -0,0 +1,80 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Pgbouncer NetworkPolicy +################################# +{{- $kedaEnabled := .Values.workers.keda.enabled }} +{{- if hasKey .Values.workers "celery" }} + {{- $kedaEnabled = or .Values.workers.celery.keda.enabled (and (not (has .Values.workers.celery.keda.enabled (list true false))) .Values.workers.keda.enabled) }} +{{- end }} +{{- $workersKedaEnabled := and $kedaEnabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }} +{{- $triggererKedaEnabled := and .Values.triggerer.enabled .Values.triggerer.keda.enabled }} +{{- if and .Values.pgbouncer.enabled .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "airflow.fullname" . }}-pgbouncer-policy + labels: + tier: airflow + component: airflow-pgbouncer-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.pgbouncer.labels }} + {{- mustMerge .Values.pgbouncer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + tier: airflow + release: {{ .Release.Name }} + {{- if or $workersKedaEnabled $triggererKedaEnabled }} + {{- if and $workersKedaEnabled (.Values.workers.celery.keda.namespaceLabels | default .Values.workers.keda.namespaceLabels) }} + - namespaceSelector: + matchLabels: {{- toYaml (.Values.workers.celery.keda.namespaceLabels | default .Values.workers.keda.namespaceLabels) | nindent 10 }} + podSelector: + {{- else if and .Values.triggerer.enabled .Values.triggerer.keda.namespaceLabels }} + - namespaceSelector: + matchLabels: {{- toYaml .Values.triggerer.keda.namespaceLabels | nindent 10 }} + podSelector: + {{- else }} + - podSelector: + {{- end }} + matchLabels: + app: keda-operator + {{- end }} + {{- if .Values.pgbouncer.extraNetworkPolicies}} + {{- toYaml .Values.pgbouncer.extraNetworkPolicies | nindent 4 }} + {{- end }} + ports: + - protocol: TCP + port: {{ .Values.ports.pgbouncer }} + - protocol: TCP + port: {{ .Values.ports.pgbouncerScrape }} +{{- end }} diff --git a/charts/airflow/templates/pgbouncer/pgbouncer-poddisruptionbudget.yaml b/charts/airflow/templates/pgbouncer/pgbouncer-poddisruptionbudget.yaml new file mode 100644 index 0000000..25640ae --- /dev/null +++ b/charts/airflow/templates/pgbouncer/pgbouncer-poddisruptionbudget.yaml @@ -0,0 +1,44 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Pgbouncer PodDisruptionBudget +################################# +{{- if and .Values.pgbouncer.enabled .Values.pgbouncer.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "airflow.fullname" . }}-pgbouncer-pdb + labels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.pgbouncer.labels }} + {{- mustMerge .Values.pgbouncer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + {{- toYaml .Values.pgbouncer.podDisruptionBudget.config | nindent 2 }} +{{- end }} diff --git a/charts/airflow/templates/pgbouncer/pgbouncer-service.yaml b/charts/airflow/templates/pgbouncer/pgbouncer-service.yaml new file mode 100644 index 0000000..1290abe --- /dev/null +++ b/charts/airflow/templates/pgbouncer/pgbouncer-service.yaml @@ -0,0 +1,59 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Pgbouncer Service +################################# +{{- if .Values.pgbouncer.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "airflow.fullname" . }}-pgbouncer + labels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.pgbouncer.labels }} + {{- mustMerge .Values.pgbouncer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.ports.pgbouncerScrape | quote }} + {{- if .Values.pgbouncer.service.extraAnnotations }} + {{- toYaml .Values.pgbouncer.service.extraAnnotations | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + {{- if .Values.pgbouncer.service.clusterIp }} + clusterIP: {{ .Values.pgbouncer.service.clusterIp }} + {{- end }} + selector: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + ports: + - name: pgbouncer + protocol: TCP + port: {{ .Values.ports.pgbouncer }} + - name: pgb-metrics + protocol: TCP + port: {{ .Values.ports.pgbouncerScrape }} +{{- end }} diff --git a/charts/airflow/templates/pgbouncer/pgbouncer-serviceaccount.yaml b/charts/airflow/templates/pgbouncer/pgbouncer-serviceaccount.yaml new file mode 100644 index 0000000..53712ff --- /dev/null +++ b/charts/airflow/templates/pgbouncer/pgbouncer-serviceaccount.yaml @@ -0,0 +1,42 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +###################################### +## Airflow Pgbouncer ServiceAccount +###################################### +{{- if and .Values.pgbouncer.serviceAccount.create .Values.pgbouncer.enabled }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.pgbouncer.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ include "pgbouncer.serviceAccountName" . }} + labels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.pgbouncer.labels }} + {{- mustMerge .Values.pgbouncer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.pgbouncer.serviceAccount.annotations }} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/priorityclasses/priority-classes.yaml b/charts/airflow/templates/priorityclasses/priority-classes.yaml new file mode 100644 index 0000000..22153f2 --- /dev/null +++ b/charts/airflow/templates/priorityclasses/priority-classes.yaml @@ -0,0 +1,38 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################################# +## Priority classes provisioned via the chart values +################################################# +{{- $Global := . }} +{{- range $e := .Values.priorityClasses }} +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: {{ $Global.Release.Name }}-{{ $e.name }} + labels: + tier: airflow + release: {{ $Global.Release.Name }} + chart: "{{ $Global.Chart.Name }}-{{ $Global.Chart.Version }}" + heritage: {{ $Global.Release.Service }} +preemptionPolicy: {{ default "PreemptLowerPriority" $e.preemptionPolicy }} +value: {{ $e.value | required "value is required for priority classes" }} +description: "This priority class will not cause other pods to be preempted." +{{- end }} diff --git a/charts/airflow/templates/rbac/job-launcher-role.yaml b/charts/airflow/templates/rbac/job-launcher-role.yaml new file mode 100644 index 0000000..c99ba3c --- /dev/null +++ b/charts/airflow/templates/rbac/job-launcher-role.yaml @@ -0,0 +1,67 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Job Launcher Role +################################# +{{- if and .Values.rbac.create .Values.allowJobLaunching }} +apiVersion: rbac.authorization.k8s.io/v1 +{{- if .Values.multiNamespaceMode }} +kind: ClusterRole +{{- else }} +kind: Role +{{- end }} +metadata: + {{- if not .Values.multiNamespaceMode }} + name: {{ include "airflow.fullname" . }}-job-launcher-role + namespace: "{{ .Release.Namespace }}" + {{- else }} + name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-job-launcher-role + {{- end }} + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if .Values.multiNamespaceMode }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + - apiGroups: + - "batch" + resources: + - "jobs" + verbs: + - "create" + - "list" + - "get" + - "patch" + - "watch" + - "delete" + - apiGroups: + - "batch" + resources: + - "jobs/status" + verbs: + - "get" + - "watch" +{{- end }} diff --git a/charts/airflow/templates/rbac/job-launcher-rolebinding.yaml b/charts/airflow/templates/rbac/job-launcher-rolebinding.yaml new file mode 100644 index 0000000..6b80e31 --- /dev/null +++ b/charts/airflow/templates/rbac/job-launcher-rolebinding.yaml @@ -0,0 +1,77 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Job Launcher Role Binding +################################# +{{- if and .Values.rbac.create .Values.allowJobLaunching }} +{{- $schedulerLaunchExecutors := list "LocalExecutor" "LocalKubernetesExecutor" "KubernetesExecutor" "CeleryKubernetesExecutor" }} +{{- $workerLaunchExecutors := list "CeleryExecutor" "LocalKubernetesExecutor" "KubernetesExecutor" "CeleryKubernetesExecutor" }} +{{- $executors := split "," .Values.executor }} +apiVersion: rbac.authorization.k8s.io/v1 +{{- if .Values.multiNamespaceMode }} +kind: ClusterRoleBinding +{{- else }} +kind: RoleBinding +{{- end }} +metadata: + {{- if not .Values.multiNamespaceMode }} + namespace: "{{ .Release.Namespace }}" + name: {{ include "airflow.fullname" . }}-job-launcher-rolebinding + {{- else }} + name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-job-launcher-rolebinding + {{- end }} + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if .Values.multiNamespaceMode }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + {{- if .Values.multiNamespaceMode }} + kind: ClusterRole + name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-job-launcher-role + {{- else }} + kind: Role + name: {{ include "airflow.fullname" . }}-job-launcher-role + {{- end }} +subjects: + {{- range $executor := $executors }} + {{- if has $executor $schedulerLaunchExecutors }} + - kind: ServiceAccount + name: {{ include "scheduler.serviceAccountName" $ }} + namespace: "{{ $.Release.Namespace }}" + {{- break }} + {{- end }} + {{- end }} + {{- range $executor := $executors }} + {{- if has $executor $workerLaunchExecutors }} + - kind: ServiceAccount + name: {{ include "worker.serviceAccountName" $ }} + namespace: "{{ $.Release.Namespace }}" + {{- break }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/rbac/pod-cleanup-role.yaml b/charts/airflow/templates/rbac/pod-cleanup-role.yaml new file mode 100644 index 0000000..d6aff40 --- /dev/null +++ b/charts/airflow/templates/rbac/pod-cleanup-role.yaml @@ -0,0 +1,44 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Cleanup Role +################################# +{{- if and .Values.rbac.create .Values.cleanup.enabled (contains "KubernetesExecutor" .Values.executor) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "airflow.fullname" . }}-cleanup-role + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + - apiGroups: + - "" + resources: + - "pods" + verbs: + - "list" + - "delete" +{{- end }} diff --git a/charts/airflow/templates/rbac/pod-cleanup-rolebinding.yaml b/charts/airflow/templates/rbac/pod-cleanup-rolebinding.yaml new file mode 100644 index 0000000..95de10c --- /dev/null +++ b/charts/airflow/templates/rbac/pod-cleanup-rolebinding.yaml @@ -0,0 +1,44 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Cleanup Role Binding +################################# +{{- if and .Values.rbac.create .Values.cleanup.enabled (contains "KubernetesExecutor" .Values.executor) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "airflow.fullname" . }}-cleanup-rolebinding + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "airflow.fullname" . }}-cleanup-role +subjects: + - kind: ServiceAccount + name: {{ include "cleanup.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" +{{- end }} diff --git a/charts/airflow/templates/rbac/pod-database-cleanup-role.yaml b/charts/airflow/templates/rbac/pod-database-cleanup-role.yaml new file mode 100644 index 0000000..6e37d3e --- /dev/null +++ b/charts/airflow/templates/rbac/pod-database-cleanup-role.yaml @@ -0,0 +1,37 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Database Cleanup Role +################################# +{{- if and .Values.rbac.create .Values.databaseCleanup.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "airflow.fullname" . }}-database-cleanup-role + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +rules: [] +{{- end }} diff --git a/charts/airflow/templates/rbac/pod-database-cleanup-rolebinding.yaml b/charts/airflow/templates/rbac/pod-database-cleanup-rolebinding.yaml new file mode 100644 index 0000000..07584c5 --- /dev/null +++ b/charts/airflow/templates/rbac/pod-database-cleanup-rolebinding.yaml @@ -0,0 +1,44 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Database Cleanup Role Binding +################################# +{{- if and .Values.rbac.create .Values.databaseCleanup.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "airflow.fullname" . }}-database-cleanup-rolebinding + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "airflow.fullname" . }}-database-cleanup-role +subjects: + - kind: ServiceAccount + name: {{ include "databaseCleanup.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" +{{- end }} diff --git a/charts/airflow/templates/rbac/pod-launcher-role.yaml b/charts/airflow/templates/rbac/pod-launcher-role.yaml new file mode 100644 index 0000000..c6f3a54 --- /dev/null +++ b/charts/airflow/templates/rbac/pod-launcher-role.yaml @@ -0,0 +1,80 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Pod Launcher Role +################################# +{{- if and .Values.rbac.create .Values.allowPodLaunching }} +apiVersion: rbac.authorization.k8s.io/v1 +{{- if .Values.multiNamespaceMode }} +kind: ClusterRole +{{- else }} +kind: Role +{{- end }} +metadata: + {{- if not .Values.multiNamespaceMode }} + name: {{ include "airflow.fullname" . }}-pod-launcher-role + namespace: "{{ .Release.Namespace }}" + {{- else }} + name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-pod-launcher-role + {{- end }} + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if .Values.multiNamespaceMode }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + - apiGroups: + - "" + resources: + - "pods" + verbs: + - "create" + - "list" + - "get" + - "patch" + - "watch" + - "delete" + - apiGroups: + - "" + resources: + - "pods/log" + verbs: + - "get" + - apiGroups: + - "" + resources: + - "pods/exec" + verbs: + - "create" + - "get" + - apiGroups: + - "" + resources: + - "events" + verbs: + - "list" + - "watch" +{{- end }} diff --git a/charts/airflow/templates/rbac/pod-launcher-rolebinding.yaml b/charts/airflow/templates/rbac/pod-launcher-rolebinding.yaml new file mode 100644 index 0000000..d4eb37a --- /dev/null +++ b/charts/airflow/templates/rbac/pod-launcher-rolebinding.yaml @@ -0,0 +1,82 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Pod Launcher Role Binding +################################# +{{- if and .Values.rbac.create .Values.allowPodLaunching }} +{{- $schedulerLaunchExecutors := list "LocalExecutor" "LocalKubernetesExecutor" "KubernetesExecutor" "CeleryKubernetesExecutor" }} +{{- $workerLaunchExecutors := list "CeleryExecutor" "LocalKubernetesExecutor" "KubernetesExecutor" "CeleryKubernetesExecutor" }} +{{- $executors := split "," .Values.executor }} +apiVersion: rbac.authorization.k8s.io/v1 +{{- if .Values.multiNamespaceMode }} +kind: ClusterRoleBinding +{{- else }} +kind: RoleBinding +{{- end }} +metadata: + {{- if not .Values.multiNamespaceMode }} + namespace: "{{ .Release.Namespace }}" + name: {{ include "airflow.fullname" . }}-pod-launcher-rolebinding + {{- else }} + name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-pod-launcher-rolebinding + {{- end }} + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if .Values.multiNamespaceMode }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + {{- if .Values.multiNamespaceMode }} + kind: ClusterRole + name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-pod-launcher-role + {{- else }} + kind: Role + name: {{ include "airflow.fullname" . }}-pod-launcher-role + {{- end }} +subjects: + {{- range $executor := $executors }} + {{- if has $executor $schedulerLaunchExecutors }} + - kind: ServiceAccount + name: {{ include "scheduler.serviceAccountName" $ }} + namespace: "{{ $.Release.Namespace }}" + {{- break }} + {{- end }} + {{- end }} + {{- range $executor := $executors }} + {{- if has $executor $workerLaunchExecutors }} + - kind: ServiceAccount + name: {{ include "worker.serviceAccountName" $ }} + namespace: "{{ $.Release.Namespace }}" + {{- break }} + {{- end }} + {{- end }} + {{- if .Values.triggerer.enabled }} + - kind: ServiceAccount + name: {{ include "triggerer.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/rbac/pod-log-reader-role.yaml b/charts/airflow/templates/rbac/pod-log-reader-role.yaml new file mode 100644 index 0000000..59563e8 --- /dev/null +++ b/charts/airflow/templates/rbac/pod-log-reader-role.yaml @@ -0,0 +1,64 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Pod Reader Role +################################# +{{- if and .Values.rbac.create .Values.webserver.allowPodLogReading }} +apiVersion: rbac.authorization.k8s.io/v1 +{{- if .Values.multiNamespaceMode }} +kind: ClusterRole +{{- else }} +kind: Role +{{- end }} +metadata: + {{- if not .Values.multiNamespaceMode }} + name: {{ include "airflow.fullname" . }}-pod-log-reader-role + namespace: "{{ .Release.Namespace }}" + {{- else }} + name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-pod-log-reader-role + {{- end }} + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if .Values.multiNamespaceMode }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + - apiGroups: + - "" + resources: + - "pods" + verbs: + - "list" + - "get" + - "watch" + - apiGroups: + - "" + resources: + - "pods/log" + verbs: + - "get" + - "list" +{{- end }} diff --git a/charts/airflow/templates/rbac/pod-log-reader-rolebinding.yaml b/charts/airflow/templates/rbac/pod-log-reader-rolebinding.yaml new file mode 100644 index 0000000..c9f7876 --- /dev/null +++ b/charts/airflow/templates/rbac/pod-log-reader-rolebinding.yaml @@ -0,0 +1,68 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Pod Reader Role Binding +################################# +{{- if and .Values.rbac.create (or (and .Values.webserver.allowPodLogReading (semverCompare "<3.0.0" .Values.airflowVersion)) (and .Values.apiServer.allowPodLogReading (semverCompare ">=3.0.0" .Values.airflowVersion))) }} +apiVersion: rbac.authorization.k8s.io/v1 +{{- if .Values.multiNamespaceMode }} +kind: ClusterRoleBinding +{{- else }} +kind: RoleBinding +{{- end }} +metadata: + {{- if not .Values.multiNamespaceMode }} + namespace: "{{ .Release.Namespace }}" + name: {{ include "airflow.fullname" . }}-pod-log-reader-rolebinding + {{- else }} + name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-pod-log-reader-rolebinding + {{- end }} + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if .Values.multiNamespaceMode }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + {{- if .Values.multiNamespaceMode }} + kind: ClusterRole + name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-pod-log-reader-role + {{- else }} + kind: Role + name: {{ include "airflow.fullname" . }}-pod-log-reader-role + {{- end }} +subjects: + {{- if and .Values.webserver.allowPodLogReading (semverCompare "<3.0.0" .Values.airflowVersion) }} + - kind: ServiceAccount + name: {{ include "webserver.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- if and .Values.apiServer.allowPodLogReading (semverCompare ">=3.0.0" .Values.airflowVersion) }} + - kind: ServiceAccount + name: {{ include "apiServer.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/rbac/security-context-constraint-rolebinding.yaml b/charts/airflow/templates/rbac/security-context-constraint-rolebinding.yaml new file mode 100644 index 0000000..45f9548 --- /dev/null +++ b/charts/airflow/templates/rbac/security-context-constraint-rolebinding.yaml @@ -0,0 +1,118 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +########################### +## Airflow SCC Role Binding +########################### +{{- if and .Values.rbac.create .Values.rbac.createSCCRoleBinding }} +apiVersion: rbac.authorization.k8s.io/v1 +{{- if .Values.multiNamespaceMode }} +kind: ClusterRoleBinding +{{- else }} +kind: RoleBinding +{{- end }} +metadata: + {{- if not .Values.multiNamespaceMode }} + name: {{ include "airflow.fullname" . }}-scc-rolebinding + namespace: "{{ .Release.Namespace }}" + {{- else }} + name: {{ .Release.Namespace }}-{{ include "airflow.fullname" . }}-scc-rolebinding + {{- end }} + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if .Values.multiNamespaceMode }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:openshift:scc:anyuid +subjects: + {{- if and .Values.webserver.enabled (semverCompare "<3.0.0" .Values.airflowVersion) }} + - kind: ServiceAccount + name: {{ include "webserver.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- if or (contains "CeleryExecutor" .Values.executor) (contains "LocalKubernetesExecutor" .Values.executor) (contains "KubernetesExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }} + - kind: ServiceAccount + name: {{ include "worker.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- if .Values.scheduler.enabled }} + - kind: ServiceAccount + name: {{ include "scheduler.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- if and .Values.apiServer.enabled (semverCompare ">=3.0.0" .Values.airflowVersion) }} + - kind: ServiceAccount + name: {{ include "apiServer.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- if and .Values.statsd.enabled }} + - kind: ServiceAccount + name: {{ include "statsd.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- if and .Values.flower.enabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }} + - kind: ServiceAccount + name: {{ include "flower.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- if .Values.redis.enabled }} + - kind: ServiceAccount + name: {{ include "redis.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- if .Values.triggerer.enabled }} + - kind: ServiceAccount + name: {{ include "triggerer.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- if .Values.migrateDatabaseJob.enabled }} + - kind: ServiceAccount + name: {{ include "migrateDatabaseJob.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- if eq (include "createUserJob.isEnabled" .) "true" }} + - kind: ServiceAccount + name: {{ include "createUserJob.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- if .Values.cleanup.enabled }} + - kind: ServiceAccount + name: {{ include "cleanup.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- if .Values.databaseCleanup.enabled }} + - kind: ServiceAccount + name: {{ include "databaseCleanup.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- if .Values.dagProcessor.enabled }} + - kind: ServiceAccount + name: {{ include "dagProcessor.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/redis/redis-networkpolicy.yaml b/charts/airflow/templates/redis/redis-networkpolicy.yaml new file mode 100644 index 0000000..8ae08e0 --- /dev/null +++ b/charts/airflow/templates/redis/redis-networkpolicy.yaml @@ -0,0 +1,65 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Redis NetworkPolicy +################################# +{{- if and .Values.redis.enabled .Values.networkPolicies.enabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "airflow.fullname" . }}-redis-policy + labels: + tier: airflow + component: redis-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.redis.labels }} + {{- mustMerge .Values.redis.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: redis + release: {{ .Release.Name }} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + tier: airflow + component: worker + release: {{ .Release.Name }} + - podSelector: + matchLabels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + - podSelector: + matchLabels: + tier: airflow + component: flower + release: {{ .Release.Name }} + ports: + - protocol: TCP + port: {{ .Values.ports.redisDB }} +{{- end }} diff --git a/charts/airflow/templates/redis/redis-service.yaml b/charts/airflow/templates/redis/redis-service.yaml new file mode 100644 index 0000000..36c9daf --- /dev/null +++ b/charts/airflow/templates/redis/redis-service.yaml @@ -0,0 +1,58 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Redis Service +################################# +{{- if and .Values.redis.enabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "airflow.fullname" . }}-redis + labels: + tier: airflow + component: redis + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.redis.labels }} + {{- mustMerge .Values.redis.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: +{{- if eq .Values.redis.service.type "ClusterIP" }} + type: ClusterIP + {{- if .Values.redis.service.clusterIP }} + clusterIP: {{ .Values.redis.service.clusterIP }} + {{- end }} +{{- else }} + type: {{ .Values.redis.service.type }} +{{- end }} + selector: + tier: airflow + component: redis + release: {{ .Release.Name }} + ports: + - name: redis-db + protocol: TCP + port: {{ .Values.ports.redisDB }} + targetPort: {{ .Values.ports.redisDB }} + {{- if and (eq .Values.redis.service.type "NodePort") (not (empty .Values.redis.service.nodePort)) }} + nodePort: {{ .Values.redis.service.nodePort }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/redis/redis-serviceaccount.yaml b/charts/airflow/templates/redis/redis-serviceaccount.yaml new file mode 100644 index 0000000..f645b4e --- /dev/null +++ b/charts/airflow/templates/redis/redis-serviceaccount.yaml @@ -0,0 +1,41 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +###################################### +## Airflow Redis ServiceAccount +###################################### +{{- if and .Values.redis.enabled .Values.redis.serviceAccount.create (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.redis.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ include "redis.serviceAccountName" . }} + labels: + tier: airflow + component: redis + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.redis.labels }} + {{- mustMerge .Values.redis.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.redis.serviceAccount.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/redis/redis-statefulset.yaml b/charts/airflow/templates/redis/redis-statefulset.yaml new file mode 100644 index 0000000..77f0c7e --- /dev/null +++ b/charts/airflow/templates/redis/redis-statefulset.yaml @@ -0,0 +1,143 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Redis StatefulSet +################################# +{{- if and .Values.redis.enabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }} +{{- $nodeSelector := or .Values.redis.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.redis.affinity .Values.affinity }} +{{- $tolerations := or .Values.redis.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.redis.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $securityContext := include "localPodSecurityContext" .Values.redis }} +{{- $containerSecurityContext := include "externalContainerSecurityContext" .Values.redis }} +{{- $containerLifecycleHooks := .Values.redis.containerLifecycleHooks }} +{{- $persistence := .Values.redis.persistence.enabled }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "airflow.fullname" . }}-redis + labels: + tier: airflow + component: redis + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.redis.annotations }} + annotations: {{- toYaml .Values.redis.annotations | nindent 4 }} + {{- end }} +spec: + serviceName: {{ include "airflow.fullname" . }}-redis + {{- if and $persistence .Values.redis.persistence.persistentVolumeClaimRetentionPolicy }} + persistentVolumeClaimRetentionPolicy: {{- toYaml .Values.redis.persistence.persistentVolumeClaimRetentionPolicy | nindent 4 }} + {{- end }} + selector: + matchLabels: + tier: airflow + component: redis + release: {{ .Release.Name }} + template: + metadata: + labels: + tier: airflow + component: redis + release: {{ .Release.Name }} + {{- if or .Values.labels .Values.redis.labels }} + {{- mustMerge .Values.redis.labels .Values.labels | toYaml | nindent 8 }} + {{- end }} + {{- if or .Values.redis.safeToEvict .Values.redis.podAnnotations }} + annotations: + {{- if .Values.redis.podAnnotations }} + {{- tpl (toYaml .Values.redis.podAnnotations) . | nindent 8 }} + {{- end }} + {{- if .Values.redis.safeToEvict }} + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + {{- end }} + {{- end }} + spec: + {{- if .Values.redis.priorityClassName }} + priorityClassName: {{ .Values.redis.priorityClassName }} + {{- end }} + nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} + affinity: {{- toYaml $affinity | nindent 8 }} + tolerations: {{- toYaml $tolerations | nindent 8 }} + topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.redis.terminationGracePeriodSeconds }} + serviceAccountName: {{ include "redis.serviceAccountName" . }} + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + imagePullSecrets: {{ include "image_pull_secrets" . | nindent 8 }} + securityContext: {{ $securityContext | nindent 8 }} + containers: + - name: redis + image: {{ template "redis_image" . }} + imagePullPolicy: {{ .Values.images.redis.pullPolicy }} + securityContext: {{ $containerSecurityContext | nindent 12 }} + {{- if $containerLifecycleHooks }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }} + {{- end }} + command: ["redis-server"] + resources: {{- toYaml .Values.redis.resources | nindent 12 }} + # Not launching via shell -- parentheses are required + # https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#use-environment-variables-to-define-arguments + args: ["--requirepass", "$(REDIS_PASSWORD)"] + ports: + - name: redis-db + containerPort: {{ .Values.ports.redisDB }} + volumeMounts: + - name: redis-db + mountPath: /data + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "redis_password_secret" . }} + key: password + {{- if not $persistence }} + volumes: + - name: redis-db + emptyDir: {{- toYaml (default (dict) .Values.redis.emptyDirConfig) | nindent 12 }} + {{- else if .Values.redis.persistence.existingClaim }} + volumes: + - name: redis-db + persistentVolumeClaim: + claimName: {{ .Values.redis.persistence.existingClaim }} + {{- else }} + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: redis-db + {{- if .Values.redis.persistence.annotations }} + annotations: {{- toYaml .Values.redis.persistence.annotations | nindent 10 }} + {{- end }} + spec: + {{- if .Values.redis.persistence.storageClassName }} + storageClassName: {{ tpl .Values.redis.persistence.storageClassName . | quote }} + {{- end }} + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.redis.persistence.size }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/resourcequota.yaml b/charts/airflow/templates/resourcequota.yaml new file mode 100644 index 0000000..cd442fe --- /dev/null +++ b/charts/airflow/templates/resourcequota.yaml @@ -0,0 +1,39 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Namespace ResourceQuota +################################# +{{- if .Values.quotas }} +apiVersion: v1 +kind: ResourceQuota +metadata: + name: {{ include "airflow.fullname" . }}-resource-quota + labels: + tier: resources + component: resourcequota + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + hard: {{- toYaml .Values.quotas | nindent 4 }} +{{- end }} diff --git a/charts/airflow/templates/scheduler/scheduler-deployment.yaml b/charts/airflow/templates/scheduler/scheduler-deployment.yaml new file mode 100644 index 0000000..4371b09 --- /dev/null +++ b/charts/airflow/templates/scheduler/scheduler-deployment.yaml @@ -0,0 +1,403 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Scheduler Deployment/StatefulSet +################################# +{{- if .Values.scheduler.enabled }} +{{- $local := contains "Local" .Values.executor }} +# Is persistence enabled on the _workers_? +# This is important because in $local mode, the scheduler assumes the role of the worker +{{- $persistence := or .Values.workers.celery.persistence.enabled (and (not (has .Values.workers.celery.persistence.enabled (list true false))) .Values.workers.persistence.enabled) }} +{{- $stateful := and $local $persistence }} +# We can skip DAGs mounts on scheduler if dagProcessor is enabled, except with $local mode +{{- $dagProcessorEnabled := .Values.dagProcessor.enabled }} +{{- if eq $dagProcessorEnabled nil}} + {{ $dagProcessorEnabled = ternary true false (semverCompare ">=3.0.0" .Values.airflowVersion) }} +{{- end }} +{{- $localOrDagProcessorDisabled := or (not $dagProcessorEnabled) $local }} +{{- $remoteLogging := or .Values.elasticsearch.enabled .Values.opensearch.enabled }} +{{- $nodeSelector := or .Values.scheduler.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.scheduler.affinity .Values.affinity }} +{{- $tolerations := or .Values.scheduler.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.scheduler.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $revisionHistoryLimit := include "airflow.revisionHistoryLimit" (list .Values.scheduler.revisionHistoryLimit .Values.revisionHistoryLimit) }} +{{- $securityContext := include "airflowPodSecurityContext" (list .Values.scheduler .Values) }} +{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.scheduler .Values) }} +{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list .Values.scheduler.waitForMigrations .Values) }} +{{- $containerSecurityContextLogGroomerSidecar := include "containerSecurityContext" (list .Values.scheduler.logGroomerSidecar .Values) }} +{{- $containerLifecycleHooks := or .Values.scheduler.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $containerLifecycleHooksLogGroomerSidecar := or .Values.scheduler.logGroomerSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }} +apiVersion: apps/v1 +kind: {{ if $stateful }}StatefulSet{{ else }}Deployment{{ end }} +metadata: + name: {{ include "airflow.fullname" . }}-scheduler + labels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + executor: {{ .Values.executor | replace "," "-" | trunc 63 | trimSuffix "-" | trimSuffix ":" | trimSuffix "_" | trimSuffix "." | quote }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.scheduler.annotations }} + annotations: {{- toYaml .Values.scheduler.annotations | nindent 4 }} + {{- end }} +spec: + {{- if $stateful }} + serviceName: {{ include "airflow.fullname" . }}-scheduler + {{- end }} + replicas: {{ .Values.scheduler.replicas }} + {{- if ne $revisionHistoryLimit "" }} + revisionHistoryLimit: {{ $revisionHistoryLimit }} + {{- end }} + {{- if and $stateful .Values.scheduler.updateStrategy }} + updateStrategy: {{- toYaml .Values.scheduler.updateStrategy | nindent 4 }} + {{- end }} + {{- if and $stateful (or .Values.workers.celery.persistence.persistentVolumeClaimRetentionPolicy .Values.workers.persistence.persistentVolumeClaimRetentionPolicy) }} + persistentVolumeClaimRetentionPolicy: {{- toYaml (.Values.workers.celery.persistence.persistentVolumeClaimRetentionPolicy | default .Values.workers.persistence.persistentVolumeClaimRetentionPolicy) | nindent 4 }} + {{- end }} + {{- if and (not $stateful) .Values.scheduler.strategy }} + strategy: {{- toYaml .Values.scheduler.strategy | nindent 4 }} + {{- end }} + selector: + matchLabels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + template: + metadata: + labels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + {{- if or .Values.labels .Values.scheduler.labels }} + {{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 8 }} + {{- end }} + annotations: + checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }} + checksum/result-backend-secret: {{ include (print $.Template.BasePath "/secrets/result-backend-connection-secret.yaml") . | sha256sum }} + checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }} + checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }} + checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }} + checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }} + {{- if and (semverCompare ">=3.0.0" .Values.airflowVersion) .Values.apiServer.enabled (not .Values.jwtSecretName) }} + checksum/jwt-secret: {{ include (print $.Template.BasePath "/secrets/jwt-secret.yaml") . | sha256sum }} + {{- end }} + {{- if .Values.scheduler.safeToEvict }} + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + {{- end }} + {{- if .Values.airflowPodAnnotations }} + {{- tpl (toYaml .Values.airflowPodAnnotations) . | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.podAnnotations }} + {{- tpl (toYaml .Values.scheduler.podAnnotations) . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.scheduler.priorityClassName }} + priorityClassName: {{ .Values.scheduler.priorityClassName }} + {{- end }} + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} + affinity: + {{- if $affinity }} + {{- toYaml $affinity | nindent 8 }} + {{- else }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + component: scheduler + topologyKey: kubernetes.io/hostname + weight: 100 + {{- end }} + tolerations: {{- toYaml $tolerations | nindent 8 }} + topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }} + restartPolicy: Always + terminationGracePeriodSeconds: {{ .Values.scheduler.terminationGracePeriodSeconds }} + serviceAccountName: {{ include "scheduler.serviceAccountName" . }} + {{- if and (eq (include "airflow.podLaunchingExecutor" .) "true") (not .Values.scheduler.serviceAccount.automountServiceAccountToken) }} + automountServiceAccountToken: false + {{- end }} + securityContext: {{ $securityContext | nindent 8 }} + imagePullSecrets: {{ include "image_pull_secrets" . | nindent 8 }} + {{- if .Values.scheduler.hostAliases }} + hostAliases: {{- toYaml .Values.scheduler.hostAliases | nindent 8 }} + {{- end }} + initContainers: + {{- if .Values.scheduler.waitForMigrations.enabled }} + - name: wait-for-airflow-migrations + resources: {{- toYaml .Values.scheduler.resources | nindent 12 }} + image: {{ template "airflow_image_for_migrations" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }} + volumeMounts: + - name: logs + mountPath: "/opt/airflow/logs" + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- include "airflow_config_mount" . | nindent 12 }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.scheduler.extraVolumeMounts }} + {{- tpl (toYaml .Values.scheduler.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + {{- include "airflow_webserver_config_mount" . | nindent 12 }} + {{- end }} + args: {{- include "wait-for-migrations-command" . | indent 10 }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- if .Values.scheduler.waitForMigrations.env }} + {{- tpl (toYaml .Values.scheduler.waitForMigrations.env) $ | nindent 12 }} + {{- end }} + {{- end }} + {{- if and $localOrDagProcessorDisabled .Values.dags.gitSync.enabled }} + {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.extraInitContainers }} + {{- tpl (toYaml .Values.scheduler.extraInitContainers) . | nindent 8 }} + {{- end }} + containers: + - name: scheduler + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContext | nindent 12 }} + {{- if $containerLifecycleHooks }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }} + {{- end }} + {{- if .Values.scheduler.command }} + command: {{ tpl (toYaml .Values.scheduler.command) . | nindent 12 }} + {{- end }} + {{- if .Values.scheduler.args }} + args: {{ tpl (toYaml .Values.scheduler.args) . | nindent 12 }} + {{- end }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" true) .) | indent 10 }} + {{- include "container_extra_envs" (list . .Values.scheduler.env) | indent 10 }} + livenessProbe: + initialDelaySeconds: {{ .Values.scheduler.livenessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.scheduler.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.scheduler.livenessProbe.failureThreshold }} + periodSeconds: {{ .Values.scheduler.livenessProbe.periodSeconds }} + exec: + command: + {{- if .Values.scheduler.livenessProbe.command }} + {{- toYaml .Values.scheduler.livenessProbe.command | nindent 16 }} + {{- else }} + {{- include "scheduler_liveness_check_command" . | indent 14 }} + {{- end }} + startupProbe: + initialDelaySeconds: {{ .Values.scheduler.startupProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.scheduler.startupProbe.timeoutSeconds }} + failureThreshold: {{ .Values.scheduler.startupProbe.failureThreshold }} + periodSeconds: {{ .Values.scheduler.startupProbe.periodSeconds }} + exec: + command: + {{- if .Values.scheduler.startupProbe.command }} + {{- toYaml .Values.scheduler.startupProbe.command | nindent 16 }} + {{- else }} + {{- include "scheduler_startup_check_command" . | indent 14 }} + {{- end }} + {{- if and $local (not $remoteLogging) }} + # Serve logs if we're in local mode and we have neither elasticsearch nor opensearch enabled. + ports: + - name: worker-logs + containerPort: {{ .Values.ports.workerLogs }} + {{- end }} + resources: {{- toYaml .Values.scheduler.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: {{ include "airflow_pod_template_file" . }}/pod_template_file.yaml + subPath: pod_template_file.yaml + readOnly: true + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- include "airflow_config_mount" . | nindent 12 }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + {{- include "airflow_webserver_config_mount" . | nindent 12 }} + {{- end }} + {{- if and $localOrDagProcessorDisabled (or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled) }} + {{- include "airflow_dags_mount" . | nindent 12 }} + {{- end }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.scheduler.extraVolumeMounts }} + {{- tpl (toYaml .Values.scheduler.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- if and (eq (include "airflow.podLaunchingExecutor" .) "true") (not .Values.scheduler.serviceAccount.automountServiceAccountToken) .Values.scheduler.serviceAccount.serviceAccountTokenVolume.enabled }} + - name: {{ .Values.scheduler.serviceAccount.serviceAccountTokenVolume.volumeName }} + mountPath: {{ .Values.scheduler.serviceAccount.serviceAccountTokenVolume.mountPath }} + readOnly: true + {{- end }} + {{- if and $localOrDagProcessorDisabled .Values.dags.gitSync.enabled }} + {{- include "git_sync_container" . | indent 8 }} + {{- end }} + {{- if .Values.scheduler.logGroomerSidecar.enabled }} + - name: scheduler-log-groomer + resources: {{- toYaml .Values.scheduler.logGroomerSidecar.resources | nindent 12 }} + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContextLogGroomerSidecar | nindent 12 }} + {{- if $containerLifecycleHooksLogGroomerSidecar }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooksLogGroomerSidecar) . | nindent 12 }} + {{- end }} + {{- if .Values.scheduler.logGroomerSidecar.command }} + command: {{ tpl (toYaml .Values.scheduler.logGroomerSidecar.command) . | nindent 12 }} + {{- end }} + {{- if .Values.scheduler.logGroomerSidecar.args }} + args: {{- tpl (toYaml .Values.scheduler.logGroomerSidecar.args) . | nindent 12 }} + {{- end }} + env: + - name: AIRFLOW_HOME + value: "{{ .Values.airflowHome }}" + {{- if .Values.scheduler.logGroomerSidecar.retentionDays }} + - name: AIRFLOW__LOG_RETENTION_DAYS + value: "{{ .Values.scheduler.logGroomerSidecar.retentionDays }}" + {{- end }} + {{- if .Values.scheduler.logGroomerSidecar.retentionMinutes }} + - name: AIRFLOW__LOG_RETENTION_MINUTES + value: "{{ .Values.scheduler.logGroomerSidecar.retentionMinutes }}" + {{- end }} + {{- if .Values.scheduler.logGroomerSidecar.frequencyMinutes }} + - name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES + value: "{{ .Values.scheduler.logGroomerSidecar.frequencyMinutes }}" + {{- end }} + {{- if .Values.scheduler.logGroomerSidecar.maxSizeBytes }} + - name: AIRFLOW__LOG_MAX_SIZE_BYTES + value: "{{ .Values.scheduler.logGroomerSidecar.maxSizeBytes | int64 }}" + {{- end }} + {{- if .Values.scheduler.logGroomerSidecar.maxSizePercent }} + - name: AIRFLOW__LOG_MAX_SIZE_PERCENT + value: "{{ .Values.scheduler.logGroomerSidecar.maxSizePercent }}" + {{- end }} + {{- if .Values.scheduler.logGroomerSidecar.env }} + {{- tpl (toYaml .Values.scheduler.logGroomerSidecar.env) $ | nindent 12 }} + {{- end }} + volumeMounts: + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.scheduler.extraVolumeMounts }} + {{- tpl (toYaml .Values.scheduler.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + {{- include "airflow_webserver_config_mount" . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.scheduler.extraContainers }} + {{- tpl (toYaml .Values.scheduler.extraContainers) . | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ template "airflow_config" . }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + - name: webserver-config + configMap: + name: {{ template "airflow_webserver_config_configmap_name" . }} + {{- end }} + {{- if $localOrDagProcessorDisabled }} + {{- if .Values.dags.persistence.enabled }} + - name: dags + persistentVolumeClaim: + claimName: {{ template "airflow_dags_volume_claim" . }} + {{- else if .Values.dags.gitSync.enabled }} + - name: dags + emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }} + {{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey}} + {{- include "git_sync_ssh_key_volume" . | indent 8 }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.extraVolumes }} + {{- tpl (toYaml .Values.scheduler.extraVolumes) . | nindent 8 }} + {{- end }} + {{- if and (eq (include "airflow.podLaunchingExecutor" .) "true") (not .Values.scheduler.serviceAccount.automountServiceAccountToken) .Values.scheduler.serviceAccount.serviceAccountTokenVolume.enabled }} + - name: {{ .Values.scheduler.serviceAccount.serviceAccountTokenVolume.volumeName }} + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + {{- if .Values.scheduler.serviceAccount.serviceAccountTokenVolume.audience }} + audience: {{ .Values.scheduler.serviceAccount.serviceAccountTokenVolume.audience }} + {{- end }} + expirationSeconds: {{ .Values.scheduler.serviceAccount.serviceAccountTokenVolume.expirationSeconds }} + path: token + - configMap: + items: + - key: ca.crt + path: ca.crt + name: kube-root-ca.crt + - downwardAPI: + items: + - fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + path: namespace + {{- end }} + {{- if .Values.logs.persistence.enabled }} + - name: logs + persistentVolumeClaim: + claimName: {{ template "airflow_logs_volume_claim" . }} + {{- else if not $stateful }} + - name: logs + emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | nindent 12 }} + {{- else }} + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: logs + {{- if or .Values.workers.celery.persistence.annotations .Values.workers.persistence.annotations }} + annotations: {{- toYaml (.Values.workers.celery.persistence.annotations | default .Values.workers.persistence.annotations) | nindent 10 }} + {{- end }} + spec: + {{- if or .Values.workers.celery.persistence.storageClassName .Values.workers.persistence.storageClassName }} + storageClassName: {{ tpl (.Values.workers.celery.persistence.storageClassName | default .Values.workers.persistence.storageClassName) . | quote }} + {{- end }} + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.workers.celery.persistence.size | default .Values.workers.persistence.size }} + {{- end }} + {{- end }} diff --git a/charts/airflow/templates/scheduler/scheduler-networkpolicy.yaml b/charts/airflow/templates/scheduler/scheduler-networkpolicy.yaml new file mode 100644 index 0000000..37d0b3e --- /dev/null +++ b/charts/airflow/templates/scheduler/scheduler-networkpolicy.yaml @@ -0,0 +1,57 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Scheduler NetworkPolicy +################################# +{{- if and .Values.scheduler.enabled .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "airflow.fullname" . }}-scheduler-policy + labels: + tier: airflow + component: airflow-scheduler-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.scheduler.labels }} + {{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + policyTypes: + - Ingress + {{- if contains "LocalExecutor" .Values.executor }} + ingress: + - from: + - podSelector: + matchLabels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + ports: + - protocol: TCP + port: {{ .Values.ports.workerLogs }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/scheduler/scheduler-poddisruptionbudget.yaml b/charts/airflow/templates/scheduler/scheduler-poddisruptionbudget.yaml new file mode 100644 index 0000000..087d0a9 --- /dev/null +++ b/charts/airflow/templates/scheduler/scheduler-poddisruptionbudget.yaml @@ -0,0 +1,44 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Scheduler PodDisruptionBudget +################################# +{{- if and .Values.scheduler.enabled .Values.scheduler.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "airflow.fullname" . }}-scheduler-pdb + labels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.scheduler.labels }} + {{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + {{- toYaml .Values.scheduler.podDisruptionBudget.config | nindent 2 }} +{{- end }} diff --git a/charts/airflow/templates/scheduler/scheduler-service.yaml b/charts/airflow/templates/scheduler/scheduler-service.yaml new file mode 100644 index 0000000..993d10e --- /dev/null +++ b/charts/airflow/templates/scheduler/scheduler-service.yaml @@ -0,0 +1,48 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Scheduler Service +################################# +{{- if and .Values.scheduler.enabled (or (contains "LocalExecutor" .Values.executor) (contains "LocalKubernetesExecutor" .Values.executor)) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "airflow.fullname" . }}-scheduler + labels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.scheduler.labels }} + {{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + clusterIP: None + selector: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + ports: + - name: task-logs + protocol: TCP + port: {{ .Values.ports.workerLogs }} + targetPort: {{ .Values.ports.workerLogs }} +{{- end }} diff --git a/charts/airflow/templates/scheduler/scheduler-serviceaccount.yaml b/charts/airflow/templates/scheduler/scheduler-serviceaccount.yaml new file mode 100644 index 0000000..0142c36 --- /dev/null +++ b/charts/airflow/templates/scheduler/scheduler-serviceaccount.yaml @@ -0,0 +1,44 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Scheduler ServiceAccount +################################# +{{- if and .Values.scheduler.enabled .Values.scheduler.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +{{- if and (include "airflow.podLaunchingExecutor" .) (not .Values.scheduler.serviceAccount.automountServiceAccountToken) }} +automountServiceAccountToken: false +{{- end }} +metadata: + name: {{ include "scheduler.serviceAccountName" . }} + labels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.scheduler.labels }} + {{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.scheduler.serviceAccount.annotations }} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/secrets/api-secret-key-secret.yaml b/charts/airflow/templates/secrets/api-secret-key-secret.yaml new file mode 100644 index 0000000..d6da362 --- /dev/null +++ b/charts/airflow/templates/secrets/api-secret-key-secret.yaml @@ -0,0 +1,44 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +############################################ +## Airflow Api Flask Secret Key Secret +############################################ +{{- if and (semverCompare ">=3.0.0" .Values.airflowVersion) .Values.apiServer.enabled (not .Values.apiSecretKeySecretName) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "airflow.fullname" . }}-api-secret-key + labels: + tier: airflow + component: api-server + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.apiSecretAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + api-secret-key: {{ (.Values.apiSecretKey) | default (randAlphaNum 32) | b64enc | quote }} +{{- end }} diff --git a/charts/airflow/templates/secrets/elasticsearch-secret.yaml b/charts/airflow/templates/secrets/elasticsearch-secret.yaml new file mode 100644 index 0000000..97d0027 --- /dev/null +++ b/charts/airflow/templates/secrets/elasticsearch-secret.yaml @@ -0,0 +1,49 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Elasticsearch Secret +################################# +{{- if and .Values.elasticsearch.enabled (not .Values.elasticsearch.secretName) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "airflow.fullname" . }}-elasticsearch + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.elasticsearch.secretAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- with .Values.elasticsearch.connection }} + {{- if and .user .pass }} + connection: {{ urlJoin (dict "scheme" (default "http" .scheme) "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery)) "host" (printf "%s:%s" .host ((default 9200 .port) | toString) ) ) | b64enc | quote }} + {{- else }} + connection: {{ urlJoin (dict "scheme" (default "http" .scheme) "host" (printf "%s:%s" .host ((default 9200 .port) | toString))) | b64enc | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/secrets/extra-secrets.yaml b/charts/airflow/templates/secrets/extra-secrets.yaml new file mode 100644 index 0000000..df2f4e8 --- /dev/null +++ b/charts/airflow/templates/secrets/extra-secrets.yaml @@ -0,0 +1,65 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################################# +## Extra Secrets provisioned via the chart values +################################################# +{{- $Global := . }} +{{- range $secretName, $secretContent := .Values.extraSecrets }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ tpl $secretName $Global | quote }} + labels: + tier: airflow + release: {{ $Global.Release.Name }} + chart: "{{ $Global.Chart.Name }}-{{ $Global.Chart.Version }}" + heritage: {{ $Global.Release.Service }} + {{- with $Global.Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if $secretContent.labels }} + {{- toYaml $secretContent.labels | nindent 4 }} + {{- end }} + {{- $annotations := dict }} + {{- if or $secretContent.useHelmHooks (not (hasKey $secretContent "useHelmHooks")) }} + {{- $_ := set $annotations "helm.sh/hook" "pre-install,pre-upgrade" }} + {{- $_ := set $annotations "helm.sh/hook-weight" "0" }} + {{- $_ := set $annotations "helm.sh/hook-delete-policy" "before-hook-creation" }} + {{- end }} + {{- with $annotations := merge $annotations ($secretContent.annotations | default dict) }} + annotations: {{- $annotations | toYaml | nindent 4 }} + {{- end }} +{{- if $secretContent.type }} +type: {{ $secretContent.type }} +{{- end }} +{{- if $secretContent.data }} +data: + {{- with $secretContent.data }} + {{- tpl . $Global | nindent 2 }} + {{- end }} +{{- end }} +{{- if $secretContent.stringData }} +stringData: + {{- with $secretContent.stringData }} + {{- tpl . $Global | nindent 2 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/airflow/templates/secrets/fernetkey-secret.yaml b/charts/airflow/templates/secrets/fernetkey-secret.yaml new file mode 100644 index 0000000..0c1f2dc --- /dev/null +++ b/charts/airflow/templates/secrets/fernetkey-secret.yaml @@ -0,0 +1,48 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Fernet Key Secret +################################# +{{- if not .Values.fernetKeySecretName }} +# Fernet key value must be b64enc +{{- $generated_fernet_key := (randAlphaNum 32 | b64enc) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "airflow.fullname" . }}-fernet-key + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + "helm.sh/hook": "pre-install" + "helm.sh/hook-delete-policy": "before-hook-creation" + "helm.sh/hook-weight": "0" + {{- with .Values.fernetKeySecretAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + fernet-key: {{ (default $generated_fernet_key .Values.fernetKey) | b64enc | quote }} +{{- end }} diff --git a/charts/airflow/templates/secrets/flower-secret.yaml b/charts/airflow/templates/secrets/flower-secret.yaml new file mode 100644 index 0000000..6639588 --- /dev/null +++ b/charts/airflow/templates/secrets/flower-secret.yaml @@ -0,0 +1,44 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Flower Secret +################################# +{{- if and .Values.flower.enabled (not .Values.flower.secretName) .Values.flower.username .Values.flower.password }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "airflow.fullname" . }}-flower + labels: + tier: airflow + component: flower + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.flower.secretAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + basicAuth: {{ (printf "%s:%s" .Values.flower.username .Values.flower.password) | b64enc | quote }} +{{- end }} diff --git a/charts/airflow/templates/secrets/git-ssh-key-secret.yaml b/charts/airflow/templates/secrets/git-ssh-key-secret.yaml new file mode 100644 index 0000000..d912f6b --- /dev/null +++ b/charts/airflow/templates/secrets/git-ssh-key-secret.yaml @@ -0,0 +1,35 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKey }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "git_sync_ssh_key" . }} + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +data: + gitSshKey: {{ .Values.dags.gitSync.sshKey | b64enc | quote }} +{{- end }} diff --git a/charts/airflow/templates/secrets/jwt-secret.yaml b/charts/airflow/templates/secrets/jwt-secret.yaml new file mode 100644 index 0000000..288a549 --- /dev/null +++ b/charts/airflow/templates/secrets/jwt-secret.yaml @@ -0,0 +1,44 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +############################################ +## Airflow JWT Secret +############################################ +{{- if and (semverCompare ">=3.0.0" .Values.airflowVersion) .Values.apiServer.enabled (not .Values.jwtSecretName) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "jwt_secret" . }} + labels: + tier: airflow + component: api-server + release: {{ .Release.Name }} + chart: {{ .Chart.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.jwtSecretAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + jwt-secret: {{ .Values.jwtSecret | default (randAlphaNum 128) | b64enc | quote }} +{{- end }} diff --git a/charts/airflow/templates/secrets/kerberos-keytab-secret.yaml b/charts/airflow/templates/secrets/kerberos-keytab-secret.yaml new file mode 100644 index 0000000..cf1bc3c --- /dev/null +++ b/charts/airflow/templates/secrets/kerberos-keytab-secret.yaml @@ -0,0 +1,40 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Kerberos Secret +################################# +{{- if and .Values.kerberos.enabled .Values.kerberos.keytabBase64Content }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "kerberos_keytab_secret" . | quote }} + labels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + kerberos.keytab: {{ .Values.kerberos.keytabBase64Content }} +{{- end }} diff --git a/charts/airflow/templates/secrets/metadata-connection-secret.yaml b/charts/airflow/templates/secrets/metadata-connection-secret.yaml new file mode 100644 index 0000000..7144175 --- /dev/null +++ b/charts/airflow/templates/secrets/metadata-connection-secret.yaml @@ -0,0 +1,70 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Metadata Secret +################################# +{{- if not .Values.data.metadataSecretName }} +{{- $defaultMetadataHost := .Values.postgresql.nameOverride | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }} +{{- $metadataHost := .Values.data.metadataConnection.host | default $defaultMetadataHost }} +{{- $pgbouncerHost := (printf "%s-%s.%s" ( include "airflow.fullname" . ) "pgbouncer" .Release.Namespace) }} +{{- $host := ternary $pgbouncerHost $metadataHost .Values.pgbouncer.enabled }} +{{- $metadataPort := .Values.data.metadataConnection.port | toString }} +{{- $port := (ternary .Values.ports.pgbouncer $metadataPort .Values.pgbouncer.enabled) | toString }} +{{- $metadataUser := tpl .Values.data.metadataConnection.user . }} +{{- $metadataDatabase := tpl .Values.data.metadataConnection.db . }} +{{- $database := ternary (printf "%s-%s" .Release.Name "metadata") $metadataDatabase .Values.pgbouncer.enabled }} +{{- $query := ternary (printf "sslmode=%s" .Values.data.metadataConnection.sslmode) "" (eq .Values.data.metadataConnection.protocol "postgresql") }} +{{- $kedaEnabled := .Values.workers.keda.enabled }} +{{- $kedaUsePgBouncer := .Values.workers.keda.usePgbouncer }} +{{- if hasKey .Values.workers "celery" }} + {{- $kedaEnabled = or .Values.workers.celery.keda.enabled (and (not (has .Values.workers.celery.keda.enabled (list true false))) .Values.workers.keda.enabled) }} + {{- $kedaUsePgBouncer = or .Values.workers.celery.keda.usePgbouncer (and (not (has .Values.workers.celery.keda.usePgbouncer (list true false))) .Values.workers.keda.usePgbouncer) }} +{{- end }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "airflow.fullname" . }}-metadata + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.data.metadataConnection.secretAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- with .Values.data.metadataConnection }} + connection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" ($metadataUser | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $host $port) "path" (printf "/%s" $database) "query" $query) | b64enc | quote }} + {{- end }} + {{- if and $kedaEnabled .Values.pgbouncer.enabled (not $kedaUsePgBouncer) }} + {{- with .Values.data.metadataConnection }} + kedaConnection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" ($metadataUser | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | b64enc | quote }} + {{- end }} + {{- else if and (or $kedaEnabled .Values.triggerer.keda.enabled) (eq .Values.data.metadataConnection.protocol "mysql") }} + {{- with .Values.data.metadataConnection }} + kedaConnection: {{ urlJoin (dict "userinfo" (printf "%s:%s" ($metadataUser | urlquery) (.pass | urlquery) ) "host" (printf "tcp(%s:%s)" $metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | trimPrefix "//" | b64enc | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/secrets/opensearch-secret.yaml b/charts/airflow/templates/secrets/opensearch-secret.yaml new file mode 100644 index 0000000..b90b4c2 --- /dev/null +++ b/charts/airflow/templates/secrets/opensearch-secret.yaml @@ -0,0 +1,45 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## OpenSearch Secret +################################# +{{- if and .Values.opensearch.enabled (not .Values.opensearch.secretName) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "airflow.fullname" . }}-opensearch + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- with .Values.opensearch.connection }} + {{- if and .user .pass }} + connection: {{ urlJoin (dict "scheme" (default "http" .scheme) "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery)) "host" (printf "%s:%s" .host ((default 9200 .port) | toString) ) ) | b64enc | quote }} + {{- else }} + connection: {{ urlJoin (dict "scheme" (default "http" .scheme) "host" (printf "%s:%s" .host ((default 9200 .port) | toString))) | b64enc | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/secrets/pgbouncer-certificates-secret.yaml b/charts/airflow/templates/secrets/pgbouncer-certificates-secret.yaml new file mode 100644 index 0000000..e826d16 --- /dev/null +++ b/charts/airflow/templates/secrets/pgbouncer-certificates-secret.yaml @@ -0,0 +1,52 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Pgbouncer Certificate Secret +################################# +{{- if and .Values.pgbouncer.enabled (or .Values.pgbouncer.ssl.ca .Values.pgbouncer.ssl.cert .Values.pgbouncer.ssl.key) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "pgbouncer_certificates_secret" . }} + labels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.pgbouncer.certificatesSecretAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- if .Values.pgbouncer.ssl.ca }} + root.crt: {{ .Values.pgbouncer.ssl.ca | b64enc }} + {{- end }} + {{- if .Values.pgbouncer.ssl.cert }} + server.crt: {{ .Values.pgbouncer.ssl.cert | b64enc }} + {{- end }} + {{- if .Values.pgbouncer.ssl.key }} + server.key: {{ .Values.pgbouncer.ssl.key | b64enc }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/secrets/pgbouncer-config-secret.yaml b/charts/airflow/templates/secrets/pgbouncer-config-secret.yaml new file mode 100644 index 0000000..4e18b91 --- /dev/null +++ b/charts/airflow/templates/secrets/pgbouncer-config-secret.yaml @@ -0,0 +1,45 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Pgbouncer Config Secret +################################# +{{- if and .Values.pgbouncer.enabled (not .Values.pgbouncer.configSecretName) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "pgbouncer_config_secret" . }} + labels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.pgbouncer.configSecretAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + pgbouncer.ini: {{ include "pgbouncer_config" . | b64enc }} + users.txt: {{ include "pgbouncer_users" . | b64enc }} +{{- end }} diff --git a/charts/airflow/templates/secrets/pgbouncer-stats-secret.yaml b/charts/airflow/templates/secrets/pgbouncer-stats-secret.yaml new file mode 100644 index 0000000..a6897fd --- /dev/null +++ b/charts/airflow/templates/secrets/pgbouncer-stats-secret.yaml @@ -0,0 +1,44 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Pgbouncer Stats Secret +################################# +{{- if and .Values.pgbouncer.enabled (not .Values.pgbouncer.metricsExporterSidecar.statsSecretName) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "pgbouncer_stats_secret" . }} + labels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.pgbouncer.metricsExporterSidecar.statsSecretAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + connection: {{ urlJoin (dict "scheme" "postgresql" "userinfo" (printf "%s:%s" (.Values.data.metadataConnection.user | urlquery) (.Values.data.metadataConnection.pass | urlquery) ) "host" (printf "127.0.0.1:%s" (.Values.ports.pgbouncer | toString)) "path" "/pgbouncer" "query" (printf "sslmode=%s" (.Values.pgbouncer.metricsExporterSidecar.sslmode | toString ))) | b64enc | quote }} +{{- end }} diff --git a/charts/airflow/templates/secrets/redis-secrets.yaml b/charts/airflow/templates/secrets/redis-secrets.yaml new file mode 100644 index 0000000..3d248ea --- /dev/null +++ b/charts/airflow/templates/secrets/redis-secrets.yaml @@ -0,0 +1,89 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +# We will create these secrets (if necessary) _even if_ we aren't +# currently using CeleryExecutor or CeleryKubernetesExecutor. As we are +# relying on the "pre-install" hack to prevent changing randomly generated passwords, +# updating the executor later doesn't give us the opportunity to deploy them +# when we need them. We will always deploy them defensively to make the executor +# update path actually work. + +################################ +## Airflow Redis Password Secret +################################# +{{- $random_redis_password := randAlphaNum 10 }} +{{- if and .Values.redis.enabled (not .Values.redis.passwordSecretName) }} +# If passwordSecretName is not set, we will either use the set password, or use the generated one +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "redis_password_secret" . }} + labels: + tier: airflow + component: redis + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + "helm.sh/hook": "pre-install" + "helm.sh/hook-delete-policy": "before-hook-creation" + "helm.sh/hook-weight": "0" + {{- with .Values.redis.passwordSecretAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + password: {{ (default $random_redis_password .Values.redis.password) | b64enc | quote }} +--- +{{- end }} +{{- if not .Values.data.brokerUrlSecretName }} +################################## +## Airflow Redis Connection Secret +################################## +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "airflow_broker_url_secret" . }} + labels: + tier: airflow + component: redis + release: {{ .Release.Name }} + chart: {{ .Chart.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + "helm.sh/hook": "pre-install" + "helm.sh/hook-delete-policy": "before-hook-creation" + "helm.sh/hook-weight": "0" + {{- with .Values.data.brokerUrlSecretAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- if .Values.redis.enabled }} + connection: {{ urlJoin (dict "scheme" "redis" "userinfo" (printf ":%s" ((default $random_redis_password .Values.redis.password) | urlquery)) "host" (printf "%s-redis:6379" (include "airflow.fullname" .) ) "path" "/0") | b64enc | quote }} + {{- else }} + connection: {{ (printf "%s" .Values.data.brokerUrl) | b64enc | quote }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/secrets/registry-secret.yaml b/charts/airflow/templates/secrets/registry-secret.yaml new file mode 100644 index 0000000..334e8b9 --- /dev/null +++ b/charts/airflow/templates/secrets/registry-secret.yaml @@ -0,0 +1,39 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Registry Secret +################################# +{{- if and .Values.registry.connection (not (or .Values.registry.secretName .Values.imagePullSecrets)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "airflow.fullname" . }}-registry + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: {{ include "registry_docker_config" . | b64enc }} +{{- end }} diff --git a/charts/airflow/templates/secrets/result-backend-connection-secret.yaml b/charts/airflow/templates/secrets/result-backend-connection-secret.yaml new file mode 100644 index 0000000..e05f3fb --- /dev/null +++ b/charts/airflow/templates/secrets/result-backend-connection-secret.yaml @@ -0,0 +1,50 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Result Backend Secret +################################# +{{- if and .Values.data.resultBackendConnection (not .Values.data.resultBackendSecretName) (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }} +{{- $connection := .Values.data.resultBackendConnection | default .Values.data.metadataConnection }} +{{- $resultBackendHost := $connection.host | default (printf "%s-%s" .Release.Name "postgresql") }} +{{- $pgbouncerHost := printf "%s-%s" .Release.Name "pgbouncer" }} +{{- $host := ternary $pgbouncerHost $resultBackendHost .Values.pgbouncer.enabled }} +{{- $port := (ternary .Values.ports.pgbouncer $connection.port .Values.pgbouncer.enabled) | toString }} +{{- $database := ternary (printf "%s-%s" .Release.Name "result-backend") $connection.db .Values.pgbouncer.enabled }} +{{- $query := ternary (printf "sslmode=%s" $connection.sslmode) "" (eq $connection.protocol "postgresql") }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "airflow.fullname" . }}-result-backend + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.data.resultBackendConnectionSecretAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + connection: {{ urlJoin (dict "scheme" (printf "db+%s" $connection.protocol) "userinfo" (printf "%s:%s" ($connection.user|urlquery) ($connection.pass | urlquery)) "host" (printf "%s:%s" $host $port) "path" (printf "/%s" $database) "query" $query) | b64enc | quote }} +{{- end }} diff --git a/charts/airflow/templates/secrets/webserver-secret-key-secret.yaml b/charts/airflow/templates/secrets/webserver-secret-key-secret.yaml new file mode 100644 index 0000000..79b0d98 --- /dev/null +++ b/charts/airflow/templates/secrets/webserver-secret-key-secret.yaml @@ -0,0 +1,44 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +############################################ +## Airflow Webserver Flask Secret Key Secret +############################################ +{{- if and (semverCompare "<3.0.0" .Values.airflowVersion) (not .Values.webserverSecretKeySecretName) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "airflow.fullname" . }}-webserver-secret-key + labels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.webserverSecretAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + webserver-secret-key: {{ (.Values.webserverSecretKey) | default (randAlphaNum 32) | b64enc | quote }} +{{- end }} diff --git a/charts/airflow/templates/statsd/statsd-deployment.yaml b/charts/airflow/templates/statsd/statsd-deployment.yaml new file mode 100644 index 0000000..0b21999 --- /dev/null +++ b/charts/airflow/templates/statsd/statsd-deployment.yaml @@ -0,0 +1,144 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow StatsD Deployment +################################# +{{- if .Values.statsd.enabled }} +{{- $nodeSelector := or .Values.statsd.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.statsd.affinity .Values.affinity }} +{{- $tolerations := or .Values.statsd.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.statsd.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $revisionHistoryLimit := include "airflow.revisionHistoryLimit" (list .Values.statsd.revisionHistoryLimit .Values.revisionHistoryLimit) }} +{{- $securityContext := include "localPodSecurityContext" .Values.statsd }} +{{- $containerSecurityContext := include "externalContainerSecurityContext" .Values.statsd }} +{{- $containerLifecycleHooks := .Values.statsd.containerLifecycleHooks }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "airflow.fullname" . }}-statsd + labels: + tier: airflow + component: statsd + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.statsd.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: 1 + {{- if ne $revisionHistoryLimit "" }} + revisionHistoryLimit: {{ $revisionHistoryLimit }} + {{- end }} + selector: + matchLabels: + tier: airflow + component: statsd + release: {{ .Release.Name }} + template: + metadata: + labels: + tier: airflow + component: statsd + release: {{ .Release.Name }} + {{- if or .Values.labels .Values.statsd.labels }} + {{- mustMerge .Values.statsd.labels .Values.labels | toYaml | nindent 8 }} + {{- end }} + {{- if or .Values.statsd.extraMappings .Values.statsd.podAnnotations }} + annotations: + checksum/statsd-config: {{ include (print $.Template.BasePath "/configmaps/statsd-configmap.yaml") . | sha256sum }} + {{- if .Values.statsd.podAnnotations }} + {{- tpl (toYaml .Values.statsd.podAnnotations) . | nindent 8 }} + {{- end }} + {{- end }} + spec: + {{- if .Values.statsd.priorityClassName }} + priorityClassName: {{ .Values.statsd.priorityClassName }} + {{- end }} + nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} + affinity: {{- toYaml $affinity | nindent 8 }} + tolerations: {{- toYaml $tolerations | nindent 8 }} + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.statsd.terminationGracePeriodSeconds }} + serviceAccountName: {{ include "statsd.serviceAccountName" . }} + securityContext: {{ $securityContext | nindent 8 }} + restartPolicy: Always + imagePullSecrets: {{ include "image_pull_secrets" . | nindent 8 }} + containers: + - name: statsd + image: {{ template "statsd_image" . }} + imagePullPolicy: {{ .Values.images.statsd.pullPolicy }} + securityContext: {{ $containerSecurityContext | nindent 12 }} + {{- if $containerLifecycleHooks }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }} + {{- end }} + args: + {{- if .Values.statsd.cache.size }} + - "--statsd.cache-size={{ .Values.statsd.cache.size }}" + {{- end }} + {{- if .Values.statsd.cache.type }} + - "--statsd.cache-type={{ .Values.statsd.cache.type }}" + {{- end }} + {{- if .Values.statsd.args }} + {{- range $arg := .Values.statsd.args }} + - {{ $arg | quote }} + {{- end }} + {{- else }} + - "--statsd.mapping-config=/etc/statsd-exporter/mappings.yml" + {{- end }} + resources: {{- toYaml .Values.statsd.resources | nindent 12 }} + {{- with .Values.statsd.env }} + env: {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: statsd-ingest + protocol: UDP + containerPort: {{ .Values.ports.statsdIngest }} + - name: statsd-scrape + containerPort: {{ .Values.ports.statsdScrape }} + livenessProbe: + httpGet: + path: /metrics + port: {{ .Values.ports.statsdScrape }} + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /metrics + port: {{ .Values.ports.statsdScrape }} + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: config + mountPath: /etc/statsd-exporter + readOnly: true + volumes: + - name: config + configMap: + name: {{ include "airflow.fullname" . }}-statsd +{{- end }} diff --git a/charts/airflow/templates/statsd/statsd-ingress.yaml b/charts/airflow/templates/statsd/statsd-ingress.yaml new file mode 100644 index 0000000..14386d6 --- /dev/null +++ b/charts/airflow/templates/statsd/statsd-ingress.yaml @@ -0,0 +1,85 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Statsd Ingress +################################# +{{- if and .Values.statsd.enabled .Values.ingress.statsd.enabled }} +{{- $fullname := include "airflow.fullname" . }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullname }}-statsd-ingress + labels: + tier: airflow + component: statsd-ingress + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.statsd.labels }} + {{- mustMerge .Values.statsd.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.ingress.statsd.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.statsd.hosts (.Values.ingress.statsd.hosts | first | kindIs "string" | not) }} + {{- $anyTlsHosts := false -}} + {{- range .Values.ingress.statsd.hosts }} + {{- if and .tls .tls.enabled }} + {{- $anyTlsHosts = true -}} + {{- break }} + {{- end }} + {{- end }} + {{- if $anyTlsHosts }} + tls: + {{- range .Values.ingress.statsd.hosts }} + {{- if and .tls .tls.enabled }} + - hosts: + - {{ tpl .name $ | quote }} + secretName: {{ .tls.secretName }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.statsd.hosts | default (list .Values.ingress.statsd.host) }} + - http: + paths: + - backend: + service: + name: {{ $fullname }}-statsd + port: + name: statsd-scrape + {{- if $.Values.ingress.statsd.path }} + path: {{ $.Values.ingress.statsd.path }} + pathType: {{ $.Values.ingress.statsd.pathType }} + {{- end }} + {{- $hostname := . -}} + {{- if . | kindIs "string" | not }} + {{- $hostname = .name -}} + {{- end }} + {{- if $hostname }} + host: {{ tpl $hostname $ | quote }} + {{- end }} + {{- end }} + {{- if .Values.ingress.statsd.ingressClassName }} + ingressClassName: {{ .Values.ingress.statsd.ingressClassName }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/statsd/statsd-networkpolicy.yaml b/charts/airflow/templates/statsd/statsd-networkpolicy.yaml new file mode 100644 index 0000000..9d4ccde --- /dev/null +++ b/charts/airflow/templates/statsd/statsd-networkpolicy.yaml @@ -0,0 +1,59 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow StatsD NetworkPolicy +################################# +{{- if and .Values.networkPolicies.enabled .Values.statsd.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "airflow.fullname" . }}-statsd-policy + labels: + tier: airflow + component: statsd-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.statsd.labels }} + {{- mustMerge .Values.statsd.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: statsd + release: {{ .Release.Name }} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + tier: airflow + release: {{ .Release.Name }} + {{- if .Values.statsd.extraNetworkPolicies }} + {{- toYaml .Values.statsd.extraNetworkPolicies | nindent 4 }} + {{- end }} + ports: + - protocol: UDP + port: {{ .Values.ports.statsdIngest }} + - protocol: TCP + port: {{ .Values.ports.statsdScrape }} +{{- end }} diff --git a/charts/airflow/templates/statsd/statsd-service.yaml b/charts/airflow/templates/statsd/statsd-service.yaml new file mode 100644 index 0000000..c7bee0e --- /dev/null +++ b/charts/airflow/templates/statsd/statsd-service.yaml @@ -0,0 +1,58 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow StatsD Service +################################# +{{- if .Values.statsd.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "airflow.fullname" . }}-statsd + labels: + tier: airflow + component: statsd + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.statsd.labels }} + {{- mustMerge .Values.statsd.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.ports.statsdScrape | quote }} + {{- if .Values.statsd.service.extraAnnotations }} + {{- toYaml .Values.statsd.service.extraAnnotations | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + selector: + tier: airflow + component: statsd + release: {{ .Release.Name }} + ports: + - name: statsd-ingest + protocol: UDP + port: {{ .Values.ports.statsdIngest }} + targetPort: {{ .Values.ports.statsdIngest }} + - name: statsd-scrape + protocol: TCP + port: {{ .Values.ports.statsdScrape }} + targetPort: {{ .Values.ports.statsdScrape }} +{{- end }} diff --git a/charts/airflow/templates/statsd/statsd-serviceaccount.yaml b/charts/airflow/templates/statsd/statsd-serviceaccount.yaml new file mode 100644 index 0000000..e265976 --- /dev/null +++ b/charts/airflow/templates/statsd/statsd-serviceaccount.yaml @@ -0,0 +1,41 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +###################################### +## Airflow StatsD ServiceAccount +###################################### +{{- if and .Values.statsd.enabled .Values.statsd.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.statsd.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ include "statsd.serviceAccountName" . }} + labels: + tier: airflow + component: statsd + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.statsd.labels }} + {{- mustMerge .Values.statsd.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.statsd.serviceAccount.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/triggerer/triggerer-deployment.yaml b/charts/airflow/templates/triggerer/triggerer-deployment.yaml new file mode 100644 index 0000000..fa7a90c --- /dev/null +++ b/charts/airflow/templates/triggerer/triggerer-deployment.yaml @@ -0,0 +1,339 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Triggerer Deployment +################################# +{{- if .Values.triggerer.enabled }} +{{- $persistence := .Values.triggerer.persistence.enabled }} +{{- $keda := .Values.triggerer.keda.enabled }} +{{- $nodeSelector := or .Values.triggerer.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.triggerer.affinity .Values.affinity }} +{{- $tolerations := or .Values.triggerer.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.triggerer.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $revisionHistoryLimit := include "airflow.revisionHistoryLimit" (list .Values.triggerer.revisionHistoryLimit .Values.revisionHistoryLimit) }} +{{- $securityContext := include "airflowPodSecurityContext" (list .Values.triggerer .Values) }} +{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.triggerer .Values) }} +{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list .Values.triggerer.waitForMigrations .Values) }} +{{- $containerSecurityContextLogGroomer := include "containerSecurityContext" (list .Values.triggerer.logGroomerSidecar .Values) }} +{{- $containerLifecycleHooks := or .Values.triggerer.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $containerLifecycleHooksLogGroomerSidecar := or .Values.triggerer.logGroomerSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }} +apiVersion: apps/v1 +kind: {{ if $persistence }}StatefulSet{{ else }}Deployment{{ end }} +metadata: + name: {{ include "airflow.fullname" . }}-triggerer + labels: + tier: airflow + component: triggerer + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.triggerer.annotations }} + annotations: {{- toYaml .Values.triggerer.annotations | nindent 4 }} + {{- end }} +spec: + {{- if $persistence }} + serviceName: {{ include "airflow.fullname" . }}-triggerer + {{- end }} + {{- if not $keda }} + replicas: {{ .Values.triggerer.replicas }} + {{- end }} + {{- if ne $revisionHistoryLimit "" }} + revisionHistoryLimit: {{ $revisionHistoryLimit }} + {{- end }} + selector: + matchLabels: + tier: airflow + component: triggerer + release: {{ .Release.Name }} + {{- if and $persistence .Values.triggerer.updateStrategy }} + updateStrategy: {{- toYaml .Values.triggerer.updateStrategy | nindent 4 }} + {{- end }} + {{- if and (not $persistence) .Values.triggerer.strategy }} + strategy: {{- toYaml .Values.triggerer.strategy | nindent 4 }} + {{- end }} + {{- if and $persistence .Values.triggerer.persistence.persistentVolumeClaimRetentionPolicy }} + persistentVolumeClaimRetentionPolicy: {{- toYaml .Values.triggerer.persistence.persistentVolumeClaimRetentionPolicy | nindent 4 }} + {{- end }} + template: + metadata: + labels: + tier: airflow + component: triggerer + release: {{ .Release.Name }} + {{- if or .Values.labels .Values.triggerer.labels }} + {{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 8 }} + {{- end }} + annotations: + checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }} + checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }} + checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }} + checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }} + checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }} + {{- if .Values.triggerer.safeToEvict }} + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + {{- end }} + {{- if .Values.airflowPodAnnotations }} + {{- tpl (toYaml .Values.airflowPodAnnotations) . | nindent 8 }} + {{- end }} + {{- if .Values.triggerer.podAnnotations }} + {{- tpl (toYaml .Values.triggerer.podAnnotations) . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.triggerer.priorityClassName }} + priorityClassName: {{ .Values.triggerer.priorityClassName }} + {{- end }} + nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + affinity: + {{- if $affinity }} + {{- toYaml $affinity | nindent 8 }} + {{- else }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + component: triggerer + topologyKey: kubernetes.io/hostname + weight: 100 + {{- end }} + tolerations: {{- toYaml $tolerations | nindent 8 }} + topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }} + {{- if .Values.triggerer.hostAliases }} + hostAliases: {{- toYaml .Values.triggerer.hostAliases | nindent 8 }} + {{- end }} + terminationGracePeriodSeconds: {{ .Values.triggerer.terminationGracePeriodSeconds }} + restartPolicy: Always + serviceAccountName: {{ include "triggerer.serviceAccountName" . }} + securityContext: {{ $securityContext | nindent 8 }} + imagePullSecrets: {{ include "image_pull_secrets" . | nindent 8 }} + initContainers: + {{- if .Values.triggerer.waitForMigrations.enabled }} + - name: wait-for-airflow-migrations + resources: + {{- toYaml .Values.triggerer.resources | nindent 12 }} + image: {{ template "airflow_image_for_migrations" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }} + volumeMounts: + - name: logs + mountPath: "/opt/airflow/logs" + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- include "airflow_config_mount" . | nindent 12 }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.triggerer.extraVolumeMounts }} + {{- tpl (toYaml .Values.triggerer.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + {{- include "airflow_webserver_config_mount" . | nindent 12 }} + {{- end }} + args: {{- include "wait-for-migrations-command" . | indent 10 }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- if .Values.triggerer.waitForMigrations.env }} + {{- tpl (toYaml .Values.triggerer.waitForMigrations.env) $ | nindent 12 }} + {{- end }} + {{- end }} + {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }} + {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }} + {{- end }} + {{- if .Values.triggerer.extraInitContainers }} + {{- tpl (toYaml .Values.triggerer.extraInitContainers) . | nindent 8 }} + {{- end }} + containers: + - name: triggerer + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContext | nindent 12 }} + {{- if $containerLifecycleHooks }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }} + {{- end }} + {{- if .Values.triggerer.command }} + command: {{ tpl (toYaml .Values.triggerer.command) . | nindent 12 }} + {{- end }} + {{- if .Values.triggerer.args }} + args: {{ tpl (toYaml .Values.triggerer.args) . | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.triggerer.resources | nindent 12 }} + volumeMounts: + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.triggerer.extraVolumeMounts }} + {{- tpl (toYaml .Values.triggerer.extraVolumeMounts) . | nindent 12 }} + {{- end }} + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- include "airflow_config_mount" . | nindent 12 }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + {{- include "airflow_webserver_config_mount" . | nindent 12 }} + {{- end }} + {{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }} + {{- include "airflow_dags_mount" . | nindent 12 }} + {{- end }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- include "container_extra_envs" (list . .Values.triggerer.env) | nindent 10 }} + livenessProbe: + initialDelaySeconds: {{ .Values.triggerer.livenessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.triggerer.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.triggerer.livenessProbe.failureThreshold }} + periodSeconds: {{ .Values.triggerer.livenessProbe.periodSeconds }} + exec: + command: + {{- if .Values.triggerer.livenessProbe.command }} + {{- toYaml .Values.triggerer.livenessProbe.command | nindent 16 }} + {{- else }} + {{- include "triggerer_liveness_check_command" . | indent 14 }} + {{- end }} + ports: + - name: triggerer-logs + containerPort: {{ .Values.ports.triggererLogs }} + {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }} + {{- include "git_sync_container" . | nindent 8 }} + {{- end }} + {{- if .Values.triggerer.logGroomerSidecar.enabled }} + - name: triggerer-log-groomer + resources: {{- toYaml .Values.triggerer.logGroomerSidecar.resources | nindent 12 }} + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContextLogGroomer | nindent 12 }} + {{- if $containerLifecycleHooksLogGroomerSidecar }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooksLogGroomerSidecar) . | nindent 12 }} + {{- end }} + {{- if .Values.triggerer.logGroomerSidecar.command }} + command: {{ tpl (toYaml .Values.triggerer.logGroomerSidecar.command) . | nindent 12 }} + {{- end }} + {{- if .Values.triggerer.logGroomerSidecar.args }} + args: {{- tpl (toYaml .Values.triggerer.logGroomerSidecar.args) . | nindent 12 }} + {{- end }} + env: + {{- if .Values.triggerer.logGroomerSidecar.retentionDays }} + - name: AIRFLOW__LOG_RETENTION_DAYS + value: "{{ .Values.triggerer.logGroomerSidecar.retentionDays }}" + {{- end }} + {{- if .Values.triggerer.logGroomerSidecar.retentionMinutes }} + - name: AIRFLOW__LOG_RETENTION_MINUTES + value: "{{ .Values.triggerer.logGroomerSidecar.retentionMinutes }}" + {{- end }} + {{- if .Values.triggerer.logGroomerSidecar.frequencyMinutes }} + - name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES + value: "{{ .Values.triggerer.logGroomerSidecar.frequencyMinutes }}" + {{- end }} + {{- if .Values.triggerer.logGroomerSidecar.maxSizeBytes }} + - name: AIRFLOW__LOG_MAX_SIZE_BYTES + value: "{{ .Values.triggerer.logGroomerSidecar.maxSizeBytes | int64 }}" + {{- end }} + {{- if .Values.triggerer.logGroomerSidecar.maxSizePercent }} + - name: AIRFLOW__LOG_MAX_SIZE_PERCENT + value: "{{ .Values.triggerer.logGroomerSidecar.maxSizePercent }}" + {{- end }} + - name: AIRFLOW_HOME + value: "{{ .Values.airflowHome }}" + {{- if .Values.triggerer.logGroomerSidecar.env }} + {{- tpl (toYaml .Values.triggerer.logGroomerSidecar.env) $ | nindent 12 }} + {{- end }} + volumeMounts: + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.triggerer.extraVolumeMounts }} + {{- tpl (toYaml .Values.triggerer.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + {{- include "airflow_webserver_config_mount" . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.triggerer.extraContainers }} + {{- tpl (toYaml .Values.triggerer.extraContainers) . | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ template "airflow_config" . }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + - name: webserver-config + configMap: + name: {{ template "airflow_webserver_config_configmap_name" . }} + {{- end }} + {{- if .Values.dags.persistence.enabled }} + - name: dags + persistentVolumeClaim: + claimName: {{ template "airflow_dags_volume_claim" . }} + {{- else if .Values.dags.gitSync.enabled }} + - name: dags + emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }} + {{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey}} + {{- include "git_sync_ssh_key_volume" . | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 8 }} + {{- end }} + {{- if .Values.triggerer.extraVolumes }} + {{- tpl (toYaml .Values.triggerer.extraVolumes) . | nindent 8 }} + {{- end }} + {{- if .Values.logs.persistence.enabled }} + - name: logs + persistentVolumeClaim: + claimName: {{ template "airflow_logs_volume_claim" . }} + {{- else if not $persistence }} + - name: logs + emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | nindent 12 }} + {{- else }} + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: logs + {{- if .Values.triggerer.persistence.annotations }} + annotations: {{- toYaml .Values.triggerer.persistence.annotations | nindent 10 }} + {{- end }} + spec: + {{- if .Values.triggerer.persistence.storageClassName }} + storageClassName: {{ tpl .Values.triggerer.persistence.storageClassName . | quote }} + {{- end }} + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.triggerer.persistence.size }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/triggerer/triggerer-kedaautoscaler.yaml b/charts/airflow/templates/triggerer/triggerer-kedaautoscaler.yaml new file mode 100644 index 0000000..bf040e6 --- /dev/null +++ b/charts/airflow/templates/triggerer/triggerer-kedaautoscaler.yaml @@ -0,0 +1,68 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Triggerer KEDA Scaler +################################ +{{- if and .Values.triggerer.enabled .Values.triggerer.keda.enabled }} +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{ include "airflow.fullname" . }}-triggerer + labels: + tier: airflow + component: triggerer-horizontalpodautoscaler + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + deploymentName: {{ .Release.Name }}-triggerer + {{- if or .Values.labels .Values.triggerer.labels }} + {{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + kind: {{ ternary "StatefulSet" "Deployment" .Values.triggerer.persistence.enabled }} + name: {{ include "airflow.fullname" . }}-triggerer + envSourceContainerName: triggerer + pollingInterval: {{ .Values.triggerer.keda.pollingInterval }} + cooldownPeriod: {{ .Values.triggerer.keda.cooldownPeriod }} + minReplicaCount: {{ .Values.triggerer.keda.minReplicaCount }} + maxReplicaCount: {{ .Values.triggerer.keda.maxReplicaCount }} + {{- if .Values.triggerer.keda.advanced }} + advanced: {{- toYaml .Values.triggerer.keda.advanced | nindent 4 }} + {{- end }} + triggers: + {{- if eq .Values.data.metadataConnection.protocol "mysql" }} + - type: "mysql" + metadata: + queryValue: "1" + connectionStringFromEnv: KEDA_DB_CONN + query: {{ tpl .Values.triggerer.keda.query . | quote }} + {{- else }} + - type: postgresql + metadata: + targetQueryValue: "1" + {{- if and .Values.pgbouncer.enabled (not .Values.triggerer.keda.usePgbouncer) }} + connectionFromEnv: KEDA_DB_CONN + {{- else }} + connectionFromEnv: AIRFLOW_CONN_AIRFLOW_DB + {{- end }} + query: {{ tpl .Values.triggerer.keda.query . | quote }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/triggerer/triggerer-networkpolicy.yaml b/charts/airflow/templates/triggerer/triggerer-networkpolicy.yaml new file mode 100644 index 0000000..a5a729c --- /dev/null +++ b/charts/airflow/templates/triggerer/triggerer-networkpolicy.yaml @@ -0,0 +1,55 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################## +## Airflow triggerer NetworkPolicy +################################## +{{- if and .Values.triggerer.enabled .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "airflow.fullname" . }}-triggerer-policy + labels: + tier: airflow + component: airflow-triggerer-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.triggerer.labels }} + {{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: triggerer + release: {{ .Release.Name }} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + tier: airflow + release: {{ .Release.Name }} + component: webserver + ports: + - protocol: TCP + port: {{ .Values.ports.triggererLogs }} +{{- end }} diff --git a/charts/airflow/templates/triggerer/triggerer-poddisruptionbudget.yaml b/charts/airflow/templates/triggerer/triggerer-poddisruptionbudget.yaml new file mode 100644 index 0000000..0835416 --- /dev/null +++ b/charts/airflow/templates/triggerer/triggerer-poddisruptionbudget.yaml @@ -0,0 +1,44 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Triggerer PodDisruptionBudget +################################# +{{- if and .Values.triggerer.enabled .Values.triggerer.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "airflow.fullname" . }}-triggerer-pdb + labels: + tier: airflow + component: triggerer + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.triggerer.labels }} + {{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + tier: airflow + component: triggerer + release: {{ .Release.Name }} + {{- toYaml .Values.triggerer.podDisruptionBudget.config | nindent 2 }} +{{- end }} diff --git a/charts/airflow/templates/triggerer/triggerer-service.yaml b/charts/airflow/templates/triggerer/triggerer-service.yaml new file mode 100644 index 0000000..f29af89 --- /dev/null +++ b/charts/airflow/templates/triggerer/triggerer-service.yaml @@ -0,0 +1,48 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow triggerer Service +################################# +{{- if .Values.triggerer.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "airflow.fullname" . }}-triggerer + labels: + tier: airflow + component: triggerer + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.triggerer.labels }} + {{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + clusterIP: None + selector: + tier: airflow + component: triggerer + release: {{ .Release.Name }} + ports: + - name: triggerer-logs + protocol: TCP + port: {{ .Values.ports.triggererLogs }} + targetPort: {{ .Values.ports.triggererLogs }} +{{- end }} diff --git a/charts/airflow/templates/triggerer/triggerer-serviceaccount.yaml b/charts/airflow/templates/triggerer/triggerer-serviceaccount.yaml new file mode 100644 index 0000000..e2fed78 --- /dev/null +++ b/charts/airflow/templates/triggerer/triggerer-serviceaccount.yaml @@ -0,0 +1,42 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################### +## Airflow Triggerer ServiceAccount +################################### +{{- if and .Values.triggerer.serviceAccount.create .Values.triggerer.enabled }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.triggerer.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ include "triggerer.serviceAccountName" . }} + labels: + tier: airflow + component: triggerer + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.triggerer.labels }} + {{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.triggerer.serviceAccount.annotations}} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/webserver/webserver-deployment.yaml b/charts/airflow/templates/webserver/webserver-deployment.yaml new file mode 100644 index 0000000..eca8c26 --- /dev/null +++ b/charts/airflow/templates/webserver/webserver-deployment.yaml @@ -0,0 +1,271 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Webserver Deployment +################################# +{{- if and .Values.webserver.enabled (semverCompare "<3.0.0" .Values.airflowVersion) }} +{{- $nodeSelector := or .Values.webserver.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.webserver.affinity .Values.affinity }} +{{- $tolerations := or .Values.webserver.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.webserver.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $revisionHistoryLimit := include "airflow.revisionHistoryLimit" (list .Values.webserver.revisionHistoryLimit .Values.revisionHistoryLimit) }} +{{- $securityContext := include "airflowPodSecurityContext" (list .Values.webserver .Values) }} +{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.webserver .Values) }} +{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list .Values.webserver.waitForMigrations .Values) }} +{{- $containerLifecycleHooks := or .Values.webserver.containerLifecycleHooks .Values.containerLifecycleHooks }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "airflow.fullname" . }}-webserver + labels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.webserver.annotations }} + annotations: {{- toYaml .Values.webserver.annotations | nindent 4 }} + {{- end }} +spec: + {{- if not .Values.webserver.hpa.enabled }} + replicas: {{ .Values.webserver.replicas }} + {{- end}} + {{- if ne $revisionHistoryLimit "" }} + revisionHistoryLimit: {{ $revisionHistoryLimit }} + {{- end }} + strategy: + {{- if .Values.webserver.strategy }} + {{- toYaml .Values.webserver.strategy | nindent 4 }} + {{- else }} + # Here we define the rolling update strategy + # - maxSurge define how many pod we can add at a time + # - maxUnavailable define how many pod can be unavailable + # during the rolling update + # Setting maxUnavailable to 0 would make sure we have the appropriate + # capacity during the rolling update. + # You can also use percentage based value instead of integer. + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + {{- end }} + selector: + matchLabels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + template: + metadata: + labels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + {{- if or .Values.labels .Values.webserver.labels }} + {{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 8 }} + {{- end }} + annotations: + checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }} + checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }} + checksum/webserver-secret-key: {{ include (print $.Template.BasePath "/secrets/webserver-secret-key-secret.yaml") . | sha256sum }} + checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }} + checksum/webserver-config: {{ include (print $.Template.BasePath "/configmaps/webserver-configmap.yaml") . | sha256sum }} + checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }} + checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }} + {{- if .Values.airflowPodAnnotations }} + {{- tpl (toYaml .Values.airflowPodAnnotations) . | nindent 8 }} + {{- end }} + {{- if .Values.webserver.podAnnotations }} + {{- tpl (toYaml .Values.webserver.podAnnotations) . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.webserver.hostAliases }} + hostAliases: {{- toYaml .Values.webserver.hostAliases | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "webserver.serviceAccountName" . }} + {{- if .Values.webserver.priorityClassName }} + priorityClassName: {{ .Values.webserver.priorityClassName }} + {{- end }} + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} + affinity: + {{- if $affinity }} + {{- toYaml $affinity | nindent 8 }} + {{- else }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + component: webserver + topologyKey: kubernetes.io/hostname + weight: 100 + {{- end }} + tolerations: {{- toYaml $tolerations | nindent 8 }} + topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }} + restartPolicy: Always + terminationGracePeriodSeconds: {{ .Values.webserver.terminationGracePeriodSeconds }} + securityContext: {{ $securityContext | nindent 8 }} + imagePullSecrets: {{- include "image_pull_secrets" . | nindent 8 }} + initContainers: + {{- if .Values.webserver.waitForMigrations.enabled }} + - name: wait-for-airflow-migrations + resources: {{- toYaml .Values.webserver.resources | nindent 12 }} + image: {{ template "airflow_image_for_migrations" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }} + volumeMounts: + {{- include "airflow_config_mount" . | nindent 12 }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.webserver.extraVolumeMounts }} + {{- tpl (toYaml .Values.webserver.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + {{- include "airflow_webserver_config_mount" . | nindent 12 }} + {{- end }} + args: {{- include "wait-for-migrations-command" . | indent 10 }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" . | indent 10 }} + {{- if .Values.webserver.waitForMigrations.env }} + {{- tpl (toYaml .Values.webserver.waitForMigrations.env) $ | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.webserver.extraInitContainers }} + {{- tpl (toYaml .Values.webserver.extraInitContainers) . | nindent 8 }} + {{- end }} + containers: + - name: webserver + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContext | nindent 12 }} + {{- if $containerLifecycleHooks }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }} + {{- end }} + {{- if .Values.webserver.command }} + command: {{ tpl (toYaml .Values.webserver.command) . | nindent 12 }} + {{- end }} + {{- if .Values.webserver.args }} + args: {{- tpl (toYaml .Values.webserver.args) . | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.webserver.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: {{ include "airflow_pod_template_file" . }}/pod_template_file.yaml + subPath: pod_template_file.yaml + readOnly: true + {{- include "airflow_config_mount" . | nindent 12 }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + {{- include "airflow_webserver_config_mount" . | nindent 12 }} + {{- end }} + {{- if .Values.logs.persistence.enabled }} + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- end }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.webserver.extraVolumeMounts }} + {{- tpl (toYaml .Values.webserver.extraVolumeMounts) . | nindent 12 }} + {{- end }} + ports: + - name: airflow-ui + containerPort: {{ .Values.ports.airflowUI }} + livenessProbe: + httpGet: + path: {{ if .Values.config.webserver.base_url }}{{- with urlParse (tpl .Values.config.webserver.base_url .) }}{{ .path }}{{ end }}{{ end }}/health + port: {{ .Values.ports.airflowUI }} + {{- if .Values.config.webserver.base_url}} + httpHeaders: + - name: Host + value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.config.webserver.base_url .)).host "" }} + {{- end }} + scheme: {{ .Values.webserver.livenessProbe.scheme | default "http" }} + initialDelaySeconds: {{ .Values.webserver.livenessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.webserver.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.webserver.livenessProbe.failureThreshold }} + periodSeconds: {{ .Values.webserver.livenessProbe.periodSeconds }} + readinessProbe: + httpGet: + path: {{ if .Values.config.webserver.base_url }}{{- with urlParse (tpl .Values.config.webserver.base_url .) }}{{ .path }}{{ end }}{{ end }}/health + port: {{ .Values.ports.airflowUI }} + {{- if .Values.config.webserver.base_url }} + httpHeaders: + - name: Host + value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.config.webserver.base_url .)).host "" }} + {{- end }} + scheme: {{ .Values.webserver.readinessProbe.scheme | default "http" }} + initialDelaySeconds: {{ .Values.webserver.readinessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.webserver.readinessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.webserver.readinessProbe.failureThreshold }} + periodSeconds: {{ .Values.webserver.readinessProbe.periodSeconds }} + startupProbe: + httpGet: + path: {{ if .Values.config.webserver.base_url }}{{- with urlParse (tpl .Values.config.webserver.base_url .) }}{{ .path }}{{ end }}{{ end }}/health + port: {{ .Values.ports.airflowUI }} + {{- if .Values.config.webserver.base_url}} + httpHeaders: + - name: Host + value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.config.webserver.base_url .)).host "" }} + {{- end }} + scheme: {{ .Values.webserver.startupProbe.scheme | default "http" }} + initialDelaySeconds: {{ .Values.webserver.startupProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.webserver.startupProbe.timeoutSeconds }} + failureThreshold: {{ .Values.webserver.startupProbe.failureThreshold }} + periodSeconds: {{ .Values.webserver.startupProbe.periodSeconds }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" . | indent 10 }} + {{- include "container_extra_envs" (list . .Values.webserver.env) | indent 10 }} + {{- if .Values.webserver.extraContainers }} + {{- tpl (toYaml .Values.webserver.extraContainers) . | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ template "airflow_config" . }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + - name: webserver-config + configMap: + name: {{ template "airflow_webserver_config_configmap_name" . }} + {{- end }} + {{- if .Values.logs.persistence.enabled }} + - name: logs + persistentVolumeClaim: + claimName: {{ template "airflow_logs_volume_claim" . }} + {{- end }} + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 8 }} + {{- end }} + {{- if .Values.webserver.extraVolumes }} + {{- tpl (toYaml .Values.webserver.extraVolumes) . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/webserver/webserver-hpa.yaml b/charts/airflow/templates/webserver/webserver-hpa.yaml new file mode 100644 index 0000000..eded09a --- /dev/null +++ b/charts/airflow/templates/webserver/webserver-hpa.yaml @@ -0,0 +1,49 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Webserver HPA +################################# +{{- if and .Values.webserver.enabled .Values.webserver.hpa.enabled (semverCompare "<3.0.0" .Values.airflowVersion) }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "airflow.fullname" . }}-webserver + labels: + tier: airflow + component: webserver-horizontalpodautoscaler + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + deploymentName: {{ .Release.Name }}-webserver + {{- if or .Values.labels .Values.webserver.labels }} + {{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "airflow.fullname" . }}-webserver + minReplicas: {{ .Values.webserver.hpa.minReplicaCount }} + maxReplicas: {{ .Values.webserver.hpa.maxReplicaCount }} + metrics: {{- toYaml .Values.webserver.hpa.metrics | nindent 4 }} + {{- with .Values.webserver.hpa.behavior }} + behavior: {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/webserver/webserver-ingress.yaml b/charts/airflow/templates/webserver/webserver-ingress.yaml new file mode 100644 index 0000000..1b9f8d4 --- /dev/null +++ b/charts/airflow/templates/webserver/webserver-ingress.yaml @@ -0,0 +1,109 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Webserver Ingress +################################# +{{- if and .Values.webserver.enabled (or .Values.ingress.web.enabled .Values.ingress.enabled) (semverCompare "<3.0.0" .Values.airflowVersion) }} +{{- $fullname := (include "airflow.fullname" .) }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullname }}-ingress + labels: + tier: airflow + component: airflow-ingress + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.webserver.labels }} + {{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.ingress.web.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.web.hosts (.Values.ingress.web.hosts | first | kindIs "string" | not) }} + {{- $anyTlsHosts := false -}} + {{- range .Values.ingress.web.hosts }} + {{- if and .tls .tls.enabled }} + {{- $anyTlsHosts = true -}} + {{- end }} + {{- end }} + {{- if $anyTlsHosts }} + tls: + {{- range .Values.ingress.web.hosts }} + {{- if and .tls .tls.enabled }} + - hosts: + - {{ tpl .name $ | quote }} + secretName: {{ .tls.secretName }} + {{- end }} + {{- end }} + {{- end }} + {{- else if .Values.ingress.web.tls.enabled }} + tls: + - hosts: + {{- range .Values.ingress.web.hosts | default (list .Values.ingress.web.host) }} + - {{ tpl . $ | quote }} + {{- end }} + secretName: {{ .Values.ingress.web.tls.secretName }} + {{- end }} + rules: + {{- range .Values.ingress.web.hosts | default (list .Values.ingress.web.host) }} + - http: + paths: + {{- range $.Values.ingress.web.precedingPaths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ .serviceName }} + port: + name: {{ .servicePort }} + {{- end }} + - backend: + service: + name: {{ $fullname }}-webserver + port: + name: airflow-ui + {{- if $.Values.ingress.web.path }} + path: {{ $.Values.ingress.web.path }} + pathType: {{ $.Values.ingress.web.pathType }} + {{- end }} + {{- range $.Values.ingress.web.succeedingPaths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ .serviceName }} + port: + name: {{ .servicePort }} + {{- end }} + {{- $hostname := . -}} + {{- if . | kindIs "string" | not }} + {{- $hostname = .name -}} + {{- end }} + {{- if $hostname }} + host: {{ tpl $hostname $ | quote }} + {{- end }} + {{- end }} + {{- if .Values.ingress.web.ingressClassName }} + ingressClassName: {{ .Values.ingress.web.ingressClassName }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/webserver/webserver-networkpolicy.yaml b/charts/airflow/templates/webserver/webserver-networkpolicy.yaml new file mode 100644 index 0000000..e2f02be --- /dev/null +++ b/charts/airflow/templates/webserver/webserver-networkpolicy.yaml @@ -0,0 +1,57 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Webserver NetworkPolicy +################################# +{{- if and .Values.webserver.enabled (semverCompare "<3.0.0" .Values.airflowVersion) .Values.networkPolicies.enabled }} +{{- $from := or .Values.webserver.networkPolicy.ingress.from .Values.webserver.extraNetworkPolicies }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "airflow.fullname" . }}-webserver-policy + labels: + tier: airflow + component: airflow-webserver-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.webserver.labels }} + {{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + policyTypes: + - Ingress + {{- if $from }} + ingress: + - from: {{- toYaml $from | nindent 6 }} + ports: + {{ range .Values.webserver.networkPolicy.ingress.ports }} + - + {{- range $key, $val := . }} + {{ $key }}: {{ tpl (toString $val) $ }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/webserver/webserver-poddisruptionbudget.yaml b/charts/airflow/templates/webserver/webserver-poddisruptionbudget.yaml new file mode 100644 index 0000000..822303d --- /dev/null +++ b/charts/airflow/templates/webserver/webserver-poddisruptionbudget.yaml @@ -0,0 +1,44 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Webserver PodDisruptionBudget +################################# +{{- if and .Values.webserver.enabled (semverCompare "<3.0.0" .Values.airflowVersion) .Values.webserver.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "airflow.fullname" . }}-webserver-pdb + labels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.webserver.labels }} + {{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + {{- toYaml .Values.webserver.podDisruptionBudget.config | nindent 2 }} +{{- end }} diff --git a/charts/airflow/templates/webserver/webserver-service.yaml b/charts/airflow/templates/webserver/webserver-service.yaml new file mode 100644 index 0000000..6a67a03 --- /dev/null +++ b/charts/airflow/templates/webserver/webserver-service.yaml @@ -0,0 +1,59 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Webserver Service +################################# +{{- if and .Values.webserver.enabled (semverCompare "<3.0.0" .Values.airflowVersion) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "airflow.fullname" . }}-webserver + labels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.webserver.labels }} + {{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.webserver.service.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.webserver.service.type }} + selector: + tier: airflow + component: webserver + release: {{ .Release.Name }} + ports: + {{ range .Values.webserver.service.ports }} + - + {{- range $key, $val := . }} + {{ $key }}: {{ tpl (toString $val) $ }} + {{- end }} + {{- end }} + {{- if .Values.webserver.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.webserver.service.loadBalancerIP }} + {{- end }} + {{- if .Values.webserver.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.webserver.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/webserver/webserver-serviceaccount.yaml b/charts/airflow/templates/webserver/webserver-serviceaccount.yaml new file mode 100644 index 0000000..c3ae6af --- /dev/null +++ b/charts/airflow/templates/webserver/webserver-serviceaccount.yaml @@ -0,0 +1,42 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +###################################### +## Airflow Webserver ServiceAccount +###################################### +{{- if and .Values.webserver.enabled .Values.webserver.serviceAccount.create (semverCompare "<3.0.0" .Values.airflowVersion) }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.webserver.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ include "webserver.serviceAccountName" . }} + labels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.webserver.labels }} + {{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.webserver.serviceAccount.annotations }} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/workers/worker-deployment.yaml b/charts/airflow/templates/workers/worker-deployment.yaml new file mode 100644 index 0000000..340c9aa --- /dev/null +++ b/charts/airflow/templates/workers/worker-deployment.yaml @@ -0,0 +1,523 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Worker Deployment +################################# +{{- $globals := deepCopy . -}} +{{- $filteredCelery := include "removeNilFields" .Values.workers.celery | fromYaml -}} +{{- $mergedWorkers := (include "workersMergeValues" (list .Values.workers $filteredCelery "" (list "kerberosInitContainer" "kerberosSidecar")) | fromYaml) -}} +{{- $_ := unset $mergedWorkers "celery" -}} +{{- $workerSets := .Values.workers.celery.sets | default list -}} +{{- if .Values.workers.celery.enableDefault -}} + {{- $workerSets = concat (list (dict "name" "default")) $workerSets -}} +{{- end -}} +{{- range $workerSet := $workerSets -}} + {{- $workers := (include "workersMergeValues" (list $mergedWorkers $workerSet "" list) | fromYaml) -}} + {{- $_ := set $globals.Values "workers" $workers -}} + {{- with $globals -}} +{{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }} +--- +{{- $persistence := or .Values.workers.persistence.enabled }} +{{- $keda := .Values.workers.keda.enabled }} +{{- $hpa := and .Values.workers.hpa.enabled (not .Values.workers.keda.enabled) }} +{{- $nodeSelector := or .Values.workers.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.workers.affinity .Values.affinity }} +{{- $tolerations := or .Values.workers.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.workers.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $revisionHistoryLimit := include "airflow.revisionHistoryLimit" (list .Values.workers.revisionHistoryLimit .Values.revisionHistoryLimit) }} +{{- $securityContext := include "airflowPodSecurityContext" (list .Values.workers .Values) }} +{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.workers .Values) }} +{{- $containerSecurityContextPersistence := include "containerSecurityContext" (list .Values.workers.persistence .Values) }} +{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list .Values.workers.waitForMigrations .Values) }} +{{- $containerSecurityContextLogGroomerSidecar := include "containerSecurityContext" (list .Values.workers.logGroomerSidecar .Values) }} +{{- $containerSecurityContextKerberosSidecar := include "containerSecurityContext" (list .Values.workers.kerberosSidecar .Values) }} +{{- $containerSecurityContextKerberosInitContainer := include "containerSecurityContext" (list .Values.workers.kerberosInitContainer .Values) }} +{{- $containerLifecycleHooksKerberosInitContainer := or .Values.workers.kerberosInitContainer.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $containerLifecycleHooks := or .Values.workers.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $containerLifecycleHooksLogGroomerSidecar := or .Values.workers.logGroomerSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $containerLifecycleHooksKerberosSidecar := or .Values.workers.kerberosSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $safeToEvict := dict "cluster-autoscaler.kubernetes.io/safe-to-evict" (.Values.workers.safeToEvict | toString) }} +{{- $podAnnotations := mergeOverwrite (deepCopy .Values.airflowPodAnnotations) $safeToEvict .Values.workers.podAnnotations }} +{{- $schedulerName := or .Values.workers.schedulerName .Values.schedulerName }} +apiVersion: apps/v1 +kind: {{ if $persistence }}StatefulSet{{ else }}Deployment{{ end }} +metadata: + name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + labels: + tier: airflow + component: worker + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.workers.annotations }} + annotations: {{- toYaml .Values.workers.annotations | nindent 4 }} + {{- end }} +spec: + {{- if $persistence }} + serviceName: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + {{- end }} + {{- if and (not $keda) (not $hpa) }} + replicas: {{ .Values.workers.replicas }} + {{- end }} + {{- if ne $revisionHistoryLimit "" }} + revisionHistoryLimit: {{ $revisionHistoryLimit }} + {{- end }} + {{- if and $persistence .Values.workers.persistence.persistentVolumeClaimRetentionPolicy }} + persistentVolumeClaimRetentionPolicy: {{- toYaml .Values.workers.persistence.persistentVolumeClaimRetentionPolicy | nindent 4 }} + {{- end }} + selector: + matchLabels: + tier: airflow + component: worker + release: {{ .Release.Name }} + {{- if ne .Values.workers.name "default" }} + worker-set: {{ .Values.workers.name }} + {{- end }} + {{- if and $persistence .Values.workers.podManagementPolicy }} + podManagementPolicy: {{ .Values.workers.podManagementPolicy }} + {{- end }} + {{- if and $persistence .Values.workers.updateStrategy }} + updateStrategy: {{- toYaml .Values.workers.updateStrategy | nindent 4 }} + {{- end }} + {{- if and (not $persistence) .Values.workers.strategy }} + strategy: {{- toYaml .Values.workers.strategy | nindent 4 }} + {{- end }} + template: + metadata: + labels: + tier: airflow + component: worker + release: {{ .Release.Name }} + {{- if ne .Values.workers.name "default" }} + worker-set: {{ .Values.workers.name }} + {{- end }} + {{- if or .Values.labels .Values.workers.labels }} + {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 8 }} + {{- end }} + annotations: + checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }} + checksum/result-backend-secret: {{ include (print $.Template.BasePath "/secrets/result-backend-connection-secret.yaml") . | sha256sum }} + checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }} + checksum/webserver-secret-key: {{ include (print $.Template.BasePath "/secrets/webserver-secret-key-secret.yaml") . | sha256sum }} + checksum/kerberos-keytab: {{ include (print $.Template.BasePath "/secrets/kerberos-keytab-secret.yaml") . | sha256sum }} + checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }} + checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }} + checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }} + {{- if $podAnnotations }} + {{- tpl (toYaml $podAnnotations) . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.workers.runtimeClassName }} + runtimeClassName: {{ .Values.workers.runtimeClassName }} + {{- end }} + {{- if .Values.workers.priorityClassName }} + priorityClassName: {{ .Values.workers.priorityClassName }} + {{- end }} + {{- if $schedulerName }} + schedulerName: {{ $schedulerName }} + {{- end }} + nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} + affinity: + {{- if $affinity }} + {{- toYaml $affinity | nindent 8 }} + {{- else }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + component: worker + topologyKey: kubernetes.io/hostname + weight: 100 + {{- end }} + tolerations: {{- toYaml $tolerations | nindent 8 }} + topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }} + {{- if .Values.workers.hostAliases }} + hostAliases: {{- toYaml .Values.workers.hostAliases | nindent 8 }} + {{- end }} + terminationGracePeriodSeconds: {{ .Values.workers.terminationGracePeriodSeconds }} + restartPolicy: Always + serviceAccountName: {{ include "worker.serviceAccountName" . }} + securityContext: {{ $securityContext | nindent 8 }} + imagePullSecrets: {{ include "image_pull_secrets" . | nindent 8 }} + initContainers: + {{- if and $persistence .Values.workers.persistence.fixPermissions }} + - name: volume-permissions + resources: {{- toYaml .Values.workers.resources | nindent 12 }} + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + command: + - chown + - -R + - "{{ include "airflowPodSecurityContextsIds" (list . .Values.workers) }}" + - {{ template "airflow_logs" . }} + securityContext: {{ $containerSecurityContextPersistence | nindent 12 }} + volumeMounts: + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- end }} + {{- if .Values.workers.kerberosInitContainer.enabled }} + - name: kerberos-init + image: {{ template "airflow_image" . }} + securityContext: {{ $containerSecurityContextKerberosInitContainer | nindent 12 }} + {{- if $containerLifecycleHooksKerberosInitContainer }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooksKerberosInitContainer) . | nindent 12 }} + {{- end }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + args: ["kerberos", "-o"] + resources: {{- toYaml .Values.workers.kerberosInitContainer.resources | nindent 12 }} + volumeMounts: + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- include "airflow_config_mount" . | nindent 12 }} + - name: config + mountPath: {{ .Values.kerberos.configPath | quote }} + subPath: krb5.conf + readOnly: true + - name: kerberos-keytab + subPath: "kerberos.keytab" + mountPath: {{ .Values.kerberos.keytabPath | quote }} + readOnly: true + - name: kerberos-ccache + mountPath: {{ .Values.kerberos.ccacheMountPath | quote }} + readOnly: false + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.workers.extraVolumeMounts }} + {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + {{- include "airflow_webserver_config_mount" . | nindent 12 }} + {{- end }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + - name: KRB5_CONFIG + value: {{ .Values.kerberos.configPath | quote }} + - name: KRB5CCNAME + value: {{ include "kerberos_ccache_path" . | quote }} + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- end }} + {{- if .Values.workers.waitForMigrations.enabled }} + - name: wait-for-airflow-migrations + resources: {{- toYaml .Values.workers.resources | nindent 12 }} + image: {{ template "airflow_image_for_migrations" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }} + volumeMounts: + - name: logs + mountPath: "/opt/airflow/logs" + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- include "airflow_config_mount" . | nindent 12 }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.workers.extraVolumeMounts }} + {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + {{- include "airflow_webserver_config_mount" . | nindent 12 }} + {{- end }} + args: {{- include "wait-for-migrations-command" . | indent 10 }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- if .Values.workers.waitForMigrations.env }} + {{- tpl (toYaml .Values.workers.waitForMigrations.env) $ | nindent 12 }} + {{- end }} + {{- end }} + {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }} + {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }} + {{- end }} + {{- if .Values.workers.extraInitContainers }} + {{- tpl (toYaml .Values.workers.extraInitContainers) . | nindent 8 }} + {{- end }} + containers: + - name: worker + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContext | nindent 12 }} + {{- if $containerLifecycleHooks }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }} + {{- end }} + {{- if .Values.workers.command }} + command: {{ tpl (toYaml .Values.workers.command) . | nindent 12 }} + {{- end }} + {{- if .Values.workers.args }} + args: {{ tpl (toYaml .Values.workers.args) . | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.workers.resources | nindent 12 }} + {{- if .Values.workers.livenessProbe.enabled }} + livenessProbe: + initialDelaySeconds: {{ .Values.workers.livenessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.workers.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.workers.livenessProbe.failureThreshold }} + periodSeconds: {{ .Values.workers.livenessProbe.periodSeconds }} + exec: + command: + {{- if .Values.workers.livenessProbe.command }} + {{- toYaml .Values.workers.livenessProbe.command | nindent 16 }} + {{- else }} + - sh + - -c + - CONNECTION_CHECK_MAX_COUNT=0 exec /entrypoint python -m celery --app {{ include "celery_executor_namespace" . }} inspect ping -d celery@$(hostname) + {{- end }} + {{- end }} + ports: + {{- if .Values.workers.extraPorts }} + {{- toYaml .Values.workers.extraPorts | nindent 12 }} + {{- end }} + - name: worker-logs + containerPort: {{ .Values.ports.workerLogs }} + volumeMounts: + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.workers.extraVolumeMounts }} + {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }} + {{- end }} + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- include "airflow_config_mount" . | nindent 12 }} + {{- if .Values.kerberos.enabled }} + - name: kerberos-keytab + subPath: "kerberos.keytab" + mountPath: {{ .Values.kerberos.keytabPath | quote }} + readOnly: true + - name: config + mountPath: {{ .Values.kerberos.configPath | quote }} + subPath: krb5.conf + readOnly: true + - name: kerberos-ccache + mountPath: {{ .Values.kerberos.ccacheMountPath | quote }} + readOnly: true + {{- end }} + {{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }} + {{- include "airflow_dags_mount" . | nindent 12 }} + {{- end }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + {{- include "airflow_webserver_config_mount" . | nindent 12 }} + {{- end }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + # Only signal the main process, not the process group, to make Warm Shutdown work properly + - name: DUMB_INIT_SETSID + value: "0" + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- include "container_extra_envs" (list . .Values.workers.env) | indent 10 }} + {{- if .Values.workers.kerberosSidecar.enabled }} + - name: KRB5_CONFIG + value: {{ .Values.kerberos.configPath | quote }} + - name: KRB5CCNAME + value: {{ include "kerberos_ccache_path" . | quote }} + {{- end }} + {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }} + {{- include "git_sync_container" . | nindent 8 }} + {{- end }} + {{- if and $persistence .Values.workers.logGroomerSidecar.enabled }} + - name: worker-log-groomer + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContextLogGroomerSidecar | nindent 12 }} + {{- if $containerLifecycleHooksLogGroomerSidecar }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooksLogGroomerSidecar) . | nindent 12 }} + {{- end }} + {{- if .Values.workers.logGroomerSidecar.command }} + command: {{ tpl (toYaml .Values.workers.logGroomerSidecar.command) . | nindent 12 }} + {{- end }} + {{- if .Values.workers.logGroomerSidecar.args }} + args: {{ tpl (toYaml .Values.workers.logGroomerSidecar.args) . | nindent 12 }} + {{- end }} + env: + {{- if .Values.workers.logGroomerSidecar.retentionDays }} + - name: AIRFLOW__LOG_RETENTION_DAYS + value: "{{ .Values.workers.logGroomerSidecar.retentionDays }}" + {{- end }} + {{- if .Values.workers.logGroomerSidecar.retentionMinutes }} + - name: AIRFLOW__LOG_RETENTION_MINUTES + value: "{{ .Values.workers.logGroomerSidecar.retentionMinutes }}" + {{- end }} + {{- if .Values.workers.logGroomerSidecar.frequencyMinutes }} + - name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES + value: "{{ .Values.workers.logGroomerSidecar.frequencyMinutes }}" + {{- end }} + {{- if .Values.workers.logGroomerSidecar.maxSizeBytes }} + - name: AIRFLOW__LOG_MAX_SIZE_BYTES + value: "{{ .Values.workers.logGroomerSidecar.maxSizeBytes | int64 }}" + {{- end }} + {{- if .Values.workers.logGroomerSidecar.maxSizePercent }} + - name: AIRFLOW__LOG_MAX_SIZE_PERCENT + value: "{{ .Values.workers.logGroomerSidecar.maxSizePercent }}" + {{- end }} + - name: AIRFLOW_HOME + value: "{{ .Values.airflowHome }}" + {{- if .Values.workers.logGroomerSidecar.env }} + {{- tpl (toYaml .Values.workers.logGroomerSidecar.env) $ | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.workers.logGroomerSidecar.resources | nindent 12 }} + volumeMounts: + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.workers.extraVolumeMounts }} + {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + {{- include "airflow_webserver_config_mount" . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.workers.kerberosSidecar.enabled }} + - name: worker-kerberos + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContextKerberosSidecar | nindent 12 }} + {{- if $containerLifecycleHooksKerberosSidecar }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooksKerberosSidecar) . | nindent 12 }} + {{- end }} + args: ["kerberos"] + resources: {{- toYaml .Values.workers.kerberosSidecar.resources | nindent 12 }} + volumeMounts: + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- if .Values.logs.persistence.subPath }} + subPath: {{ .Values.logs.persistence.subPath }} + {{- end }} + {{- include "airflow_config_mount" . | nindent 12 }} + - name: config + mountPath: {{ .Values.kerberos.configPath | quote }} + subPath: krb5.conf + readOnly: true + - name: kerberos-keytab + subPath: "kerberos.keytab" + mountPath: {{ .Values.kerberos.keytabPath | quote }} + readOnly: true + - name: kerberos-ccache + mountPath: {{ .Values.kerberos.ccacheMountPath | quote }} + readOnly: false + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.workers.extraVolumeMounts }} + {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + {{- include "airflow_webserver_config_mount" . | nindent 12 }} + {{- end }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + - name: KRB5_CONFIG + value: {{ .Values.kerberos.configPath | quote }} + - name: KRB5CCNAME + value: {{ include "kerberos_ccache_path" . | quote }} + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- end }} + {{- if .Values.workers.extraContainers }} + {{- tpl (toYaml .Values.workers.extraContainers) . | nindent 8 }} + {{- end }} + volumes: + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 8 }} + {{- end }} + {{- if .Values.workers.extraVolumes }} + {{- tpl (toYaml .Values.workers.extraVolumes) . | nindent 8 }} + {{- end }} + - name: config + configMap: + name: {{ template "airflow_config" . }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + - name: webserver-config + configMap: + name: {{ template "airflow_webserver_config_configmap_name" . }} + {{- end }} + {{- if .Values.kerberos.enabled }} + - name: kerberos-keytab + secret: + secretName: {{ include "kerberos_keytab_secret" . | quote }} + - name: kerberos-ccache + emptyDir: {} + {{- end }} + {{- if .Values.dags.persistence.enabled }} + - name: dags + persistentVolumeClaim: + claimName: {{ template "airflow_dags_volume_claim" . }} + {{- else if .Values.dags.gitSync.enabled }} + - name: dags + emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }} + {{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey}} + {{- include "git_sync_ssh_key_volume" . | indent 8 }} + {{- end }} + {{- end }} + {{- if .Values.logs.persistence.enabled }} + - name: logs + persistentVolumeClaim: + claimName: {{ template "airflow_logs_volume_claim" . }} + {{- else if not $persistence }} + - name: logs + emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | nindent 12 }} + {{- end }} + {{- if and $persistence (or (not .Values.logs.persistence.enabled) .Values.workers.volumeClaimTemplates) }} + volumeClaimTemplates: + {{- if not .Values.logs.persistence.enabled }} + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: logs + {{- if .Values.workers.persistence.annotations }} + annotations: {{- toYaml .Values.workers.persistence.annotations | nindent 10 }} + {{- end }} + spec: + {{- if .Values.workers.persistence.storageClassName }} + storageClassName: {{ tpl .Values.workers.persistence.storageClassName . | quote }} + {{- end }} + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.workers.persistence.size }} + {{- end }} + {{- with .Values.workers.volumeClaimTemplates }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/airflow/templates/workers/worker-hpa.yaml b/charts/airflow/templates/workers/worker-hpa.yaml new file mode 100644 index 0000000..0e9f681 --- /dev/null +++ b/charts/airflow/templates/workers/worker-hpa.yaml @@ -0,0 +1,64 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Worker HPA +################################# +{{- $globals := deepCopy . -}} +{{- $filteredCelery := include "removeNilFields" .Values.workers.celery | fromYaml -}} +{{- $mergedWorkers := (include "workersMergeValues" (list .Values.workers $filteredCelery "" (list "kerberosInitContainer" "kerberosSidecar")) | fromYaml) -}} +{{- $_ := unset $mergedWorkers "celery" -}} +{{- $workerSets := .Values.workers.celery.sets | default list -}} +{{- if .Values.workers.celery.enableDefault -}} + {{- $workerSets = concat (list (dict "name" "default")) $workerSets -}} +{{- end -}} +{{- range $workerSet := $workerSets -}} + {{- $workers := (include "workersMergeValues" (list $mergedWorkers $workerSet "" list) | fromYaml) -}} + {{- $_ := set $globals.Values "workers" $workers -}} + {{- with $globals -}} +{{- if and (and (not .Values.workers.keda.enabled) .Values.workers.hpa.enabled) (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }} +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + labels: + tier: airflow + component: worker-horizontalpodautoscaler + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + deploymentName: {{ .Release.Name }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + {{- if or .Values.labels .Values.workers.labels }} + {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: {{ ternary "StatefulSet" "Deployment" .Values.workers.persistence.enabled }} + name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + minReplicas: {{ .Values.workers.hpa.minReplicaCount }} + maxReplicas: {{ .Values.workers.hpa.maxReplicaCount }} + metrics: {{- toYaml .Values.workers.hpa.metrics | nindent 4 }} + {{- with .Values.workers.hpa.behavior }} + behavior: {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/airflow/templates/workers/worker-kedaautoscaler.yaml b/charts/airflow/templates/workers/worker-kedaautoscaler.yaml new file mode 100644 index 0000000..5d746d6 --- /dev/null +++ b/charts/airflow/templates/workers/worker-kedaautoscaler.yaml @@ -0,0 +1,83 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Worker KEDA Scaler +################################# +{{- $globals := deepCopy . -}} +{{- $filteredCelery := include "removeNilFields" .Values.workers.celery | fromYaml -}} +{{- $mergedWorkers := (include "workersMergeValues" (list .Values.workers $filteredCelery "" (list "kerberosInitContainer" "kerberosSidecar")) | fromYaml) -}} +{{- $_ := unset $mergedWorkers "celery" -}} +{{- $workerSets := .Values.workers.celery.sets | default list -}} +{{- if .Values.workers.celery.enableDefault -}} + {{- $workerSets = concat (list (dict "name" "default")) $workerSets -}} +{{- end -}} +{{- range $workerSet := $workerSets -}} + {{- $workers := (include "workersMergeValues" (list $mergedWorkers $workerSet "" list) | fromYaml) -}} + {{- $_ := set $globals.Values "workers" $workers -}} + {{- with $globals -}} +{{- if and .Values.workers.keda.enabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) ) }} +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + labels: + tier: airflow + component: worker-horizontalpodautoscaler + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + deploymentName: {{ .Release.Name }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + {{- if or .Values.labels .Values.workers.labels }} + {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + kind: {{ ternary "StatefulSet" "Deployment" .Values.workers.persistence.enabled }} + name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + envSourceContainerName: worker + pollingInterval: {{ .Values.workers.keda.pollingInterval }} + cooldownPeriod: {{ .Values.workers.keda.cooldownPeriod }} + minReplicaCount: {{ .Values.workers.keda.minReplicaCount }} + maxReplicaCount: {{ .Values.workers.keda.maxReplicaCount }} + {{- if .Values.workers.keda.advanced }} + advanced: {{- toYaml .Values.workers.keda.advanced | nindent 4 }} + {{- end }} + triggers: + {{- if eq .Values.data.metadataConnection.protocol "mysql" }} + - type: "mysql" + metadata: + queryValue: "1" + connectionStringFromEnv: KEDA_DB_CONN + query: {{ tpl .Values.workers.keda.query . | quote }} + {{- else }} + - type: "postgresql" + metadata: + targetQueryValue: "1" + {{- if and .Values.pgbouncer.enabled (not .Values.workers.keda.usePgbouncer) }} + connectionFromEnv: KEDA_DB_CONN + {{- else }} + connectionFromEnv: AIRFLOW_CONN_AIRFLOW_DB + {{- end }} + query: {{ tpl .Values.workers.keda.query . | quote }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/airflow/templates/workers/worker-kubernetes-serviceaccount.yaml b/charts/airflow/templates/workers/worker-kubernetes-serviceaccount.yaml new file mode 100644 index 0000000..b74474b --- /dev/null +++ b/charts/airflow/templates/workers/worker-kubernetes-serviceaccount.yaml @@ -0,0 +1,41 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +########################################### +## Airflow Worker Kubernetes ServiceAccount +########################################### +{{- if and .Values.workers.kubernetes.serviceAccount.create (contains "KubernetesExecutor" .Values.executor) }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ or .Values.workers.kubernetes.serviceAccount.automountServiceAccountToken (and (not (has .Values.workers.kubernetes.serviceAccount.automountServiceAccountToken (list true false))) .Values.workers.serviceAccount.automountServiceAccountToken) }} +metadata: + name: {{ include "worker.kubernetes.serviceAccountName" . }} + labels: + tier: airflow + component: worker + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.workers.labels .Values.workers.kubernetes.labels }} + {{- mustMerge (.Values.workers.kubernetes.labels | default .Values.workers.labels) .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with (.Values.workers.kubernetes.serviceAccount.annotations | default .Values.workers.serviceAccount.annotations) }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/airflow/templates/workers/worker-networkpolicy.yaml b/charts/airflow/templates/workers/worker-networkpolicy.yaml new file mode 100644 index 0000000..2fcb666 --- /dev/null +++ b/charts/airflow/templates/workers/worker-networkpolicy.yaml @@ -0,0 +1,73 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Worker NetworkPolicy +################################# +{{- $globals := deepCopy . -}} +{{- $filteredCelery := include "removeNilFields" .Values.workers.celery | fromYaml -}} +{{- $mergedWorkers := (include "workersMergeValues" (list .Values.workers $filteredCelery "" (list "kerberosInitContainer" "kerberosSidecar")) | fromYaml) -}} +{{- $_ := unset $mergedWorkers "celery" -}} +{{- $workerSets := .Values.workers.celery.sets | default list -}} +{{- if .Values.workers.celery.enableDefault -}} + {{- $workerSets = concat (list (dict "name" "default")) $workerSets -}} +{{- end -}} +{{- range $workerSet := $workerSets -}} + {{- $workers := (include "workersMergeValues" (list $mergedWorkers $workerSet "" list) | fromYaml) -}} + {{- $_ := set $globals.Values "workers" $workers -}} + {{- with $globals -}} +{{- if and .Values.networkPolicies.enabled (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "airflow.fullname" . }}-worker-policy{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + labels: + tier: airflow + component: airflow-worker-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.workers.labels }} + {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: worker + release: {{ .Release.Name }} + {{- if ne .Values.workers.name "default" }} + worker-set: {{ .Values.workers.name }} + {{- end }} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + tier: airflow + release: {{ .Release.Name }} + component: webserver + ports: + - protocol: TCP + port: {{ .Values.ports.workerLogs }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/airflow/templates/workers/worker-poddisruptionbudget.yaml b/charts/airflow/templates/workers/worker-poddisruptionbudget.yaml new file mode 100644 index 0000000..8aed138 --- /dev/null +++ b/charts/airflow/templates/workers/worker-poddisruptionbudget.yaml @@ -0,0 +1,62 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Worker PodDisruptionBudget +################################# +{{- $globals := deepCopy . -}} +{{- $filteredCelery := include "removeNilFields" .Values.workers.celery | fromYaml -}} +{{- $mergedWorkers := (include "workersMergeValues" (list .Values.workers $filteredCelery "" (list "kerberosInitContainer" "kerberosSidecar")) | fromYaml) -}} +{{- $_ := unset $mergedWorkers "celery" -}} +{{- $workerSets := .Values.workers.celery.sets | default list -}} +{{- if .Values.workers.celery.enableDefault -}} + {{- $workerSets = concat (list (dict "name" "default")) $workerSets -}} +{{- end -}} +{{- range $workerSet := $workerSets -}} + {{- $workers := (include "workersMergeValues" (list $mergedWorkers $workerSet "" list) | fromYaml) -}} + {{- $_ := set $globals.Values "workers" $workers -}} + {{- with $globals -}} +{{- if .Values.workers.podDisruptionBudget.enabled }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "airflow.fullname" . }}-worker-pdb{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + labels: + tier: airflow + component: worker + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.workers.labels }} + {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + tier: airflow + component: worker + release: {{ .Release.Name }} + {{- if ne .Values.workers.name "default" }} + worker-set: {{ .Values.workers.name }} + {{- end }} + {{- toYaml .Values.workers.podDisruptionBudget.config | nindent 2 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/airflow/templates/workers/worker-service.yaml b/charts/airflow/templates/workers/worker-service.yaml new file mode 100644 index 0000000..23a8a23 --- /dev/null +++ b/charts/airflow/templates/workers/worker-service.yaml @@ -0,0 +1,66 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Worker Service +################################# +{{- $globals := deepCopy . -}} +{{- $filteredCelery := include "removeNilFields" .Values.workers.celery | fromYaml -}} +{{- $mergedWorkers := (include "workersMergeValues" (list .Values.workers $filteredCelery "" (list "kerberosInitContainer" "kerberosSidecar")) | fromYaml) -}} +{{- $_ := unset $mergedWorkers "celery" -}} +{{- $workerSets := .Values.workers.celery.sets | default list -}} +{{- if .Values.workers.celery.enableDefault -}} + {{- $workerSets = concat (list (dict "name" "default")) $workerSets -}} +{{- end -}} +{{- range $workerSet := $workerSets -}} + {{- $workers := (include "workersMergeValues" (list $mergedWorkers $workerSet "" list) | fromYaml) -}} + {{- $_ := set $globals.Values "workers" $workers -}} + {{- with $globals -}} +{{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + labels: + tier: airflow + component: worker + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.workers.labels }} + {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + clusterIP: None + selector: + tier: airflow + component: worker + release: {{ .Release.Name }} + {{- if ne .Values.workers.name "default" }} + worker-set: {{ .Values.workers.name }} + {{- end }} + ports: + - name: worker-logs + protocol: TCP + port: {{ .Values.ports.workerLogs }} + targetPort: {{ .Values.ports.workerLogs }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/airflow/templates/workers/worker-serviceaccount.yaml b/charts/airflow/templates/workers/worker-serviceaccount.yaml new file mode 100644 index 0000000..cbcf953 --- /dev/null +++ b/charts/airflow/templates/workers/worker-serviceaccount.yaml @@ -0,0 +1,57 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow Worker ServiceAccount +################################# +{{- $globals := deepCopy . -}} +{{- $filteredCelery := include "removeNilFields" .Values.workers.celery | fromYaml -}} +{{- $mergedWorkers := (include "workersMergeValues" (list .Values.workers $filteredCelery "" (list "kerberosInitContainer" "kerberosSidecar")) | fromYaml) -}} +{{- $_ := unset $mergedWorkers "celery" -}} +{{- $workerSets := .Values.workers.celery.sets | default list -}} +{{- if .Values.workers.celery.enableDefault -}} + {{- $workerSets = concat (list (dict "name" "default")) $workerSets -}} +{{- end -}} +{{- range $workerSet := $workerSets -}} + {{- $workers := (include "workersMergeValues" (list $mergedWorkers $workerSet "" list) | fromYaml) -}} + {{- $_ := set $globals.Values "workers" $workers -}} + {{- with $globals -}} +{{- if and .Values.workers.serviceAccount.create (include "airflow.podLaunchingExecutor" .) }} +--- +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.workers.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ include "worker.serviceAccountName" . }} + labels: + tier: airflow + component: worker + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.workers.labels }} + {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.workers.serviceAccount.annotations}} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/airflow/values.schema.json b/charts/airflow/values.schema.json new file mode 100644 index 0000000..6ee2b52 --- /dev/null +++ b/charts/airflow/values.schema.json @@ -0,0 +1,15388 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "description": "Default values for airflow. Declare variables to be passed into your templates.", + "type": "object", + "x-docsSectionOrder": [ + "Common", + "Airflow", + "Images", + "Ports", + "Database", + "PgBouncer", + "API Server", + "Scheduler", + "Webserver", + "Workers", + "Triggerer", + "DagProcessor", + "Flower", + "Redis", + "StatsD", + "Jobs", + "Kubernetes", + "Ingress", + "Kerberos" + ], + "properties": { + "fullnameOverride": { + "description": "Provide a name to substitute for the full names of resources", + "type": "string", + "default": "", + "x-docsSection": "Common" + }, + "revisionHistoryLimit": { + "description": "Global number of old ReplicaSets to retain. Can be overridden by each deployment's `revisionHistoryLimit`", + "type": [ + "integer", + "null" + ], + "default": null, + "x-docsSection": "Kubernetes" + }, + "nameOverride": { + "description": "Override the name of the chart", + "type": "string", + "default": "", + "x-docsSection": "Common" + }, + "useStandardNaming": { + "description": "Use standard naming for all resources using airflow.fullname template", + "type": "boolean", + "default": false, + "x-docsSection": "Common" + }, + "uid": { + "description": "User of airflow user.", + "type": "integer", + "default": 50000, + "x-docsSection": "Airflow" + }, + "gid": { + "description": "Group of airflow user.", + "type": "integer", + "default": 0, + "x-docsSection": "Airflow" + }, + "airflowHome": { + "description": "Airflow home directory. Used for mount paths.", + "type": "string", + "default": "/opt/airflow", + "x-docsSection": "Airflow" + }, + "defaultAirflowRepository": { + "description": "Default airflow repository. Overrides all the specific images below.", + "type": "string", + "default": "apache/airflow", + "x-docsSection": "Common" + }, + "defaultAirflowTag": { + "description": "Default airflow tag to deploy.", + "type": "string", + "default": "3.2.0", + "x-docsSection": "Common" + }, + "defaultAirflowDigest": { + "description": "Default airflow digest to deploy. Overrides tag.", + "type": [ + "string", + "null" + ], + "default": null, + "x-docsSection": "Common" + }, + "airflowVersion": { + "description": "Airflow version (Used to make some decisions based on Airflow Version being deployed). Version 2.11.0 and above is supported.", + "type": "string", + "default": "3.2.0", + "x-docsSection": "Common" + }, + "securityContext": { + "description": "Default pod security context definition (deprecated, use `securityContexts` instead). The values in this parameter will be used when `securityContext` is not defined for specific Pods", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "securityContexts": { + "description": "Default security context definition. The values in this parameter will be used when `securityContexts` is not defined for specific Pods/Container.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Default pod security context definition. The values in this parameter will be used when `securityContexts` is not defined for specific Pods.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "containers": { + "description": "Default container security context definition. The values in this parameter will be used when `securityContexts` is not defined for specific containers", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false + } + ] + } + } + }, + "containerLifecycleHooks": { + "description": "Default Container Lifecycle Hooks definition. The values in this parameter will be used when `containerLifecycleHooks` is not defined for specific containers.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "nodeSelector": { + "description": "Select certain nodes for all pods.", + "type": "object", + "default": {}, + "x-docsSection": "Kubernetes", + "additionalProperties": { + "type": "string" + } + }, + "affinity": { + "description": "Specify scheduling constraints for all pods.", + "type": "object", + "default": {}, + "x-docsSection": "Kubernetes", + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for all pods.", + "type": "array", + "default": [], + "x-docsSection": "Kubernetes", + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for all pods.", + "type": "array", + "default": [], + "x-docsSection": "Kubernetes", + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "schedulerName": { + "description": "Specify kube scheduler name for Pods.", + "type": [ + "string", + "null" + ], + "default": null, + "x-docsSection": "Common" + }, + "labels": { + "description": "Add common labels to all objects and pods defined in this chart.", + "type": "object", + "default": {}, + "x-docsSection": "Kubernetes", + "additionalProperties": { + "type": "string" + } + }, + "imagePullSecrets": { + "description": "List of existing Kubernetes secrets containing Base64 encoded credentials to connect to private registries (will get passed to imagePullSecrets).", + "type": "array", + "default": [], + "x-docsSection": "Kubernetes", + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "$ref": "#/definitions/io.k8s.api.core.v1.LocalObjectReference" + } + ] + }, + "uniqueItems": true + }, + "ingress": { + "description": "Ingress configuration.", + "type": "object", + "x-docsSection": "Ingress", + "properties": { + "enabled": { + "description": "Enable all ingress resources (deprecated - use ingress.web.enabled and ingress.flower.enabled).", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "apiServer": { + "description": "Configuration for the Ingress of the API server.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable API server ingress resource.", + "type": "boolean", + "default": false + }, + "annotations": { + "description": "Annotations for the API server Ingress.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "path": { + "description": "The path for the API server Ingress.", + "type": "string", + "default": "/" + }, + "pathType": { + "description": "The pathType for the API server Ingress (required for Kubernetes 1.19 and above).", + "type": "string", + "default": "ImplementationSpecific" + }, + "host": { + "description": "The hostname for the API server Ingress. (Deprecated - renamed to `ingress.apiServer.hosts`)", + "type": "string", + "default": "" + }, + "hosts": { + "description": "The hostnames or hosts configuration for the API server Ingress.", + "type": "array", + "default": [], + "items": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "description": "The hostname for the API server Ingress.", + "type": "string", + "default": "" + }, + "tls": { + "description": "Configuration for API server Ingress TLS.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable TLS termination for the API server Ingress.", + "type": "boolean", + "default": false + }, + "secretName": { + "description": "The name of a pre-created Secret containing a TLS private key and certificate.", + "type": "string", + "default": "" + } + } + } + }, + "required": [ + "name" + ] + }, + { + "type": "string", + "default": "", + "$comment": "Deprecated by object above" + } + ] + } + }, + "ingressClassName": { + "description": "The Ingress Class for the API server Ingress.", + "type": "string", + "default": "" + }, + "tls": { + "description": "Configuration for API server Ingress TLS. (Deprecated - renamed to `ingress.apiServer.hosts[*].tls`)", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable TLS termination for the API server Ingress.", + "type": "boolean", + "default": false + }, + "secretName": { + "description": "The name of a pre-created Secret containing a TLS private key and certificate.", + "type": "string", + "default": "" + } + } + }, + "precedingPaths": { + "description": "HTTP paths to add to the API server Ingress before the default path.", + "type": "array", + "default": [] + }, + "succeedingPaths": { + "description": "HTTP paths to add to the API server Ingress after the default path.", + "type": "array", + "default": [] + } + } + }, + "web": { + "description": "Configuration for the Ingress of the web Service.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable web ingress resource.", + "type": "boolean", + "default": false + }, + "annotations": { + "description": "Annotations for the web Ingress.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "path": { + "description": "The path for the web Ingress.", + "type": "string", + "default": "/" + }, + "pathType": { + "description": "The pathType for the web Ingress (required for Kubernetes 1.19 and above).", + "type": "string", + "default": "ImplementationSpecific" + }, + "host": { + "description": "The hostname for the web Ingress. (Deprecated - renamed to `ingress.web.hosts`)", + "type": "string", + "default": "" + }, + "hosts": { + "description": "The hostnames or hosts configuration for the web Ingress.", + "type": "array", + "default": [], + "items": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "description": "The hostname for the web Ingress.", + "type": "string", + "default": "" + }, + "tls": { + "description": "Configuration for web Ingress TLS.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable TLS termination for the web Ingress.", + "type": "boolean", + "default": false + }, + "secretName": { + "description": "The name of a pre-created Secret containing a TLS private key and certificate.", + "type": "string", + "default": "" + } + } + } + }, + "required": [ + "name" + ] + }, + { + "type": "string", + "default": "", + "$comment": "Deprecated by object above" + } + ] + } + }, + "ingressClassName": { + "description": "The Ingress Class for the web Ingress.", + "type": "string", + "default": "" + }, + "tls": { + "description": "Configuration for web Ingress TLS. (Deprecated - renamed to `ingress.web.hosts[*].tls`)", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable TLS termination for the web Ingress.", + "type": "boolean", + "default": false + }, + "secretName": { + "description": "The name of a pre-created Secret containing a TLS private key and certificate.", + "type": "string", + "default": "" + } + } + }, + "precedingPaths": { + "description": "HTTP paths to add to the web Ingress before the default path.", + "type": "array", + "default": [] + }, + "succeedingPaths": { + "description": "HTTP paths to add to the web Ingress after the default path.", + "type": "array", + "default": [] + } + } + }, + "flower": { + "description": "Configuration for the Ingress of the flower Service.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable flower ingress resource.", + "type": "boolean", + "default": false + }, + "annotations": { + "description": "Annotations for the flower Ingress.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "path": { + "description": "The path for the flower Ingress.", + "type": "string", + "default": "/" + }, + "pathType": { + "description": "The pathType for the flower Ingress (required for Kubernetes 1.19 and above).", + "type": "string", + "default": "ImplementationSpecific" + }, + "host": { + "description": "The hostname for the flower Ingress. (Deprecated - renamed to `ingress.flower.hosts`)", + "type": "string", + "default": "" + }, + "hosts": { + "description": "The hostnames or hosts configuration for the flower Ingress.", + "type": "array", + "default": [], + "items": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "description": "The hostname for the web Ingress.", + "type": "string", + "default": "" + }, + "tls": { + "description": "Configuration for web Ingress TLS.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable TLS termination for the web Ingress.", + "type": "boolean", + "default": false + }, + "secretName": { + "description": "The name of a pre-created Secret containing a TLS private key and certificate.", + "type": "string", + "default": "" + } + } + } + }, + "required": [ + "name" + ] + }, + { + "type": "string", + "default": "", + "$comment": "Deprecated by object above" + } + ] + } + }, + "ingressClassName": { + "description": "The Ingress Class for the flower Ingress.", + "type": "string", + "default": "" + }, + "tls": { + "description": "Configuration for flower Ingress TLS. (Deprecated - renamed to `ingress.flower.hosts[*].tls`)", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable TLS termination for the flower Ingress.", + "type": "boolean", + "default": false + }, + "secretName": { + "description": "The name of a pre-created Secret containing a TLS private key and certificate.", + "type": "string", + "default": "" + } + } + } + } + }, + "statsd": { + "description": "Configuration for the Ingress of the statsd Service.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable statsd ingress resource.", + "type": "boolean", + "default": false + }, + "annotations": { + "description": "Annotations for the statsd Ingress.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "path": { + "description": "The path for the statsd Ingress.", + "type": "string", + "default": "/metrics" + }, + "pathType": { + "description": "The pathType for the statsd Ingress (required for Kubernetes 1.19 and above).", + "type": "string", + "default": "ImplementationSpecific" + }, + "host": { + "description": "The hostname for the statsd Ingress. (Deprecated - renamed to `ingress.statsd.hosts`)", + "type": "string", + "default": "" + }, + "hosts": { + "description": "The hostnames or hosts configuration for the statsd Ingress.", + "type": "array", + "default": [], + "items": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "description": "The hostname for the web Ingress.", + "type": "string", + "default": "" + }, + "tls": { + "description": "Configuration for web Ingress TLS.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable TLS termination for the web Ingress.", + "type": "boolean", + "default": false + }, + "secretName": { + "description": "The name of a pre-created Secret containing a TLS private key and certificate.", + "type": "string", + "default": "" + } + } + } + }, + "required": [ + "name" + ] + }, + { + "type": "string", + "default": "", + "$comment": "Deprecated by object above" + } + ] + } + }, + "ingressClassName": { + "description": "The Ingress Class for the statsd Ingress.", + "type": "string", + "default": "" + } + } + }, + "pgbouncer": { + "description": "Configuration for the Ingress of the PgBouncer Service.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable PgBouncer ingress resource.", + "type": "boolean", + "default": false + }, + "annotations": { + "description": "Annotations for the PgBouncer Ingress.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "path": { + "description": "The path for the PgBouncer Ingress.", + "type": "string", + "default": "/metrics" + }, + "pathType": { + "description": "The pathType for the PgBouncer Ingress (required for Kubernetes 1.19 and above).", + "type": "string", + "default": "ImplementationSpecific" + }, + "host": { + "description": "The hostname for the PgBouncer Ingress. (Deprecated - renamed to `ingress.pgbouncer.hosts`)", + "type": "string", + "default": "" + }, + "hosts": { + "description": "The hostnames or hosts configuration for the PgBouncer Ingress.", + "type": "array", + "default": [], + "items": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "description": "The hostname for the web Ingress.", + "type": "string", + "default": "" + }, + "tls": { + "description": "Configuration for web Ingress TLS.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable TLS termination for the web Ingress.", + "type": "boolean", + "default": false + }, + "secretName": { + "description": "The name of a pre-created Secret containing a TLS private key and certificate.", + "type": "string", + "default": "" + } + } + } + }, + "required": [ + "name" + ] + }, + { + "type": "string", + "default": "", + "$comment": "Deprecated by object above" + } + ] + } + }, + "ingressClassName": { + "description": "The Ingress Class for the PgBouncer Ingress.", + "type": "string", + "default": "" + } + } + } + } + }, + "networkPolicies": { + "description": "Network policy configuration.", + "type": "object", + "x-docsSection": "Kubernetes", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enabled network policies.", + "type": "boolean", + "default": false + } + } + }, + "airflowPodAnnotations": { + "description": "Extra annotations to apply to all Airflow pods (templated).", + "type": "object", + "default": {}, + "x-docsSection": "Kubernetes", + "additionalProperties": { + "type": "string" + } + }, + "airflowConfigAnnotations": { + "description": "Extra annotations to apply to the main Airflow configmap.", + "type": "object", + "default": {}, + "x-docsSection": "Kubernetes", + "additionalProperties": { + "type": "string" + } + }, + "airflowLocalSettings": { + "description": "`airflow_local_settings` file as a string (templated). You can bake an `airflow_local_settings.py` into your image instead. In that case, set this value to null.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Common", + "default": "See values.yaml" + }, + "rbac": { + "description": "Enable RBAC (default on most clusters these days).", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "create": { + "description": "Specifies whether RBAC resources should be created.", + "type": "boolean", + "default": true + }, + "createSCCRoleBinding": { + "description": "Specifies whether SCC RoleBinding resource should be created (refer to :doc:`Production Guide `).", + "type": "boolean", + "default": false + } + } + }, + "executor": { + "description": "Airflow executor.", + "type": "string", + "x-docsSection": "Common", + "default": "CeleryExecutor", + "pattern": "^(([a-zA-Z_][a-zA-Z0-9_]*.)*[A-Z][a-zA-Z0-9]+Executor)(,(([a-zA-Z_][a-zA-Z0-9_]*.)*[A-Z][a-zA-Z0-9]+Executor))*$" + }, + "allowPodLaunching": { + "description": "Whether various Airflow components launch pods.", + "type": "boolean", + "x-docsSection": "Airflow", + "default": true + }, + "allowJobLaunching": { + "description": "Whether various Airflow components launch jobs.", + "type": "boolean", + "x-docsSection": "Airflow", + "default": false + }, + "images": { + "description": "Images.", + "type": "object", + "x-docsSection": "Images", + "additionalProperties": false, + "properties": { + "airflow": { + "description": "Configuration of the airflow image.", + "type": "object", + "additionalProperties": false, + "properties": { + "repository": { + "description": "The airflow image repository.", + "type": [ + "string", + "null" + ], + "default": null + }, + "tag": { + "description": "The airflow image tag.", + "type": [ + "string", + "null" + ], + "default": null + }, + "digest": { + "description": "The airflow image digest. If set, it will override the tag.", + "type": [ + "string", + "null" + ], + "default": null + }, + "pullPolicy": { + "description": "The airflow image pull policy.", + "type": "string", + "enum": [ + "Always", + "Never", + "IfNotPresent" + ], + "default": "IfNotPresent" + } + } + }, + "useDefaultImageForMigration": { + "description": "To avoid images with user code for running and waiting for DB migrations set this to ``true``. ", + "type": "boolean", + "x-docsSection": "Images", + "default": false + }, + "migrationsWaitTimeout": { + "description": "The time (in seconds) to wait for the DB migrations to complete.", + "type": "number", + "x-docsSection": "Images", + "default": 60 + }, + "pod_template": { + "description": "Configuration of the pod_template image.", + "type": "object", + "additionalProperties": false, + "properties": { + "repository": { + "description": "The pod_template image repository. If ``config.kubernetes.worker_container_repository`` is set, k8s executor will use config value instead.", + "type": [ + "string", + "null" + ], + "default": null + }, + "tag": { + "description": "The pod_template image tag. If ``config.kubernetes.worker_container_tag`` is set, k8s executor will use config value instead.", + "type": [ + "string", + "null" + ], + "default": null + }, + "pullPolicy": { + "description": "The pod_template image pull policy.", + "type": "string", + "enum": [ + "Always", + "Never", + "IfNotPresent" + ], + "default": "IfNotPresent" + } + } + }, + "flower": { + "description": "Configuration of the flower image.", + "type": "object", + "additionalProperties": false, + "properties": { + "repository": { + "description": "The flower image repository.", + "type": [ + "string", + "null" + ], + "default": null + }, + "tag": { + "description": "The flower image tag.", + "type": [ + "string", + "null" + ], + "default": null + }, + "pullPolicy": { + "description": "The flower image pull policy.", + "type": "string", + "enum": [ + "Always", + "Never", + "IfNotPresent" + ], + "default": "IfNotPresent" + } + } + }, + "statsd": { + "description": "Configuration of the StatsD image.", + "type": "object", + "additionalProperties": false, + "properties": { + "repository": { + "description": "The StatsD image repository.", + "type": "string", + "default": "quay.io/prometheus/statsd-exporter" + }, + "tag": { + "description": "The StatsD image tag.", + "type": "string", + "default": "v0.29.0" + }, + "pullPolicy": { + "description": "The StatsD image pull policy.", + "type": "string", + "enum": [ + "Always", + "Never", + "IfNotPresent" + ], + "default": "IfNotPresent" + } + } + }, + "redis": { + "description": "Configuration of the redis image.", + "type": "object", + "additionalProperties": false, + "properties": { + "repository": { + "description": "The redis image repository.", + "type": "string", + "default": "redis" + }, + "tag": { + "description": "The redis image tag.", + "type": "string", + "default": "7.2-bookworm" + }, + "pullPolicy": { + "description": "The redis image pull policy.", + "type": "string", + "enum": [ + "Always", + "Never", + "IfNotPresent" + ], + "default": "IfNotPresent" + } + } + }, + "pgbouncer": { + "description": "Configuration of the PgBouncer image.", + "type": "object", + "additionalProperties": false, + "properties": { + "repository": { + "description": "The PgBouncer image repository.", + "type": "string", + "default": "apache/airflow" + }, + "tag": { + "description": "The PgBouncer image tag.", + "type": "string", + "default": "airflow-pgbouncer-2025.03.05-1.23.1" + }, + "pullPolicy": { + "description": "The PgBouncer image pull policy.", + "type": "string", + "enum": [ + "Always", + "Never", + "IfNotPresent" + ], + "default": "IfNotPresent" + } + } + }, + "pgbouncerExporter": { + "description": "Configuration of the PgBouncer exporter image.", + "type": "object", + "additionalProperties": false, + "properties": { + "repository": { + "description": "The PgBouncer exporter image repository.", + "type": "string", + "default": "apache/airflow" + }, + "tag": { + "description": "The PgBouncer exporter image tag.", + "type": "string", + "default": "airflow-pgbouncer-exporter-2025.03.05-0.18.0" + }, + "pullPolicy": { + "description": "The PgBouncer exporter image pull policy.", + "type": "string", + "enum": [ + "Always", + "Never", + "IfNotPresent" + ], + "default": "IfNotPresent" + } + } + }, + "gitSync": { + "description": "Configuration of the gitSync image.", + "type": "object", + "additionalProperties": false, + "properties": { + "repository": { + "description": "The gitSync image repository.", + "type": "string", + "default": "registry.k8s.io/git-sync/git-sync" + }, + "tag": { + "description": "The gitSync image tag.", + "type": "string", + "default": "v4.4.2" + }, + "pullPolicy": { + "description": "The gitSync image pull policy.", + "type": "string", + "enum": [ + "Always", + "Never", + "IfNotPresent" + ], + "default": "IfNotPresent" + } + } + } + } + }, + "env": { + "description": "Environment variables for all Airflow containers.", + "type": "array", + "x-docsSection": "Airflow", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": false + }, + "examples": [ + { + "name": "MYENVVAR", + "value": "something_fun" + } + ] + }, + "volumes": { + "description": "Volumes for all Airflow containers.", + "x-docsSection": "Airflow", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "volumeMounts": { + "description": "VolumeMounts for all Airflow containers.", + "x-docsSection": "Airflow", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "secret": { + "description": "Secrets for all Airflow containers.", + "type": "array", + "x-docsSection": "Airflow", + "default": [], + "items": { + "type": "object", + "properties": { + "envName": { + "description": "The name of the environment variable under which the secret will be available", + "type": "string" + }, + "secretName": { + "description": "The name of the Kubernetes secret that will be read", + "type": "string" + }, + "secretKey": { + "description": "The key of the Kubernetes secret", + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "envName", + "secretName" + ] + }, + "examples": [ + { + "envName": "SecretEnvVar", + "secretName": "somesecret", + "secretKey": "somekey" + } + ] + }, + "enableBuiltInSecretEnvVars": { + "description": "Uses built-in secret values set as environment variables passed to Airflow. You should supply corresponding environment variables as ``extraEnv`` variables if you disable them here.", + "type": "object", + "additionalProperties": false, + "x-docsSection": "Airflow", + "properties": { + "AIRFLOW__CORE__FERNET_KEY": { + "description": "Enable ``AIRFLOW__CORE__FERNET_KEY`` variable to be read from the Fernet key Secret", + "type": "boolean", + "default": true + }, + "AIRFLOW__DATABASE__SQL_ALCHEMY_CONN": { + "description": "Enable ``AIRFLOW__DATABASE__SQL_ALCHEMY_CONN`` variable to be read from the Metadata Secret", + "type": "boolean", + "default": true + }, + "AIRFLOW_CONN_AIRFLOW_DB": { + "description": "Enable ``AIRFLOW_CONN_AIRFLOW_DB`` variable to be read from the Metadata Secret", + "type": "boolean", + "default": true + }, + "AIRFLOW__API__SECRET_KEY": { + "description": "Enable ``AIRFLOW__API__SECRET_KEY`` variable to be read from the Api Secret Key Secret", + "type": "boolean", + "default": true + }, + "AIRFLOW__API_AUTH__JWT_SECRET": { + "description": "Enable ``AIRFLOW__API_AUTH__JWT_SECRET`` variable to be read from the JWT Secret", + "type": "boolean", + "default": true + }, + "AIRFLOW__WEBSERVER__SECRET_KEY": { + "description": "Enable ``AIRFLOW__WEBSERVER__SECRET_KEY`` variable to be read from the Webserver Secret Key Secret", + "type": "boolean", + "default": true + }, + "AIRFLOW__CELERY__RESULT_BACKEND": { + "description": "Enable ``AIRFLOW__CELERY__RESULT_BACKEND`` variable to be read from the Celery Result Backend Secret", + "type": "boolean", + "default": true + }, + "AIRFLOW__CELERY__BROKER_URL": { + "description": "Enable ``AIRFLOW__CELERY__BROKER_URL`` variable to be read from the Celery Broker URL Secret", + "type": "boolean", + "default": true + }, + "AIRFLOW__ELASTICSEARCH__HOST": { + "description": "Enable ``AIRFLOW__ELASTICSEARCH__HOST`` variable to be read from the Elasticsearch Host Secret", + "type": "boolean", + "default": true + }, + "AIRFLOW__OPENSEARCH__HOST": { + "description": "Enable ``AIRFLOW__OPENSEARCH__HOST`` variable to be read from the OpenSearch Host Secret", + "type": "boolean", + "default": true + } + } + }, + "extraEnv": { + "description": "Extra env 'items' that will be added to the definition of Airflow containers; a string is expected (templated).", + "type": [ + "null", + "string" + ], + "x-docsSection": "Airflow", + "default": null, + "examples": [ + "- name: AIRFLOW__CORE__LOAD_EXAMPLES\n value: True" + ] + }, + "extraEnvFrom": { + "description": "Extra envFrom 'items' that will be added to the definition of Airflow containers; a string is expected (templated).", + "type": [ + "null", + "string" + ], + "x-docsSection": "Airflow", + "default": null, + "examples": [ + "- secretRef:\n name: '{{ .Release.Name }}-airflow-connections'", + "- configMapRef:\n name: '{{ .Release.Name }}-airflow-variables'" + ] + }, + "priorityClasses": { + "description": "Priority Classes created by helm charts", + "type": "array", + "x-docsSection": "Kubernetes", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "preemptionPolicy": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + }, + "default": [], + "examples": [ + { + "name": "class1", + "preemptionPolicy": "PreemptLowerPriority", + "value": 10000 + }, + { + "name": "class2", + "preemptionPolicy": "Never", + "value": 100000 + } + ] + }, + "extraSecrets": { + "description": "Extra secrets that will be managed by the chart.", + "type": "object", + "x-docsSection": "Kubernetes", + "default": {}, + "additionalProperties": { + "description": "Name of the secret (templated).", + "type": "object", + "minProperties": 1, + "additionalProperties": false, + "properties": { + "type": { + "description": "Type **as string** of secret E.G. Opaque, kubernetes.io/dockerconfigjson, etc.", + "type": "string" + }, + "labels": { + "description": "Labels for the secret", + "type": "object", + "default": null, + "additionalProperties": { + "type": "string" + } + }, + "annotations": { + "description": "Annotations for the secret", + "type": "object", + "default": null, + "additionalProperties": { + "type": "string" + } + }, + "useHelmHooks": { + "description": "Specify if you want to use the default Helm Hook annotations", + "type": "boolean", + "default": true + }, + "data": { + "description": "Content **as string** for the 'data' item of the secret (templated)", + "type": "string" + }, + "stringData": { + "description": "Content **as string** for the 'stringData' item of the secret (templated)", + "type": "string" + } + } + }, + "examples": [ + { + "{{ .Release.Name }}-airflow-connections": { + "data": "AIRFLOW_CONN_GCP: 'base64_encoded_gcp_conn_string'\nAIRFLOW_CONN_AWS: 'base64_encoded_aws_conn_string'", + "stringData": "AIRFLOW_CONN_OTHER: 'other_conn'" + } + } + ] + }, + "extraConfigMaps": { + "description": "Extra ConfigMaps that will be managed by the chart.", + "type": "object", + "x-docsSection": "Kubernetes", + "default": {}, + "additionalProperties": { + "description": "Name of the configMap (templated).", + "type": "object", + "minProperties": 1, + "additionalProperties": false, + "properties": { + "labels": { + "description": "Labels for the configmap", + "type": "object", + "default": null, + "additionalProperties": { + "type": "string" + } + }, + "annotations": { + "description": "Annotations for the configmap", + "type": "object", + "default": null, + "additionalProperties": { + "type": "string" + } + }, + "useHelmHooks": { + "description": "Specify if you want to use the default Helm Hook annotations", + "type": "boolean", + "default": true + }, + "data": { + "description": "Content **as string** for the 'data' item of the configmap (templated)", + "type": "string" + } + } + }, + "examples": [ + { + "{{ .Release.Name }}-airflow-variables": { + "data": "AIRFLOW_VAR_HELLO_MESSAGE: 'Hi!'\nAIRFLOW_VAR_KUBERNETES_NAMESPACE: '{{ .Release.Namespace }}'" + } + } + ] + }, + "data": { + "description": "Airflow database & redis configuration.", + "type": "object", + "x-docsSection": "Database", + "additionalProperties": false, + "properties": { + "metadataSecretName": { + "description": "Metadata connection string secret.", + "type": [ + "string", + "null" + ], + "default": null + }, + "resultBackendSecretName": { + "description": "Result backend connection string secret.", + "type": [ + "string", + "null" + ], + "default": null + }, + "brokerUrlSecretName": { + "description": "Redis broker URL secret.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Redis", + "default": null + }, + "metadataConnection": { + "description": "Metadata connection configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "user": { + "description": "The database user.", + "type": "string", + "default": "postgres" + }, + "pass": { + "description": "The user's password.", + "type": "string", + "default": "postgres" + }, + "protocol": { + "description": "The database protocol.", + "type": "string", + "default": "postgresql" + }, + "host": { + "description": "The database host.", + "type": [ + "string", + "null" + ], + "default": null + }, + "port": { + "description": "The database port.", + "type": "integer", + "default": 5432 + }, + "db": { + "description": "The name of the database.", + "type": "string", + "default": "postgres" + }, + "sslmode": { + "description": "The database SSL parameter.", + "type": "string", + "default": "disable" + }, + "secretAnnotations": { + "description": "Annotations to add to the metadata connection secret.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "resultBackendConnection": { + "description": "Result backend connection configuration.", + "type": [ + "object", + "null" + ], + "default": null, + "additionalProperties": false, + "properties": { + "user": { + "description": "The database user.", + "type": "string", + "default": null + }, + "pass": { + "description": "The database password.", + "type": "string", + "default": null + }, + "protocol": { + "description": "The database protocol.", + "type": "string", + "default": null + }, + "host": { + "description": "The database host.", + "type": [ + "string", + "null" + ], + "default": null + }, + "port": { + "description": "The database port.", + "type": "integer", + "default": null + }, + "db": { + "description": "The name of the database.", + "type": "string", + "default": null + }, + "sslmode": { + "description": "The database SSL parameter.", + "type": "string", + "default": null + } + }, + "required": [ + "user", + "pass", + "protocol", + "host", + "port", + "db", + "sslmode" + ] + }, + "resultBackendConnectionSecretAnnotations": { + "description": "Annotations to add to the result backend connection secret.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "brokerUrl": { + "description": "Direct url to the redis broker (when using an external redis instance) (can only be set during install, not upgrade).", + "type": [ + "string", + "null" + ], + "x-docsSection": "Redis", + "default": null + }, + "brokerUrlSecretAnnotations": { + "description": "Annotations to add to the broker url secret.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "fernetKey": { + "description": "The Fernet key used to encrypt passwords (can only be set during install, not upgrade).", + "type": [ + "string", + "null" + ], + "x-docsSection": "Common", + "default": null + }, + "fernetKeySecretName": { + "description": "The Fernet key secret name.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Airflow", + "default": null + }, + "fernetKeySecretAnnotations": { + "description": "Annotations to add to the Fernet Key secret.", + "type": "object", + "x-docsSection": "Common", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "apiSecretKey": { + "description": "The Flask secret key for Airflow Api to encrypt browser session.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Common", + "default": null + }, + "apiSecretAnnotations": { + "description": "Annotations to add to the Api secret.", + "type": "object", + "x-docsSection": "Common", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "apiSecretKeySecretName": { + "description": "The Secret name containing Flask secret_key for the Api.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Airflow", + "default": null + }, + "jwtSecret": { + "description": "Secret key used to encode and decode JWTs to authenticate to public and private APIs. Note: It is not recommended to use in production as during helm upgrade it will be changed which can cause dag failures during component rollover.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Common", + "default": null + }, + "jwtSecretName": { + "description": "The JWT secret name.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Airflow", + "default": null + }, + "jwtSecretAnnotations": { + "description": "Annotations to add to the JWT secret.", + "type": "object", + "x-docsSection": "Common", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "webserverSecretKey": { + "description": "The Flask secret key for Airflow Webserver to encrypt browser session.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Common", + "default": null + }, + "webserverSecretAnnotations": { + "description": "Annotations to add to the webserver secret.", + "type": "object", + "x-docsSection": "Common", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "webserverSecretKeySecretName": { + "description": "The Secret name containing Flask secret_key for the Webserver.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Airflow", + "default": null + }, + "kerberos": { + "description": "Kerberos configurations for airflow", + "type": "object", + "x-docsSection": "Kerberos", + "properties": { + "enabled": { + "description": "Enable kerberos.", + "type": "boolean", + "default": false + }, + "ccacheMountPath": { + "description": "Path to mount shared volume for kerberos credentials cache.", + "type": "string", + "default": "/var/kerberos-ccache" + }, + "ccacheFileName": { + "description": "Name for kerberos credentials cache file.", + "type": "string", + "default": "cache" + }, + "configPath": { + "description": "Path to mount krb5.conf kerberos configuration file.", + "type": "string", + "default": "/etc/krb5.conf" + }, + "keytabBase64Content": { + "description": "Kerberos keytab base64 encoded content.", + "type": [ + "string", + "null" + ], + "default": null + }, + "keytabPath": { + "description": "Path to mount the keytab for refreshing credentials in the kerberos sidecar.", + "type": "string", + "default": "/etc/airflow.keytab" + }, + "principal": { + "description": "Principal to use when refreshing kerberos credentials.", + "type": "string", + "default": "airflow@FOO.COM" + }, + "reinitFrequency": { + "description": "How often (in minutes) airflow kerberos will reinitialize the credentials cache.", + "type": "integer", + "default": 3600 + }, + "config": { + "description": "Contents of krb5.conf.", + "type": "string", + "default": "See values.yaml" + } + } + }, + "workers": { + "description": "Airflow Worker configuration.", + "type": "object", + "x-docsSection": "Workers", + "additionalProperties": false, + "properties": { + "replicas": { + "description": "Number of Airflow Celery workers (deprecated, use `workers.celery.replicas` instead).", + "type": "integer", + "default": 1 + }, + "revisionHistoryLimit": { + "description": "Max number of old Airflow Celery workers ReplicaSets to retain (deprecated, use `workers.celery.revisionHistoryLimit` instead).", + "type": [ + "integer", + "null" + ], + "default": null, + "x-docsSection": "Kubernetes" + }, + "command": { + "description": "Command to use when running Airflow Celery workers and using pod-template-file (templated) (deprecated, use ``workers.celery.command`` and/or ``workers.kubernetes.command`` instead).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running Airflow Celery workers (templated) (deprecated, use `workers.celery.args` instead).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "bash", + "-c", + "exec \\\nairflow celery worker\n{{- if and .Values.workers.queue (ne .Values.workers.queue \"default\") }}\n{{- \" -q \" }}{{ .Values.workers.queue }}\n{{- end }}" + ] + }, + "livenessProbe": { + "description": "Liveness probe configuration for Airflow Celery worker containers (deprecated, use `workers.celery.livenessProbe` section instead).", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable liveness probe for Airflow Celery workers (deprecated, use `workers.celery.livenessProbe.enabled` instead).", + "type": "boolean", + "default": true + }, + "initialDelaySeconds": { + "description": "Number of seconds after the container has started before liveness probes are initiated (deprecated, use `workers.celery.livenessProbe.initialDelaySeconds` instead).", + "type": "integer", + "default": 10 + }, + "timeoutSeconds": { + "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds (deprecated, use `workers.celery.livenessProbe.timeoutSeconds` instead).", + "type": "integer", + "default": 20 + }, + "failureThreshold": { + "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1 (deprecated, use `workers.celery.livenessProbe.failureThreshold` instead).", + "type": "integer", + "default": 5 + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe. Minimum value is 1 (deprecated, use `workers.celery.livenessProbe.periodSeconds` instead).", + "type": "integer", + "default": 60 + }, + "command": { + "description": "Command for LivenessProbe (deprecated, use `workers.celery.livenessProbe.command` instead)", + "type": [ + "array", + "null" + ], + "default": null, + "items": { + "type": "string" + } + } + } + }, + "updateStrategy": { + "description": "Specifies the strategy used to replace old Airflow Celery worker pods by new ones when deployed as a StatefulSet (deprecated, use `workers.celery.updateStrategy` instead).", + "type": [ + "null", + "object" + ], + "default": null + }, + "podManagementPolicy": { + "description": "Specifies the policy for managing pods within the Airflow Celery worker (deprecated, use `workers.celery.podManagementPolicy` instead). Only applicable to StatefulSet.", + "type": [ + "null", + "string" + ], + "default": null, + "enum": [ + "OrderedReady", + "Parallel" + ] + }, + "strategy": { + "description": "Specifies the strategy used to replace old Airflow Celery worker pods by new ones when deployed as a Deployment (deprecated, use `workers.celery.strategy` instead).", + "type": [ + "null", + "object" + ], + "default": { + "rollingUpdate": { + "maxSurge": "100%", + "maxUnavailable": "50%" + } + } + }, + "serviceAccount": { + "description": "Create ServiceAccount for Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.serviceAccount`` and/or ``workers.kubernetes.serviceAccount`` instead).", + "type": "object", + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods (deprecated, use ``workers.celery.serviceAccount.automountServiceAccountToken`` and/or ``workers.kubernetes.serviceAccount.automountServiceAccountToken`` instead)", + "type": "boolean", + "default": true + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created (deprecated, use ``workers.celery.serviceAccount.create`` and/or ``workers.kubernetes.serviceAccount.create`` instead).", + "type": "boolean", + "default": true + }, + "name": { + "description": "The name of the ServiceAccount to use (deprecated, use ``workers.celery.serviceAccount.name`` and/or ``workers.kubernetes.serviceAccount.name`` instead). If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the worker Kubernetes ServiceAccount (deprecated, use ``workers.celery.serviceAccount.annotations`` and/or ``workers.kubernetes.serviceAccount.annotations`` instead).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "keda": { + "description": "KEDA configuration of Airflow Celery workers (deprecated, use `workers.celery.keda` instead).", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Allow KEDA autoscaling (deprecated, use `workers.celery.keda.enabled` instead).", + "type": "boolean", + "default": false + }, + "namespaceLabels": { + "description": "Labels used in `matchLabels` for namespace in the PgBouncer NetworkPolicy (deprecated, use `workers.celery.keda.namespaceLabels` instead).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "pollingInterval": { + "description": "How often KEDA polls the airflow DB to report new scale requests to the HPA (deprecated, use `workers.celery.keda.pollingInterval` instead).", + "type": "integer", + "default": 5 + }, + "cooldownPeriod": { + "description": "How many seconds KEDA will wait before scaling to zero (deprecated, use `workers.celery.keda.cooldownPeriod` instead).", + "type": "integer", + "default": 30 + }, + "minReplicaCount": { + "description": "Minimum number of Airflow Celery workers created by KEDA (deprecated, use `workers.celery.keda.minReplicaCount` instead).", + "type": "integer", + "default": 0 + }, + "maxReplicaCount": { + "description": "Maximum number of Airflow Celery workers created by KEDA (deprecated, use `workers.celery.keda.maxReplicaCount` instead).", + "type": "integer", + "default": 10 + }, + "advanced": { + "description": "Advanced KEDA configuration (deprecated, use `workers.celery.keda.advanced` instead).", + "type": "object", + "default": {}, + "additionalProperties": false, + "properties": { + "horizontalPodAutoscalerConfig": { + "description": "HorizontalPodAutoscalerConfig specifies horizontal scale config (deprecated, use `workers.celery.keda.advanced.horizontalPodAutoscalerConfig` instead).", + "type": "object", + "default": {}, + "properties": { + "behavior": { + "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target (deprecated, use `workers.celery.keda.advanced.horizontalPodAutoscalerConfig.behavior` instead).", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HorizontalPodAutoscalerBehavior" + } + } + } + } + }, + "query": { + "description": "Query to use for KEDA autoscaling (deprecated, use `workers.celery.keda.query` instead). Must return a single integer.", + "type": "string", + "default": "SELECT ceil(COUNT(*)::decimal / {{ .Values.config.celery.worker_concurrency }}) FROM task_instance WHERE (state='running' OR state='queued') AND queue IN ( {{- range $i, $q := splitList \",\" .Values.workers.queue -}} {{- if $i }},{{ end }}'{{ $q | trim }}' {{- end -}} ) {{- if contains \"CeleryKubernetesExecutor\" .Values.executor }} AND queue != '{{ .Values.config.celery_kubernetes_executor.kubernetes_queue }}' {{- else if contains \"KubernetesExecutor\" .Values.executor }} AND executor IS DISTINCT FROM 'KubernetesExecutor' {{- else if contains \"airflow.providers.edge3.executors.EdgeExecutor\" .Values.executor }} AND executor IS DISTINCT FROM 'EdgeExecutor' {{- end }}" + }, + "usePgbouncer": { + "description": "Weather to use PGBouncer to connect to the database or not when it is enabled (deprecated, use `workers.celery.keda.usePgbouncer` instead). This configuration will be ignored if PGBouncer is not enabled.", + "type": "boolean", + "default": true + } + } + }, + "hpa": { + "description": "HPA configuration for Airflow Celery workers (deprecated, use ``workers.celery.hpa`` instead).", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Allow HPA autoscaling (KEDA must be disabled) (deprecated, use ``workers.celery.hpa.enabled`` instead).", + "type": "boolean", + "default": false + }, + "minReplicaCount": { + "description": "Minimum number of Airflow Celery workers created by HPA (deprecated, use ``workers.celery.hpa.minReplicaCount`` instead).", + "type": "integer", + "default": 0 + }, + "maxReplicaCount": { + "description": "Maximum number of Airflow Celery workers created by HPA (deprecated, use ``workers.celery.hpa.maxReplicaCount`` instead).", + "type": "integer", + "default": 5 + }, + "metrics": { + "description": "Specifications for which to use to calculate the desired replica count (deprecated, use ``workers.celery.hpa.metrics`` instead).", + "type": "array", + "default": [ + { + "type": "Resource", + "resource": { + "name": "cpu", + "target": { + "type": "Utilization", + "averageUtilization": 80 + } + } + } + ], + "items": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricSpec" + } + }, + "behavior": { + "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target (deprecated, use ``workers.celery.hpa.behavior`` instead).", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HorizontalPodAutoscalerBehavior" + } + } + }, + "persistence": { + "description": "Persistence configuration for Airflow Celery workers (deprecated, use `workers.celery.persistence` instead).", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable persistent volumes (deprecated, use `workers.celery.persistence.enabled` instead).", + "type": "boolean", + "default": true + }, + "persistentVolumeClaimRetentionPolicy": { + "$ref": "#/definitions/persistentVolumeClaimRetentionPolicy", + "description": "PersistentVolumeClaim retention policy to be used in the lifecycle of a StatefulSet (deprecated, use `workers.celery.persistence.persistentVolumeClaimRetentionPolicy` instead)." + }, + "size": { + "description": "Volume size for Airflow Celery worker StatefulSet (deprecated, use `workers.celery.persistence.size` instead).", + "type": "string", + "default": "100Gi" + }, + "storageClassName": { + "description": "If using a custom StorageClass, pass name ref to all StatefulSets here (templated) (deprecated, use `workers.celery.persistence.storageClassName` instead).", + "type": [ + "string", + "null" + ], + "default": null + }, + "fixPermissions": { + "description": "Execute init container to chown log directory. This is currently only needed in kind, due to usage of local-path provisioner (deprecated, use `workers.celery.persistence.fixPermissions` instead).", + "type": "boolean", + "default": false + }, + "annotations": { + "description": "Annotations to add to Airflow Celery worker volumes (deprecated, use `workers.celery.persistence.annotations` instead).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "securityContexts": { + "description": "Security context definition for the persistence. If not set, the values from global `securityContexts` will be used (deprecated, use `workers.celery.persistence.securityContexts` instead).", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the persistence.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "kerberosSidecar": { + "description": "Kerberos sidecar for Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.kerberosSidecar`` and/or ``workers.kubernetes.kerberosSidecar`` instead).", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable Kerberos sidecar (deprecated, use ``workers.celery.kerberosSidecar.enabled`` and/or ``workers.kubernetes.kerberosSidecar.enabled`` instead).", + "type": "boolean", + "default": false + }, + "resources": { + "description": "Resources on kerberos sidecar (deprecated, use ``workers.celery.kerberosSidecar.resources`` and/or ``workers.kubernetes.kerberosSidecar.resources`` instead).", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the kerberos sidecar (deprecated, use ``workers.celery.kerberosSidecar.containerLifecycleHooks`` and/or ``workers.kubernetes.kerberosSidecar.containerLifecycleHooks`` instead). If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the kerberos sidecar (deprecated, use ``workers.celery.kerberosSidecar.securityContexts`` and/or ``workers.kubernetes.kerberosSidecar.securityContexts`` instead). If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the kerberos sidecar (deprecated, use ``workers.celery.kerberosSidecar.securityContexts.container`` and/or ``workers.kubernetes.kerberosSidecar.securityContexts.container`` instead).", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "kerberosInitContainer": { + "description": "Kerberos init container for Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.kerberosInitContainer`` and/or ``workers.kubernetes.kerberosInitContainer`` instead).", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable Kerberos init container (deprecated, use ``workers.celery.kerberosInitContainer.enabled`` and/or ``workers.kubernetes.kerberosInitContainer.enabled`` instead).", + "type": "boolean", + "default": false + }, + "resources": { + "description": "Resources on kerberos init container (deprecated, use ``workers.celery.kerberosInitContainer.resources`` and/or ``workers.kubernetes.kerberosInitContainer.resources`` instead).", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the kerberos init container (deprecated, use ``workers.celery.kerberosInitContainer.containerLifecycleHooks`` and/or ``workers.kubernetes.kerberosInitContainer.containerLifecycleHooks`` instead). If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the kerberos init container (deprecated, use ``workers.celery.kerberosInitContainer.securityContexts`` and/or ``workers.kubernetes.kerberosInitContainer.securityContexts`` instead). If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the kerberos init container (deprecated, use ``workers.celery.kerberosInitContainer.securityContexts.container`` and/or ``workers.kubernetes.kerberosInitContainer.securityContexts.container`` instead).", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "podDisruptionBudget": { + "description": "Worker pod disruption budget (deprecated, use `workers.celery.podDisruptionBudget` instead).", + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "description": "Enable pod disruption budget (deprecated, use `workers.celery.podDisruptionBudget.enabled` instead).", + "type": "boolean", + "default": false + }, + "config": { + "description": "Disruption budget configuration (deprecated, use `workers.celery.podDisruptionBudget.config` instead).", + "type": "object", + "additionalProperties": true, + "properties": { + "maxUnavailable": { + "description": "Max unavailable pods for worker (deprecated, use `workers.celery.podDisruptionBudget.config.maxUnavailable` instead).", + "type": [ + "integer", + "string" + ], + "default": 1 + }, + "minAvailable": { + "description": "Min available pods for worker (deprecated, use `workers.celery.podDisruptionBudget.config.minAvailable` instead).", + "type": [ + "integer", + "string" + ], + "default": 1 + } + } + } + } + }, + "resources": { + "description": "Resource configuration for Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.resources`` or/and ``workers.kubernetes.resources`` instead).", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "terminationGracePeriodSeconds": { + "description": "Grace period for tasks to finish after SIGTERM is sent from Kubernetes. It is used by Airflow Celery workers and pod-template-file (deprecated, use ``workers.celery.terminationGracePeriodSeconds`` or/and ``workers.kubernetes.terminationGracePeriodSeconds`` instead).", + "type": "integer", + "default": 600 + }, + "safeToEvict": { + "description": "This setting tells Kubernetes that it's ok to evict when it wants to scale a node down. It is used by Airflow Celery workers and pod-template-file (deprecated, use ``workers.celery.safeToEvict`` or/and ``workers.kubernetes.safeToEvict`` instead).", + "type": "boolean", + "default": false + }, + "extraContainers": { + "description": "Launch additional containers into Airflow Celery workers and pods created with pod-template-file (templated) (deprecated, use ``workers.celery.extraContainers`` and/or ``workers.kubernetes.extraContainers`` instead). Note, if used with KubernetesExecutor, you are responsible for signaling sidecars to exit when the main container finishes so Airflow can continue the worker shutdown process!", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraInitContainers": { + "description": "Add additional init containers into Airflow Celery workers and pods created with pod-template-file (templated) (deprecated, use ``workers.celery.extraInitContainers`` and/or ``workers.kubernetes.extraInitContainers`` instead).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraPorts": { + "description": "Expose additional ports of Airflow Celery worker container (deprecated, use `workers.celery.extraPorts` instead).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.ContainerPort" + } + }, + "extraVolumes": { + "description": "Additional volumes attached to the Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.extraVolumes`` and/or ``workers.kubernetes.extraVolumes`` instead).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "extraVolumeMounts": { + "description": "Additional volume mounts attached to the Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.extraVolumeMounts`` and/or ``workers.kubernetes.extraVolumeMounts`` instead).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "nodeSelector": { + "description": "Select certain nodes for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.nodeSelector`` or/and ``workers.kubernetes.nodeSelector`` instead).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "runtimeClassName": { + "description": "Specify runtime for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.runtimeClassName`` and/or ``workers.kubernetes.runtimeClassName`` instead).", + "type": [ + "string", + "null" + ], + "default": null + }, + "priorityClassName": { + "description": "Specify priority for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.priorityClassName`` and/or ``workers.kubernetes.priorityClassName`` instead).", + "type": [ + "string", + "null" + ], + "default": null + }, + "affinity": { + "description": "Specify scheduling constraints for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.affinity`` and/or ``workers.kubernetes.affinity`` instead).", + "type": "object", + "default": "See values.yaml", + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.tolerations`` and/or ``workers.kubernetes.tolerations`` instead).", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.topologySpreadConstraints`` and/or ``workers.kubernetes.topologySpreadConstraints`` instead).", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "hostAliases": { + "description": "Specify HostAliases for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.hostAliases`` and/or ``workers.kubernetes.hostAliases`` instead).", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.HostAlias" + }, + "type": "array", + "default": [], + "examples": [ + { + "ip": "127.0.0.2", + "hostnames": [ + "test.hostname.one" + ] + }, + { + "ip": "127.0.0.3", + "hostnames": [ + "test.hostname.two" + ] + } + ] + }, + "annotations": { + "description": "Annotations to add to the Airflow Celery worker deployment (deprecated, use ``workers.celery.annotations`` instead).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "podAnnotations": { + "description": "Annotations to add to the Airflow Celery workers and pods created with pod-template-file (templated) (deprecated, use ``workers.celery.podAnnotations`` and/or ``workers.kubernetes.podAnnotations`` instead).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "schedulerName": { + "description": "Specify kube scheduler name for Airflow Celery workers objects and pods created with pod-template-file (deprecated, use ``workers.celery.schedulerName`` and/or ``workers.kubernetes.schedulerName`` instead).", + "type": [ + "string", + "null" + ], + "default": null, + "x-docsSection": "Common" + }, + "labels": { + "description": "Labels to add to the Airflow Celery workers objects and pods created with pod-template-file (deprecated, use ``workers.celery.labels`` and/or ``workers.kubernetes.labels`` instead).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "logGroomerSidecar": { + "description": "Configuration for Airflow Celery worker log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar`` instead).", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Whether to deploy the Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.enabled`` instead).", + "type": "boolean", + "default": true + }, + "command": { + "description": "Command to use when running the Airflow log groomer sidecar (templated) (deprecated, use ``workers.celery.logGroomerSidecar.command`` instead).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running the Airflow log groomer sidecar (templated) (deprecated, use ``workers.celery.logGroomerSidecar.args`` instead).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "bash", + "/clean-logs" + ] + }, + "retentionDays": { + "description": "Number of days to retain the logs when running the Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.retentionDays`` instead). Total retention time is ``retentionDays`` + ``retentionMinutes``.", + "type": "integer", + "default": 15 + }, + "retentionMinutes": { + "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.retentionMinutes`` instead). Total retention time is ``retentionDays`` + ``retentionMinutes``.", + "type": "integer", + "default": 0 + }, + "frequencyMinutes": { + "description": "Number of minutes between attempts to groom the Airflow logs in log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.frequencyMinutes`` instead).", + "type": "integer", + "default": 15 + }, + "maxSizeBytes": { + "description": "Max size of logs directory in bytes (deprecated, use ``workers.celery.logGroomerSidecar.maxSizeBytes`` instead). When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled.", + "type": "integer", + "default": 0, + "minimum": 0 + }, + "maxSizePercent": { + "description": "Max size of logs as a percentage of total disk space (deprecated, use ``workers.celery.logGroomerSidecar.maxSizePercent`` instead). When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if ``maxSizeBytes`` is set.", + "type": "integer", + "default": 0, + "minimum": 0, + "maximum": 100 + }, + "env": { + "description": "Add additional env vars to log groomer sidecar container (templated) (deprecated, use ``workers.celery.logGroomerSidecar.env`` instead).", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar" + }, + "type": "array", + "default": [], + "x-kubernetes-patch-merge-key": "name", + "x-kubernetes-patch-strategy": "merge" + }, + "resources": { + "description": "Resources for Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.resources`` instead).", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.containerLifecycleHooks`` instead). If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.securityContexts`` instead). If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.securityContexts.container`` instead).", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "securityContext": { + "description": "Security context for the Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.securityContexts`` and/or ``workers.kubernetes.securityContexts`` instead). If not set, the values from `securityContext` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.containerLifecycleHooks`` and/or ``workers.kubernetes.containerLifecycleHooks`` instead). If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the Airflow Celery workers and pod-template-file (deprecated, use ``workers.celery.securityContexts`` and/or ``workers.kubernetes.securityContexts`` instead). If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition (deprecated, use ``workers.celery.securityContexts.pod`` and/or ``workers.kubernetes.securityContexts.pod`` instead).", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition (deprecated, use ``workers.celery.securityContexts.container`` and/or ``workers.kubernetes.securityContexts.container`` instead).", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "waitForMigrations": { + "description": "Configuration of wait-for-airflow-migration init container for Airflow Celery workers (deprecated, use ``workers.celery.waitForMigrations`` instead).", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable wait-for-airflow-migrations init container (deprecated, use ``workers.celery.waitForMigrations.enabled`` instead).", + "type": "boolean", + "default": true + }, + "env": { + "description": "Add additional env vars to wait-for-airflow-migrations init container (deprecated, use ``workers.celery.waitForMigrations.env`` instead).", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": false + } + }, + "securityContexts": { + "description": "Security context definition for the wait-for-airflow-migrations container (deprecated, use ``workers.celery.waitForMigrations.securityContexts`` instead). If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the wait-for-airflow-migrations container (deprecated, use ``workers.celery.waitForMigrations.securityContexts.container`` instead).", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "env": { + "description": "Add additional env vars to the Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.env`` and/or ``workers.kubernetes.env`` instead).", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] + } + }, + "required": [ + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } + ], + "additionalProperties": false + } + }, + "volumeClaimTemplates": { + "description": "Specify additional volume claim template for Airflow Celery workers (deprecated, use ``workers.celery.volumeClaimTemplates`` instead).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.PersistentVolumeClaimTemplate" + }, + "examples": [ + { + "name": "data-volume-1", + "storageClassName": "storage-class-1", + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "10Gi" + } + } + }, + { + "name": "data-volume-2", + "storageClassName": "storage-class-2", + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "20Gi" + } + } + } + ] + }, + "celery": { + "description": "Airflow Celery Workers configuration.", + "type": "object", + "x-docsSection": "Workers", + "properties": { + "replicas": { + "description": "Number of Airflow Celery workers.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "revisionHistoryLimit": { + "description": "Max number of old Airflow Celery workers ReplicaSets to retain.", + "type": [ + "integer", + "null" + ], + "default": null, + "x-docsSection": "Kubernetes" + }, + "command": { + "description": "Command to use when running Airflow Celery workers (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running Airflow Celery workers (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "livenessProbe": { + "description": "Liveness probe configuration for Airflow Celery worker containers.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable liveness probe for Airflow Celery workers.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "initialDelaySeconds": { + "description": "Number of seconds after the container has started before liveness probes are initiated.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "timeoutSeconds": { + "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "failureThreshold": { + "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe. Minimum value is 1.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "command": { + "description": "Command for LivenessProbe", + "type": [ + "array", + "null" + ], + "default": null, + "items": { + "type": "string" + } + } + } + }, + "updateStrategy": { + "description": "Specifies the strategy used to replace old Airflow Celery worker pods by new ones when deployed as a StatefulSet.", + "type": [ + "null", + "object" + ], + "default": null + }, + "strategy": { + "description": "Specifies the strategy used to replace old Airflow Celery worker pods by new ones when deployed as a Deployment.", + "type": [ + "null", + "object" + ], + "default": null + }, + "podManagementPolicy": { + "description": "Specifies the policy for managing pods within the Airflow Celery worker. Only applicable to StatefulSet.", + "type": [ + "null", + "string" + ], + "default": null, + "enum": [ + "OrderedReady", + "Parallel" + ] + }, + "enableDefault": { + "description": "Enable the default worker defined by the `workers` and `workers.celery` configurations.", + "type": "boolean", + "default": true + }, + "queue": { + "description": "Queue name for the worker.", + "type": "string", + "default": "default" + }, + "sets": { + "description": "List of worker sets. Each item can overwrite values from the parent `workers` and `workers.celery` sections.", + "type": "array", + "default": [], + "items": { + "type": "object", + "additionalProperties": {} + } + }, + "securityContexts": { + "description": "Security context definition for the Airflow Celery workers. If not set, the values from `workers.securityContexts` section will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for Airflow Celery workers. If not set, the values from `workers.containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "podDisruptionBudget": { + "description": "Airflow Celery worker pod disruption budget.", + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "description": "Enable pod disruption budget.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "config": { + "description": "Disruption budget configuration.", + "type": "object", + "additionalProperties": true, + "properties": { + "maxUnavailable": { + "description": "Max unavailable pods for worker.", + "type": [ + "integer", + "string", + "null" + ], + "default": null + }, + "minAvailable": { + "description": "Min available pods for worker.", + "type": [ + "integer", + "string", + "null" + ], + "default": null + } + } + } + } + }, + "serviceAccount": { + "description": "Create ServiceAccount for Airflow Celery workers.", + "type": "object", + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the worker Kubernetes ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "keda": { + "description": "KEDA configuration of Airflow Celery workers.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Allow KEDA autoscaling.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "namespaceLabels": { + "description": "Labels used in `matchLabels` for namespace in the PgBouncer NetworkPolicy.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "pollingInterval": { + "description": "How often KEDA polls the airflow DB to report new scale requests to the HPA.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "cooldownPeriod": { + "description": "How many seconds KEDA will wait before scaling to zero.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "minReplicaCount": { + "description": "Minimum number of Airflow Celery workers created by KEDA.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "maxReplicaCount": { + "description": "Maximum number of Airflow Celery workers created by KEDA.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "advanced": { + "description": "Advanced KEDA configuration.", + "type": "object", + "default": {}, + "additionalProperties": false, + "properties": { + "horizontalPodAutoscalerConfig": { + "description": "HorizontalPodAutoscalerConfig specifies horizontal scale config.", + "type": "object", + "default": {}, + "properties": { + "behavior": { + "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HorizontalPodAutoscalerBehavior" + } + } + } + } + }, + "query": { + "description": "Query to use for KEDA autoscaling. Must return a single integer.", + "type": [ + "string", + "null" + ], + "default": null + }, + "usePgbouncer": { + "description": "Weather to use PGBouncer to connect to the database or not when it is enabled. This configuration will be ignored if PGBouncer is not enabled.", + "type": [ + "boolean", + "null" + ], + "default": null + } + } + }, + "hpa": { + "description": "HPA configuration for Airflow Celery workers.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Allow HPA autoscaling (KEDA must be disabled).", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "minReplicaCount": { + "description": "Minimum number of Airflow Celery workers created by HPA.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "maxReplicaCount": { + "description": "Maximum number of Airflow Celery workers created by HPA.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "metrics": { + "description": "Specifications for which to use to calculate the desired replica count.", + "type": [ + "array", + "null" + ], + "default": null, + "items": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricSpec" + } + }, + "behavior": { + "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HorizontalPodAutoscalerBehavior" + } + } + }, + "persistence": { + "description": "Persistence configuration for Airflow Celery workers.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable persistent volumes.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "persistentVolumeClaimRetentionPolicy": { + "$ref": "#/definitions/persistentVolumeClaimRetentionPolicy", + "description": "PersistentVolumeClaim retention policy to be used in the lifecycle of a StatefulSet." + }, + "size": { + "description": "Volume size for Airflow Celery worker StatefulSet.", + "type": [ + "string", + "null" + ], + "default": null + }, + "storageClassName": { + "description": "If using a custom StorageClass, pass name ref to all StatefulSets here (templated).", + "type": [ + "string", + "null" + ], + "default": null + }, + "fixPermissions": { + "description": "Execute init container to chown log directory. This is currently only needed in kind, due to usage of local-path provisioner.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to Airflow Celery worker volumes.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "securityContexts": { + "description": "Security context definition for the persistence. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the persistence.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "kerberosSidecar": { + "description": "Kerberos sidecar for Airflow Celery workers.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable Kerberos sidecar.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "resources": { + "description": "Resources on kerberos sidecar.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the kerberos sidecar. If not set, the values from `workers.containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the kerberos sidecar. If not set, the values from `workers.securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the kerberos sidecar.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "kerberosInitContainer": { + "description": "Kerberos init container for Airflow Celery workers.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable Kerberos init container.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "resources": { + "description": "Resources on kerberos init container.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the kerberos init container. If not set, the values from `workers.kerberosInitContainer.containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the kerberos init container. If not set, the values from `workers.kerberosInitContainer.securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the kerberos init container.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "resources": { + "description": "Resource configuration for Airflow Celery workers.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "terminationGracePeriodSeconds": { + "description": "Grace period for tasks to finish after SIGTERM is sent from Kubernetes.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "safeToEvict": { + "description": "This setting tells Kubernetes that it's ok to evict when it wants to scale a node down.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "extraContainers": { + "description": "Launch additional containers into Airflow Celery worker (templated).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraInitContainers": { + "description": "Add additional init containers into Airflow Celery workers (templated).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraVolumes": { + "description": "Additional volumes attached to the Airflow Celery workers.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "extraVolumeMounts": { + "description": "Additional volume mounts attached to the Airflow Celery workers.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "extraPorts": { + "description": "Expose additional ports of Airflow Celery worker container.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.ContainerPort" + } + }, + "nodeSelector": { + "description": "Select certain nodes for Airflow Celery worker pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "runtimeClassName": { + "description": "Specify runtime for Airflow Celery worker pods.", + "type": [ + "string", + "null" + ], + "default": null + }, + "priorityClassName": { + "description": "Specify priority for Airflow Celery worker pods.", + "type": [ + "string", + "null" + ], + "default": null + }, + "affinity": { + "description": "Specify scheduling constraints for Airflow Celery worker pods.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for Airflow Celery worker pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for Airflow Celery worker pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "hostAliases": { + "description": "Specify HostAliases for Airflow Celery worker pods.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.HostAlias" + }, + "type": "array", + "default": [], + "examples": [ + { + "ip": "127.0.0.2", + "hostnames": [ + "test.hostname.one" + ] + }, + { + "ip": "127.0.0.3", + "hostnames": [ + "test.hostname.two" + ] + } + ] + }, + "annotations": { + "description": "Annotations to add to the Airflow Celery worker deployment.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "podAnnotations": { + "description": "Annotations to add to the Airflow Celery workers (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "description": "Labels to add to the Airflow Celery workers objects.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "logGroomerSidecar": { + "description": "Configuration for Airflow Celery worker log groomer sidecar.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Whether to deploy the Airflow log groomer sidecar.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "command": { + "description": "Command to use when running the Airflow log groomer sidecar (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running the Airflow log groomer sidecar (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [] + }, + "retentionDays": { + "description": "Number of days to retain the logs when running the Airflow log groomer sidecar. Total retention time is ``retentionDays`` + ``retentionMinutes``.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "retentionMinutes": { + "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar. Total retention time is ``retentionDays`` + ``retentionMinutes``.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "frequencyMinutes": { + "description": "Number of minutes between attempts to groom the Airflow logs in log groomer sidecar.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "maxSizeBytes": { + "description": "Max size of logs directory in bytes. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled.", + "type": [ + "integer", + "null" + ], + "default": null, + "minimum": 0 + }, + "maxSizePercent": { + "description": "Max size of logs as a percentage of total disk space. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if ``maxSizeBytes`` is set.", + "type": [ + "integer", + "null" + ], + "default": null, + "minimum": 0, + "maximum": 100 + }, + "env": { + "description": "Add additional env vars to log groomer sidecar container (templated).", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar" + }, + "type": "array", + "default": [], + "x-kubernetes-patch-merge-key": "name", + "x-kubernetes-patch-strategy": "merge" + }, + "resources": { + "description": "Resources for Airflow log groomer sidecar.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the log groomer sidecar. If not set, the values from ``workers.containerLifecycleHooks`` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the log groomer sidecar. If not set, the values from ``workers.securityContexts`` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the log groomer sidecar.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "waitForMigrations": { + "description": "Configuration of wait-for-airflow-migration init container for Airflow Celery workers.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Whether to create init container to wait for db migrations.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "env": { + "description": "Add additional env vars to wait-for-airflow-migrations init container.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": false + } + }, + "securityContexts": { + "description": "Security context definition for the wait-for-airflow-migrations container. If not set, the values from ``workers.waitForMigrations.securityContexts`` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the wait-for-airflow-migrations container.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "env": { + "description": "Add additional env vars to the Airflow Celery workers.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] + } + }, + "required": [ + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } + ], + "additionalProperties": false + } + }, + "volumeClaimTemplates": { + "description": "Specify additional volume claim template for Airflow Celery workers.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.PersistentVolumeClaimTemplate" + }, + "examples": [ + { + "name": "data-volume-1", + "storageClassName": "storage-class-1", + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "10Gi" + } + } + }, + { + "name": "data-volume-2", + "storageClassName": "storage-class-2", + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "20Gi" + } + } + } + ] + }, + "schedulerName": { + "description": "Specify kube scheduler name for Airflow Celery worker pods.", + "type": [ + "string", + "null" + ], + "default": null, + "x-docsSection": "Common" + } + } + }, + "kubernetes": { + "description": "Airflow pod-template-file configuration.", + "type": "object", + "x-docsSection": "Workers", + "properties": { + "command": { + "description": "Command to use in pod-template-file (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "securityContexts": { + "description": "Security context definition for the pod-template-file. If not set, the values from `workers.securityContexts` section will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for pods created with pod-template-file. If not set, the values from `workers.containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "serviceAccount": { + "description": "Create ServiceAccount for pods created with pod-template-file. When this section is specified, the Service Account is created from ``templates/workers/worker-kubernetes-serviceaccount.yaml`` file.", + "type": "object", + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods. If not specified, the ``workers.serviceAccount.automountServiceAccountToken`` value will be taken.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created. If not specified, the ServiceAccount will be generated and used from ``templates/workers/worker-serviceaccount.yaml`` file if ``workers.serviceAccount.create`` will be 'true'.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and ``create`` is 'true', a name is generated using the release name with kubernetes dedicated name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the worker Kubernetes ServiceAccount. If not specified, the ``workers.serviceAccount.annotations`` value will be taken.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "kerberosSidecar": { + "description": "Kerberos sidecar for pods created with pod-template-file.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable Kerberos sidecar.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "resources": { + "description": "Resources on kerberos sidecar.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the kerberos sidecar. If not set, the values from `workers.containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the kerberos sidecar. If not set, the values from `workers.securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the kerberos sidecar.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "kerberosInitContainer": { + "description": "Kerberos init container for pods created with pod-template-file.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable kerberos init container.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "resources": { + "description": "Resources on kerberos init container.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the kerberos init container. If not set, the values from `workers.kerberosInitContainer.containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the kerberos init container. If not set, the values from `workers.kerberosInitContainer.securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the kerberos init container.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "resources": { + "description": "Resource configuration for pods created with pod-template-file.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "terminationGracePeriodSeconds": { + "description": "Grace period for tasks to finish after SIGTERM is sent from Kubernetes.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "safeToEvict": { + "description": "This setting tells Kubernetes that it's ok to evict when it wants to scale a node down.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "extraContainers": { + "description": "Launch additional containers into pods created with pod-template-file (templated). Note, you are responsible for signaling sidecars to exit when the main container finishes so Airflow can continue the worker shutdown process!", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraInitContainers": { + "description": "Add additional init containers into pods created with pod-template-file (templated).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraVolumes": { + "description": "Additional volumes attached to the pods created with pod-template-file.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "extraVolumeMounts": { + "description": "Additional volume mounts attached to the pods created with pod-template-file.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "nodeSelector": { + "description": "Select certain nodes for pods created with pod-template-file.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "runtimeClassName": { + "description": "Specify runtime for pods created with pod-template-file.", + "type": [ + "string", + "null" + ], + "default": null + }, + "priorityClassName": { + "description": "Specify priority for pods created with pod-template-file.", + "type": [ + "string", + "null" + ], + "default": null + }, + "affinity": { + "description": "Specify scheduling constraints for pods created with pod-template-file.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for pods created with pod-template-file.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for pods created with pod-template-file.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "hostAliases": { + "description": "Specify HostAliases for pods created with pod-template-file.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.HostAlias" + }, + "type": "array", + "default": [], + "examples": [ + { + "ip": "127.0.0.2", + "hostnames": [ + "test.hostname.one" + ] + }, + { + "ip": "127.0.0.3", + "hostnames": [ + "test.hostname.two" + ] + } + ] + }, + "podAnnotations": { + "description": "Annotations to add to the pods created with pod-template-file (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "description": "Labels to add to the pods created with pod-template-file.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "env": { + "description": "Add additional env vars to the pods created with pod-template-file.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] + } + }, + "required": [ + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } + ], + "additionalProperties": false + } + }, + "schedulerName": { + "description": "Specify kube scheduler name for pods created with pod-template-file.", + "type": [ + "string", + "null" + ], + "default": null, + "x-docsSection": "Common" + } + } + } + } + }, + "scheduler": { + "description": "Airflow scheduler settings.", + "type": "object", + "x-docsSection": "Scheduler", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable scheduler", + "type": "boolean", + "default": true + }, + "hostAliases": { + "description": "HostAliases for the scheduler pod.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.HostAlias" + }, + "type": "array", + "default": [], + "examples": [ + { + "ip": "127.0.0.1", + "hostnames": [ + "foo.local" + ] + }, + { + "ip": "10.1.2.3", + "hostnames": [ + "foo.remote" + ] + } + ] + }, + "livenessProbe": { + "description": "Liveness probe configuration for scheduler container.", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "Number of seconds after the container has started before liveness probes are initiated.", + "type": "integer", + "default": 10 + }, + "timeoutSeconds": { + "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds.", + "type": "integer", + "default": 20 + }, + "failureThreshold": { + "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1.", + "type": "integer", + "default": 5 + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe. Minimum value is 1.", + "type": "integer", + "default": 60 + }, + "command": { + "description": "Command for LivenessProbe", + "type": [ + "array", + "null" + ], + "default": null, + "items": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "startupProbe": { + "description": "Startup probe configuration for scheduler container.", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "Number of seconds after the container has started before startup probes are initiated.", + "type": "integer", + "default": 0 + }, + "timeoutSeconds": { + "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds.", + "type": "integer", + "default": 20 + }, + "failureThreshold": { + "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1.", + "type": "integer", + "default": 6 + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe. Minimum value is 1.", + "type": "integer", + "default": 10 + }, + "command": { + "description": "Command for LivenessProbe", + "type": [ + "array", + "null" + ], + "default": null, + "items": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "replicas": { + "description": "Amount of scheduler replicas", + "type": "integer", + "default": 1 + }, + "revisionHistoryLimit": { + "description": "Number of old ReplicaSets to retain.", + "type": [ + "integer", + "null" + ], + "default": null, + "x-docsSection": "Kubernetes" + }, + "command": { + "description": "Command to use when running the Airflow scheduler (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running the Airflow scheduler (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "bash", + "-c", + "exec airflow scheduler" + ] + }, + "updateStrategy": { + "description": "Specifies the strategy used to replace old Pods by new ones when deployed as a StatefulSet (when using LocalExecutor and workers.persistence).", + "type": [ + "null", + "object" + ], + "default": null + }, + "strategy": { + "description": "Specifies the strategy used to replace old Pods by new ones when deployed as a Deployment (when not using LocalExecutor and workers.persistence).", + "type": [ + "null", + "object" + ], + "default": null + }, + "terminationGracePeriodSeconds": { + "description": "Grace period for scheduler to finish after SIGTERM is sent from Kubernetes.", + "type": "integer", + "default": 10 + }, + "serviceAccount": { + "description": "Create ServiceAccount.", + "type": "object", + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods. When false, you can use `serviceAccountTokenVolume` to manually configure service account token volume for pod-launching executors.", + "type": "boolean", + "default": true + }, + "serviceAccountTokenVolume": { + "description": "Configuration for manual service account token volume. Only used when automountServiceAccountToken is false and for pod-launching executors. (CeleryExecutor, CeleryKubernetesExecutor, KubernetesExecutor, LocalKubernetesExecutor)", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable manual service account token volume configuration.", + "type": "boolean", + "default": false + }, + "mountPath": { + "description": "Path where the service account token volume will be mounted.", + "type": "string", + "default": "/var/run/secrets/kubernetes.io/serviceaccount" + }, + "volumeName": { + "description": "Name of the service account token volume.", + "type": "string", + "default": "kube-api-access" + }, + "expirationSeconds": { + "description": "Token expiration time in seconds.", + "type": "integer", + "minimum": 600, + "maximum": 7776000, + "default": 3600 + }, + "audience": { + "description": "Intended audience of the token. Optional - defaults to the identifier of the Kubernetes API server.", + "type": [ + "string", + "null" + ], + "default": null + } + } + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created.", + "type": "boolean", + "default": true + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the scheduler Kubernetes ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "podDisruptionBudget": { + "description": "Scheduler pod disruption budget.", + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "description": "Enable pod disruption budget.", + "type": "boolean", + "default": false + }, + "config": { + "description": "Disruption budget configuration.", + "type": "object", + "additionalProperties": true, + "properties": { + "maxUnavailable": { + "description": "Max unavailable pods for scheduler.", + "type": [ + "integer", + "string" + ], + "default": 1 + }, + "minAvailable": { + "description": "Min available pods for scheduler.", + "type": [ + "integer", + "string" + ], + "default": 1 + } + } + } + } + }, + "resources": { + "description": "Resources for scheduler pods.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "safeToEvict": { + "description": "This setting tells Kubernetes that its ok to evict when it wants to scale a node down.", + "type": "boolean", + "default": true + }, + "extraContainers": { + "description": "Launch additional containers into scheduler (templated).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraInitContainers": { + "description": "Add additional init containers into scheduler (templated).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraVolumes": { + "description": "Mount additional volumes into scheduler.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "extraVolumeMounts": { + "description": "Mount additional volumes into scheduler.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "nodeSelector": { + "description": "Select certain nodes for scheduler pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "priorityClassName": { + "description": "Specify priority for scheduler pods.", + "type": [ + "string", + "null" + ], + "default": null + }, + "affinity": { + "description": "Specify scheduling constraints for scheduler pods.", + "type": "object", + "default": "See values.yaml", + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for scheduler pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for scheduler pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "annotations": { + "description": "Annotations to add to the scheduler deployment", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "podAnnotations": { + "description": "Annotations to add to the scheduler pods (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "description": "Labels to add to the scheduler objects and pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "logGroomerSidecar": { + "$ref": "#/definitions/logGroomerConfigType", + "description": "Configuration for the schedulers log groomer sidecar." + }, + "securityContext": { + "description": "Security context for the scheduler pod (deprecated, use `securityContexts` instead). If not set, the values from `securityContext` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the scheduler. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the scheduler. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition for the scheduler.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition for the scheduler.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "waitForMigrations": { + "description": "wait-for-airflow-migrations init container.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable wait-for-airflow-migrations init container.", + "type": "boolean", + "default": true + }, + "env": { + "description": "Add additional env vars to wait-for-airflow-migrations init container.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": false + } + }, + "securityContexts": { + "description": "Security context definition for the wait for migrations. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the wait for migrations.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "env": { + "description": "Add additional env vars to scheduler.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] + } + }, + "required": [ + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } + ], + "additionalProperties": false + } + } + } + }, + "triggerer": { + "description": "Airflow triggerer settings.", + "type": "object", + "x-docsSection": "Triggerer", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable triggerer", + "type": "boolean", + "default": true + }, + "hostAliases": { + "description": "HostAliases for the triggerer pod.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.HostAlias" + }, + "type": "array", + "default": [], + "examples": [ + { + "ip": "127.0.0.1", + "hostnames": [ + "foo.local" + ] + }, + { + "ip": "10.1.2.3", + "hostnames": [ + "foo.remote" + ] + } + ] + }, + "livenessProbe": { + "description": "Liveness probe configuration for triggerer.", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "Number of seconds after the container has started before liveness probes are initiated.", + "type": "integer", + "default": 10 + }, + "timeoutSeconds": { + "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds.", + "type": "integer", + "default": 20 + }, + "failureThreshold": { + "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1.", + "type": "integer", + "default": 5 + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe. Minimum value is 1.", + "type": "integer", + "default": 60 + }, + "command": { + "description": "Command for LivenessProbe", + "type": [ + "array", + "null" + ], + "default": null, + "items": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "replicas": { + "description": "Number of triggerers to run.", + "type": "integer", + "default": 1 + }, + "revisionHistoryLimit": { + "description": "Number of old ReplicaSets to retain.", + "type": [ + "integer", + "null" + ], + "default": null, + "x-docsSection": "Kubernetes" + }, + "command": { + "description": "Command to use when running the Airflow triggerer (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running the Airflow triggerer (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "bash", + "-c", + "exec airflow triggerer" + ] + }, + "updateStrategy": { + "description": "Specifies the strategy used to replace old Pods by new ones when deployed as a StatefulSet.", + "type": [ + "null", + "object" + ], + "default": null + }, + "strategy": { + "description": "Specifies the strategy used to replace old Pods by new ones when deployed as a Deployment.", + "type": [ + "null", + "object" + ], + "$ref": "#/definitions/io.k8s.api.apps.v1.DeploymentStrategy", + "default": { + "rollingUpdate": { + "maxSurge": "100%", + "maxUnavailable": "50%" + } + } + }, + "serviceAccount": { + "description": "Create ServiceAccount.", + "type": "object", + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods.", + "type": "boolean", + "default": true + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created.", + "type": "boolean", + "default": true + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the triggerer Kubernetes ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "persistence": { + "description": "Persistence configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable persistent volumes.", + "type": "boolean", + "default": true + }, + "size": { + "description": "Volume size for triggerer StatefulSet.", + "type": "string", + "default": "100Gi" + }, + "persistentVolumeClaimRetentionPolicy": { + "$ref": "#/definitions/persistentVolumeClaimRetentionPolicy", + "description": "PersistentVolumeClaim retention policy to be used in the lifecycle of a StatefulSet" + }, + "storageClassName": { + "description": "If using a custom StorageClass, pass name ref to all StatefulSets here (templated).", + "type": [ + "string", + "null" + ], + "default": null + }, + "fixPermissions": { + "description": "Execute init container to chown log directory. This is currently only needed in kind, due to usage of local-path provisioner.", + "type": "boolean", + "default": false + }, + "annotations": { + "description": "Annotations to add to triggerer volumes.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "podDisruptionBudget": { + "description": "Triggerer pod disruption budget.", + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "description": "Enable pod disruption budget.", + "type": "boolean", + "default": false + }, + "config": { + "description": "Disruption budget configuration.", + "type": "object", + "additionalProperties": true, + "properties": { + "maxUnavailable": { + "description": "Max unavailable pods for triggerer.", + "type": [ + "integer", + "string" + ], + "default": 1 + }, + "minAvailable": { + "description": "Min available pods for triggerer.", + "type": [ + "integer", + "string" + ], + "default": 1 + } + } + } + } + }, + "resources": { + "description": "Resources for triggerer pods.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "safeToEvict": { + "description": "This setting tells Kubernetes that its ok to evict when it wants to scale a node down.", + "type": "boolean", + "default": true + }, + "extraContainers": { + "description": "Launch additional containers into triggerer (templated).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraInitContainers": { + "description": "Add additional init containers into triggerer (templated).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraVolumes": { + "description": "Mount additional volumes into triggerer.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "extraVolumeMounts": { + "description": "Mount additional volumes into triggerer.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "nodeSelector": { + "description": "Select certain nodes for triggerer pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "priorityClassName": { + "description": "Specify priority for triggerer pods.", + "type": [ + "string", + "null" + ], + "default": null + }, + "affinity": { + "description": "Specify scheduling constraints for triggerer pods.", + "type": "object", + "default": "See values.yaml", + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for triggerer pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for triggerer pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "terminationGracePeriodSeconds": { + "description": "Grace period for tasks to finish after SIGTERM is sent from Kubernetes.", + "type": "integer", + "default": 60 + }, + "annotations": { + "description": "Annotations to add to the triggerer deployment", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "podAnnotations": { + "description": "Annotations to add to the triggerer pods (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "description": "Labels to add to the triggerer objects and pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "securityContext": { + "description": "Security context for the triggerer pod (deprecated, use `securityContexts` instead). If not set, the values from `securityContext` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the triggerer. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the triggerer. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition for the triggerer.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition for the triggerer.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "logGroomerSidecar": { + "$ref": "#/definitions/logGroomerConfigType", + "description": "Configuration for log groomer sidecar" + }, + "waitForMigrations": { + "description": "wait-for-airflow-migrations init container.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable wait-for-airflow-migrations init container.", + "type": "boolean", + "default": true + }, + "env": { + "description": "Add additional env vars to wait-for-airflow-migrations init container.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": false + } + }, + "securityContexts": { + "description": "Security context definition for the wait for migrations. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the wait for migrations.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "env": { + "description": "Add additional env vars to triggerer.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] + } + }, + "required": [ + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } + ], + "additionalProperties": false + } + }, + "keda": { + "description": "KEDA configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Allow KEDA autoscaling.", + "type": "boolean", + "default": false + }, + "namespaceLabels": { + "description": "Labels used in `matchLabels` for namespace in the PgBouncer NetworkPolicy.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "pollingInterval": { + "description": "How often KEDA polls the airflow DB to report new scale requests to the HPA.", + "type": "integer", + "default": 5 + }, + "cooldownPeriod": { + "description": "How many seconds KEDA will wait before scaling to zero.", + "type": "integer", + "default": 30 + }, + "minReplicaCount": { + "description": "Minimum number of triggerers created by KEDA.", + "type": "integer", + "default": 0 + }, + "maxReplicaCount": { + "description": "Maximum number of triggerers created by KEDA.", + "type": "integer", + "default": 10 + }, + "advanced": { + "description": "Advanced KEDA configuration.", + "type": "object", + "default": {}, + "additionalProperties": false, + "properties": { + "horizontalPodAutoscalerConfig": { + "description": "HorizontalPodAutoscalerConfig specifies horizontal scale config.", + "type": "object", + "default": {}, + "properties": { + "behavior": { + "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HorizontalPodAutoscalerBehavior" + } + } + } + } + }, + "query": { + "description": "Query to use for KEDA autoscaling. Must return a single integer.", + "type": "string", + "default": "SELECT ceil(COUNT(*)::decimal / {{ include \"triggerer.capacity\" . }}) FROM trigger" + }, + "usePgbouncer": { + "description": "Whether to use PGBouncer to connect to the database or not when it is enabled. This configuration will be ignored if PGBouncer is not enabled.", + "type": "boolean", + "default": false + } + } + } + } + }, + "dagProcessor": { + "description": "Airflow dag processor settings.", + "type": "object", + "x-docsSection": "DagProcessor", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable standalone dag processor.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "dagBundleConfigList": { + "description": "Define Dag bundles in a structured YAML format. This will be automatically converted to JSON string format for config.dag_processor.dag_bundle_config_list.", + "type": "array", + "default": [ + { + "name": "dags-folder", + "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", + "kwargs": {} + } + ], + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "classpath" + ], + "properties": { + "name": { + "description": "Name of the Dag bundle.", + "type": "string" + }, + "classpath": { + "description": "Class path of the Dag bundle class.", + "type": "string" + }, + "kwargs": { + "description": "Keyword arguments for the Dag bundle.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": [ + "string", + "number", + "boolean", + "null" + ] + } + } + } + } + }, + "livenessProbe": { + "description": "Liveness probe configuration for dag processor.", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "Number of seconds after the container has started before liveness probes are initiated.", + "type": "integer", + "default": 10 + }, + "timeoutSeconds": { + "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds.", + "type": "integer", + "default": 20 + }, + "failureThreshold": { + "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1.", + "type": "integer", + "default": 5 + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe. Minimum value is 1.", + "type": "integer", + "default": 60 + }, + "command": { + "description": "Command for LivenessProbe", + "type": [ + "array", + "null" + ], + "default": null, + "items": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "replicas": { + "description": "Number of dag processors to run.", + "type": "integer", + "default": 1 + }, + "revisionHistoryLimit": { + "description": "Number of old ReplicaSets to retain.", + "type": [ + "integer", + "null" + ], + "default": null, + "x-docsSection": "Kubernetes" + }, + "command": { + "description": "Command to use when running the Airflow dag processor (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running the Airflow dag processor (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "bash", + "-c", + "exec airflow dag-processor" + ] + }, + "strategy": { + "description": "Specifies the strategy used to replace old Pods by new ones when deployed as a Deployment.", + "type": [ + "null", + "object" + ], + "$ref": "#/definitions/io.k8s.api.apps.v1.DeploymentStrategy", + "default": { + "rollingUpdate": { + "maxSurge": "100%", + "maxUnavailable": "50%" + } + } + }, + "serviceAccount": { + "description": "Create ServiceAccount.", + "type": "object", + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods.", + "type": "boolean", + "default": true + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created.", + "type": "boolean", + "default": true + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the dag processor Kubernetes ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "podDisruptionBudget": { + "description": "Dag processor pod disruption budget.", + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "description": "Enable pod disruption budget.", + "type": "boolean", + "default": false + }, + "config": { + "description": "Disruption budget configuration.", + "type": "object", + "additionalProperties": true, + "properties": { + "maxUnavailable": { + "description": "Max unavailable pods for dag processor.", + "type": [ + "integer", + "string" + ], + "default": 1 + }, + "minAvailable": { + "description": "Min available pods for dag processor.", + "type": [ + "integer", + "string" + ], + "default": 1 + } + } + } + } + }, + "resources": { + "description": "Resources for dag processor pods.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "safeToEvict": { + "description": "This setting tells Kubernetes that its ok to evict when it wants to scale a node down.", + "type": "boolean", + "default": true + }, + "extraContainers": { + "description": "Launch additional containers into dag processor (templated).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraInitContainers": { + "description": "Add additional init containers into dag processor (templated).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraVolumes": { + "description": "Mount additional volumes into dag processor.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "extraVolumeMounts": { + "description": "Mount additional volumes into dag processor.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "nodeSelector": { + "description": "Select certain nodes for dag processor pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "priorityClassName": { + "description": "Specify priority for dag processor pods.", + "type": [ + "string", + "null" + ], + "default": null + }, + "affinity": { + "description": "Specify scheduling constraints for dag processor pods.", + "type": "object", + "default": "See values.yaml", + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for dag processor pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for dag processor pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "terminationGracePeriodSeconds": { + "description": "Grace period for tasks to finish after SIGTERM is sent from Kubernetes.", + "type": "integer", + "default": 60 + }, + "annotations": { + "description": "Annotations to add to the dag processor deployment", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "podAnnotations": { + "description": "Annotations to add to the dag processor pods (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "securityContext": { + "description": "Security context for the dag processor pod (deprecated, use `securityContexts` instead). If not set, the values from `securityContext` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the dag processor. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the dag processor. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition for the dag processor.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition for the dag processor.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "logGroomerSidecar": { + "$ref": "#/definitions/logGroomerConfigType", + "description": "Configuration for log groomer sidecar" + }, + "waitForMigrations": { + "description": "wait-for-airflow-migrations init container.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable wait-for-airflow-migrations init container.", + "type": "boolean", + "default": true + }, + "env": { + "description": "Add additional env vars to wait-for-airflow-migrations init container.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": false + } + }, + "securityContexts": { + "description": "Security context definition for the wait for migrations. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the wait for migrations.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "labels": { + "description": "Labels specific to dag processor objects and pods", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "env": { + "description": "Add additional env vars to dag processor.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] + } + }, + "required": [ + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } + ], + "additionalProperties": false + } + } + } + }, + "createUserJob": { + "description": "Airflow job to create a user settings.", + "type": "object", + "x-docsSection": "Jobs", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Whether the create user job should be created.", + "type": "boolean", + "default": true + }, + "defaultUser": { + "description": "Optional default Airflow user information", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable default user creation.", + "type": "boolean", + "x-docsSection": "Common", + "default": true + }, + "role": { + "description": "Default user role.", + "type": "string", + "default": "Admin" + }, + "username": { + "description": "Default user username.", + "type": "string", + "default": "admin" + }, + "email": { + "description": "Default user email address.", + "type": "string", + "default": "admin@example.com" + }, + "firstName": { + "description": "Default user firstname.", + "type": "string", + "default": "admin" + }, + "lastName": { + "description": "Default user lastname.", + "type": "string", + "default": "user" + }, + "password": { + "description": "Default user password.", + "type": "string", + "default": "admin" + } + } + }, + "command": { + "description": "Command to use when running create user job (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running create user job (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "bash", + "-c", + "exec \\\nairflow users create \"$@\"", + "--", + "-r", + "{{ if .Values.webserver.defaultUser }}{{ .Values.webserver.defaultUser.role }}{{ else }}{{ .Values.createUserJob.defaultUser.role }}{{ end }}", + "-u", + "{{ if .Values.webserver.defaultUser }}{{ .Values.webserver.defaultUser.username }}{{ else }}{{ .Values.createUserJob.defaultUser.username }}{{ end }}", + "-e", + "{{ if .Values.webserver.defaultUser }}{{ .Values.webserver.defaultUser.email }}{{ else }}{{ .Values.createUserJob.defaultUser.email }}{{ end }}", + "-f", + "{{ if .Values.webserver.defaultUser }}{{ .Values.webserver.defaultUser.firstName }}{{ else }}{{ .Values.createUserJob.defaultUser.firstName }}{{ end }}", + "-l", + "{{ if .Values.webserver.defaultUser }}{{ .Values.webserver.defaultUser.lastName }}{{ else }}{{ .Values.createUserJob.defaultUser.lastName }}{{ end }}", + "-p", + "{{ if .Values.webserver.defaultUser }}{{ .Values.webserver.defaultUser.password }}{{ else }}{{ .Values.createUserJob.defaultUser.password }}{{ end }}" + ] + }, + "annotations": { + "description": "Annotations to add to the create user job pod (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "jobAnnotations": { + "description": "Annotations to add to the create user job job.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "restartPolicy": { + "type": "string", + "description": "Restart policy for the database migration job.", + "enum": [ + "OnFailure", + "Never" + ], + "default": "OnFailure" + }, + "labels": { + "description": "Labels to add to the create user job objects and pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "serviceAccount": { + "description": "Create ServiceAccount.", + "type": "object", + "additionalProperties": false, + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods.", + "type": "boolean", + "default": true + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created.", + "type": "boolean", + "default": true + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the create user job Kubernetes ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "extraContainers": { + "description": "Launch additional containers for the create user job pod", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraInitContainers": { + "description": "Add additional init containers into create user job pod (templated).", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + }, + "type": "array", + "default": [] + }, + "extraVolumes": { + "description": "Mount additional volumes into create user job", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "extraVolumeMounts": { + "description": "Mount additional volumes into create user job", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "nodeSelector": { + "description": "Select certain nodes for the create user job pod.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "affinity": { + "description": "Specify scheduling constraints for the create user job pod.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for the create user job pod.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for the create user job pod.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "securityContext": { + "description": "Security context for the create user job pod (deprecated, use `securityContexts` instead). If not set, the values from `securityContext` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the create user job. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the create user job. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition for the create user job.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition for the create user job.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "resources": { + "description": "Resources for the create user job pod", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ] + }, + "priorityClassName": { + "description": "Specify priority for the create user job pod.", + "type": [ + "string", + "null" + ], + "default": null + }, + "useHelmHooks": { + "description": "Specify if you want to use the default Helm Hook annotations", + "type": "boolean", + "default": true + }, + "applyCustomEnv": { + "description": "Specify if you want additional configured env vars applied to this job", + "type": "boolean", + "default": true + }, + "ttlSecondsAfterFinished": { + "description": "Limit the lifetime of the job object after it finished execution", + "type": [ + "integer", + "null" + ], + "default": 300 + }, + "env": { + "description": "Add additional env vars to the create user job pod.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] + } + }, + "required": [ + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } + ], + "additionalProperties": false + } + } + } + }, + "migrateDatabaseJob": { + "description": "Airflow job to migrate databases settings.", + "type": "object", + "x-docsSection": "Jobs", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable migrate database job.", + "type": "boolean", + "default": true + }, + "command": { + "description": "Command to use when running migrate database job (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running migrate database job (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "bash", + "-c", + "exec \\\nairflow db migrate" + ] + }, + "annotations": { + "description": "Annotations to add to the migrate database job pod (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "jobAnnotations": { + "description": "Annotations to add to the migrate database job.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "restartPolicy": { + "type": "string", + "description": "Restart policy for the database migration job.", + "enum": [ + "OnFailure", + "Never" + ], + "default": "OnFailure" + }, + "labels": { + "description": "Labels to add to the migrate database job objects and pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "serviceAccount": { + "description": "Create ServiceAccount.", + "type": "object", + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods.", + "type": "boolean", + "default": true + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created.", + "type": "boolean", + "default": true + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the migrate database job Kubernetes ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "resources": { + "description": "Resources for the migrate database job pod", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ] + }, + "extraInitContainers": { + "description": "Add additional init containers into migrate database job (templated).", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + }, + "type": "array", + "default": [] + }, + "extraContainers": { + "description": "Launch additional containers for the migrate database job pod", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraVolumes": { + "description": "Mount additional volumes into migrate database job", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "extraVolumeMounts": { + "description": "Mount additional volumes into migrate database job", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "nodeSelector": { + "description": "Select certain nodes for the migrate database job pod.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "affinity": { + "description": "Specify scheduling constraints for the migrate database job pod.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for the migrate database job pod.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for migrate database job pod.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "securityContext": { + "description": "Security context for the migrate database job pod (deprecated, use `securityContexts` instead). If not set, the values from `securityContext` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the migrate database job. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the migrate database job. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition for the migrate database job.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition for the migrate database job.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "priorityClassName": { + "description": "Specify priority for the migrate database job pod.", + "type": [ + "string", + "null" + ], + "default": null + }, + "useHelmHooks": { + "description": "Specify if you want to use the default Helm Hook annotations", + "type": "boolean", + "default": true + }, + "applyCustomEnv": { + "description": "Specify if you want additional configured env vars applied to this job", + "type": "boolean", + "default": true + }, + "ttlSecondsAfterFinished": { + "description": "Limit the lifetime of the job object after it finished execution", + "type": [ + "integer", + "null" + ], + "default": 300 + }, + "env": { + "description": "Add additional env vars to migrate database job.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar" + }, + "type": "array", + "default": [], + "x-kubernetes-patch-merge-key": "name", + "x-kubernetes-patch-strategy": "merge" + } + } + }, + "apiServer": { + "description": "Airflow API server settings. Airflow 3+ only.", + "type": "object", + "x-docsSection": "API Server", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable Airflow API server deployment.", + "type": "boolean", + "default": true + }, + "configMapAnnotations": { + "description": "Extra annotations to apply to the API server configmap.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "hostAliases": { + "description": "HostAliases for the API server pod.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.HostAlias" + }, + "type": "array", + "default": [], + "examples": [ + { + "ip": "127.0.0.1", + "hostnames": [ + "foo.local" + ] + }, + { + "ip": "10.1.2.3", + "hostnames": [ + "foo.remote" + ] + } + ] + }, + "allowPodLogReading": { + "description": "Allow API server to read k8s pod logs. Useful when you don't have an external log store.", + "type": "boolean", + "default": true + }, + "livenessProbe": { + "description": "Liveness probe configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "API server Liveness probe initial delay.", + "type": "integer", + "default": 15 + }, + "timeoutSeconds": { + "description": "API server Liveness probe timeout seconds.", + "type": "integer", + "default": 5 + }, + "failureThreshold": { + "description": "API server Liveness probe failure threshold.", + "type": "integer", + "default": 5 + }, + "periodSeconds": { + "description": "API server Liveness probe period seconds.", + "type": "integer", + "default": 10 + }, + "scheme": { + "description": "API server Liveness probe scheme.", + "type": "string", + "default": "HTTP" + } + } + }, + "readinessProbe": { + "description": "Readiness probe configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "API server Readiness probe initial delay.", + "type": "integer", + "default": 15 + }, + "timeoutSeconds": { + "description": "API server Readiness probe timeout seconds.", + "type": "integer", + "default": 5 + }, + "failureThreshold": { + "description": "API server Readiness probe failure threshold.", + "type": "integer", + "default": 5 + }, + "periodSeconds": { + "description": "API server Readiness probe period seconds.", + "type": "integer", + "default": 10 + }, + "scheme": { + "description": "API server Readiness probe scheme.", + "type": "string", + "default": "HTTP" + } + } + }, + "startupProbe": { + "description": "Startup probe configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "API server Startup probe initial delay seconds.", + "type": "integer", + "default": 0 + }, + "timeoutSeconds": { + "description": "API server Startup probe timeout seconds.", + "type": "integer", + "default": 20 + }, + "failureThreshold": { + "description": "API server Startup probe failure threshold.", + "type": "integer", + "default": 6 + }, + "periodSeconds": { + "description": "API server Startup probe period seconds.", + "type": "integer", + "default": 10 + }, + "scheme": { + "description": "API server Startup probe scheme.", + "type": "string", + "default": "HTTP" + } + } + }, + "replicas": { + "description": "How many Airflow API server replicas should run. Omitted from the Deployment, when HPA is enabled.", + "type": [ + "integer", + "null" + ], + "default": 1 + }, + "revisionHistoryLimit": { + "description": "Number of old ReplicaSets to retain.", + "type": [ + "integer", + "null" + ], + "default": null, + "x-docsSection": "Kubernetes" + }, + "command": { + "description": "Command to use when running the Airflow API server (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running the Airflow API server (templated). When running behind a reverse proxy, add `--proxy-headers` to enable Uvicorn to respect X-Forwarded-Proto, X-Forwarded-For, and X-Forwarded-Port headers.", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "bash", + "-c", + "exec airflow api-server" + ], + "examples": [ + [ + "bash", + "-c", + "exec airflow api-server --proxy-headers" + ] + ] + }, + "strategy": { + "description": "Specifies the strategy used to replace old Pods by new ones.", + "type": [ + "null", + "object" + ], + "default": null + }, + "hpa": { + "description": "Horizontal Pod Autoscaler (HPA) configuration for apiServer. (optional)", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable HPA autoscaling for API server", + "type": "boolean", + "default": false + }, + "minReplicaCount": { + "description": "Minimum number of API server replicas created by HPA if HPA is enabled.", + "type": "integer", + "default": 1 + }, + "maxReplicaCount": { + "description": "Maximum number of API server replicas created by HPA if HPA is enabled.", + "type": "integer", + "default": 5 + }, + "metrics": { + "description": "Specifications for which to use to calculate the desired replica count.", + "type": "array", + "default": [ + { + "type": "Resource", + "resource": { + "name": "cpu", + "target": { + "type": "Utilization", + "averageUtilization": 50 + } + } + } + ], + "items": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricSpec" + } + }, + "behavior": { + "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HorizontalPodAutoscalerBehavior" + } + } + }, + "serviceAccount": { + "description": "Create ServiceAccount.", + "type": "object", + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods.", + "type": "boolean", + "default": true + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created.", + "type": "boolean", + "default": true + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the API server Kubernetes ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "podDisruptionBudget": { + "description": "API server pod disruption budget.", + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "description": "Enable pod disruption budget.", + "type": "boolean", + "default": false + }, + "config": { + "description": "Disruption budget configuration.", + "type": "object", + "additionalProperties": true, + "properties": { + "maxUnavailable": { + "description": "Max unavailable pods for API server.", + "type": [ + "integer", + "string" + ], + "default": 1 + }, + "minAvailable": { + "description": "Min available pods for API server.", + "type": [ + "integer", + "string" + ], + "default": 1 + } + } + } + } + }, + "networkPolicy": { + "description": "API server NetworkPolicy configuration", + "type": "object", + "additionalProperties": false, + "properties": { + "ingress": { + "description": "API server NetworkPolicyingress configuration", + "type": "object", + "additionalProperties": false, + "properties": { + "from": { + "description": "Peers for API server NetworkPolicyingress.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyPeer" + } + }, + "ports": { + "description": "Ports for API server NetworkPolicyingress (if `from` is set).", + "type": "array", + "items": { + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyPort" + }, + "default": [ + { + "port": "{{ .Values.ports.apiServer }}" + } + ], + "examples": [ + { + "port": 8080 + } + ] + } + } + } + } + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the API server. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the API server. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition for the API server.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition for the API server.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "resources": { + "description": "Resources for API server pods.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "apiServerConfig": { + "description": "This string (templated) will be mounted into the Airflow API Server as a custom `webserver_config.py`. You can bake a `webserver_config.py` in to your image instead or specify a configmap containing the webserver_config.py.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Common", + "default": null, + "examples": [ + "from airflow import configuration as conf\n\n# The SQLAlchemy connection string.\nSQLALCHEMY_DATABASE_URI = conf.get('database', 'SQL_ALCHEMY_CONN')\n\n# Flask-WTF flag for CSRF\nCSRF_ENABLED = True" + ] + }, + "apiServerConfigConfigMapName": { + "description": "The configmap name containing the webserver_config.py.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Common", + "default": null, + "examples": [ + "my-api-server-configmap" + ] + }, + "extraContainers": { + "description": "Launch additional containers into API server.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraInitContainers": { + "description": "Add additional init containers into API server.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraVolumes": { + "description": "Mount additional volumes into API server.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "extraVolumeMounts": { + "description": "Mount additional volumes into API server.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "service": { + "description": "API server Service configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "description": "API server Service type.", + "type": "string", + "default": "ClusterIP" + }, + "annotations": { + "description": "Annotations for the API server Service.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "ports": { + "description": "Ports for the API server Service.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "port": { + "type": [ + "string", + "integer" + ] + }, + "targetPort": { + "type": [ + "string", + "integer" + ] + }, + "nodePort": { + "type": [ + "string", + "integer" + ] + }, + "protocol": { + "type": "string" + } + } + }, + "default": [ + { + "name": "api-server", + "port": "{{ .Values.ports.apiServer }}" + } + ], + "examples": [ + { + "name": "api-server", + "port": 8080, + "targetPort": "api-server" + }, + { + "name": "only_sidecar", + "port": 9080, + "targetPort": 8888 + } + ] + }, + "loadBalancerIP": { + "description": "API server Service loadBalancerIP.", + "type": [ + "string", + "null" + ], + "default": null + }, + "loadBalancerSourceRanges": { + "description": "API server Service ``loadBalancerSourceRanges``.", + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "examples": [ + "10.123.0.0/16" + ] + } + } + }, + "nodeSelector": { + "description": "Select certain nodes for API server pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "priorityClassName": { + "description": "Specify priority for API server pods.", + "type": [ + "string", + "null" + ], + "default": null + }, + "affinity": { + "description": "Specify scheduling constraints for API server pods.", + "type": "object", + "default": "See values.yaml", + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for API server pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for API server pods.", + "type": "array", + "default": [], + "x-docsSection": "Kubernetes", + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "annotations": { + "description": "Annotations to add to the API server deployment", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "podAnnotations": { + "description": "Annotations to add to the API server pods (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "description": "Labels to add to the API server objects and pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "waitForMigrations": { + "description": "wait-for-airflow-migrations init container.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable wait-for-airflow-migrations init container.", + "type": "boolean", + "default": true + }, + "env": { + "description": "Add additional env vars to wait-for-airflow-migrations init container.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": false + } + }, + "securityContexts": { + "description": "Security context definition for the wait for migrations. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the wait for migrations.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "env": { + "description": "Add additional env vars to API server. When running behind a reverse proxy, set `FORWARDED_ALLOW_IPS` to specify which IPs are trusted to send X-Forwarded-* headers. Use `\"*\"` for trusted environments, or specify proxy IP ranges for production.", + "type": "array", + "default": [], + "examples": [ + [ + { + "name": "FORWARDED_ALLOW_IPS", + "value": "*" + } + ] + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] + } + }, + "required": [ + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } + ], + "additionalProperties": false + } + } + } + }, + "webserver": { + "description": "Airflow webserver settings. Airflow 2 only.", + "type": "object", + "x-docsSection": "Webserver", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable webserver", + "type": "boolean", + "default": true + }, + "configMapAnnotations": { + "description": "Extra annotations to apply to the webserver configmap.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "hostAliases": { + "description": "HostAliases for the webserver pod.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.HostAlias" + }, + "type": "array", + "default": [], + "examples": [ + { + "ip": "127.0.0.1", + "hostnames": [ + "foo.local" + ] + }, + { + "ip": "10.1.2.3", + "hostnames": [ + "foo.remote" + ] + } + ] + }, + "allowPodLogReading": { + "description": "Allow webserver to read k8s pod logs. Useful when you don't have an external log store.", + "type": "boolean", + "default": true + }, + "livenessProbe": { + "description": "Liveness probe configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "Webserver Liveness probe initial delay.", + "type": "integer", + "default": 15 + }, + "timeoutSeconds": { + "description": "Webserver Liveness probe timeout seconds.", + "type": "integer", + "default": 5 + }, + "failureThreshold": { + "description": "Webserver Liveness probe failure threshold.", + "type": "integer", + "default": 5 + }, + "periodSeconds": { + "description": "Webserver Liveness probe period seconds.", + "type": "integer", + "default": 10 + }, + "scheme": { + "description": "Webserver Liveness probe scheme.", + "type": "string", + "default": "HTTP" + } + } + }, + "readinessProbe": { + "description": "Readiness probe configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "Webserver Readiness probe initial delay.", + "type": "integer", + "default": 15 + }, + "timeoutSeconds": { + "description": "Webserver Readiness probe timeout seconds.", + "type": "integer", + "default": 5 + }, + "failureThreshold": { + "description": "Webserver Readiness probe failure threshold.", + "type": "integer", + "default": 5 + }, + "periodSeconds": { + "description": "Webserver Readiness probe period seconds.", + "type": "integer", + "default": 10 + }, + "scheme": { + "description": "Webserver Readiness probe scheme.", + "type": "string", + "default": "HTTP" + } + } + }, + "startupProbe": { + "description": "Startup probe configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "Webserver Startup probe initial delay seconds.", + "type": "integer", + "default": 0 + }, + "timeoutSeconds": { + "description": "Webserver Startup probe timeout seconds.", + "type": "integer", + "default": 20 + }, + "failureThreshold": { + "description": "Webserver Startup probe failure threshold.", + "type": "integer", + "default": 6 + }, + "periodSeconds": { + "description": "Webserver Startup probe period seconds.", + "type": "integer", + "default": 10 + }, + "scheme": { + "description": "Webserver Startup probe scheme.", + "type": "string", + "default": "HTTP" + } + } + }, + "replicas": { + "description": "How many Airflow webserver replicas should run.", + "type": "integer", + "default": 1 + }, + "revisionHistoryLimit": { + "description": "Number of old ReplicaSets to retain.", + "type": [ + "integer", + "null" + ], + "default": null, + "x-docsSection": "Kubernetes" + }, + "command": { + "description": "Command to use when running the Airflow webserver (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running the Airflow webserver (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "bash", + "-c", + "exec airflow webserver" + ] + }, + "strategy": { + "description": "Specifies the strategy used to replace old Pods by new ones.", + "type": [ + "null", + "object" + ], + "default": null + }, + "terminationGracePeriodSeconds": { + "description": "Grace period for webserver to finish after SIGTERM is sent from Kubernetes.", + "type": "integer", + "default": 30 + }, + "hpa": { + "description": "HPA configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Allow HPA autoscaling", + "type": "boolean", + "default": false + }, + "minReplicaCount": { + "description": "Minimum number of webservers created by HPA.", + "type": "integer", + "default": 1 + }, + "maxReplicaCount": { + "description": "Maximum number of webservers created by HPA.", + "type": "integer", + "default": 5 + }, + "metrics": { + "description": "Specifications for which to use to calculate the desired replica count.", + "type": "array", + "default": [ + { + "type": "Resource", + "resource": { + "name": "cpu", + "target": { + "type": "Utilization", + "averageUtilization": 80 + } + } + } + ], + "items": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricSpec" + } + }, + "behavior": { + "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HorizontalPodAutoscalerBehavior" + } + } + }, + "serviceAccount": { + "description": "Create ServiceAccount.", + "type": "object", + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods.", + "type": "boolean", + "default": true + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created.", + "type": "boolean", + "default": true + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the webserver Kubernetes ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "podDisruptionBudget": { + "description": "Webserver pod disruption budget.", + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "description": "Enable pod disruption budget.", + "type": "boolean", + "default": false + }, + "config": { + "description": "Disruption budget configuration.", + "type": "object", + "additionalProperties": true, + "properties": { + "maxUnavailable": { + "description": "Max unavailable pods for webserver.", + "type": [ + "integer", + "string" + ], + "default": 1 + }, + "minAvailable": { + "description": "Min available pods for webserver.", + "type": [ + "integer", + "string" + ], + "default": 1 + } + } + } + } + }, + "extraNetworkPolicies": { + "description": "Additional NetworkPolicies as needed (Deprecated - renamed to `webserver.networkPolicy.ingress.from`).", + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyPeer" + }, + "default": [] + }, + "networkPolicy": { + "description": "Webserver NetworkPolicy configuration", + "type": "object", + "additionalProperties": false, + "properties": { + "ingress": { + "description": "Webserver NetworkPolicyingress configuration", + "type": "object", + "additionalProperties": false, + "properties": { + "from": { + "description": "Peers for webserver NetworkPolicyingress.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyPeer" + } + }, + "ports": { + "description": "Ports for webserver NetworkPolicyingress (if `from` is set).", + "type": "array", + "items": { + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyPort" + }, + "default": [ + { + "port": "{{ .Values.ports.airflowUI }}" + } + ], + "examples": [ + { + "port": 8070 + } + ] + } + } + } + } + }, + "securityContext": { + "description": "Security context for the webserver job pod (deprecated, use `securityContexts` instead). If not set, the values from `securityContext` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the webserver. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the webserver. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition for the webserver.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition for the webserver.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "resources": { + "description": "Resources for webserver pods.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "defaultUser": { + "description": "Optional default Airflow user information (deprecated, use createUserJob section instead)", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable default user creation.", + "type": "boolean", + "x-docsSection": "Common" + }, + "role": { + "description": "Default user role.", + "type": "string" + }, + "username": { + "description": "Default user username.", + "type": "string" + }, + "email": { + "description": "Default user email address.", + "type": "string" + }, + "firstName": { + "description": "Default user firstname.", + "type": "string" + }, + "lastName": { + "description": "Default user lastname.", + "type": "string" + }, + "password": { + "description": "Default user password.", + "type": "string" + } + } + }, + "extraContainers": { + "description": "Launch additional containers into webserver (templated).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraInitContainers": { + "description": "Add additional init containers into webserver (templated).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraVolumes": { + "description": "Mount additional volumes into webserver.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "extraVolumeMounts": { + "description": "Mount additional volumes into webserver.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "webserverConfig": { + "description": "This string (templated) will be mounted into the Airflow webserver as a custom `webserver_config.py`. You can bake a `webserver_config.py` in to your image instead or specify a configmap containing the webserver_config.py.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Common", + "default": null, + "examples": [ + "from airflow import configuration as conf\n\n# The SQLAlchemy connection string.\nSQLALCHEMY_DATABASE_URI = conf.get('database', 'SQL_ALCHEMY_CONN')\n\n# Flask-WTF flag for CSRF\nCSRF_ENABLED = True" + ] + }, + "webserverConfigConfigMapName": { + "description": "The configmap name containing the webserver_config.py.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Common", + "default": null, + "examples": [ + "my-webserver-configmap" + ] + }, + "service": { + "description": "Webserver Service configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "description": "Webserver Service type.", + "type": "string", + "default": "ClusterIP" + }, + "annotations": { + "description": "Annotations for the webserver Service.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "ports": { + "description": "Ports for the webserver Service.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "port": { + "type": [ + "string", + "integer" + ] + }, + "targetPort": { + "type": [ + "string", + "integer" + ] + }, + "nodePort": { + "type": [ + "string", + "integer" + ] + }, + "protocol": { + "type": "string" + } + } + }, + "default": [ + { + "name": "airflow-ui", + "port": "{{ .Values.ports.airflowUI }}" + } + ], + "examples": [ + { + "name": "airflow-ui", + "port": 80, + "targetPort": "airflow-ui" + }, + { + "name": "only_sidecar", + "port": 80, + "targetPort": 8888 + } + ] + }, + "loadBalancerIP": { + "description": "Webserver Service loadBalancerIP.", + "type": [ + "string", + "null" + ], + "default": null + }, + "loadBalancerSourceRanges": { + "description": "Webserver Service ``loadBalancerSourceRanges``.", + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "examples": [ + "10.123.0.0/16" + ] + } + } + }, + "nodeSelector": { + "description": "Select certain nodes for webserver pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "priorityClassName": { + "description": "Specify priority for webserver pods.", + "type": [ + "string", + "null" + ], + "default": null + }, + "affinity": { + "description": "Specify scheduling constraints for webserver pods.", + "type": "object", + "default": "See values.yaml", + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for webserver pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for webserver pods.", + "type": "array", + "default": [], + "x-docsSection": "Kubernetes", + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "annotations": { + "description": "Annotations to add to the webserver deployment", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "podAnnotations": { + "description": "Annotations to add to the webserver pods (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "description": "Labels to add to the webserver objects and pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "waitForMigrations": { + "description": "wait-for-airflow-migrations init container.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable wait-for-airflow-migrations init container.", + "type": "boolean", + "default": true + }, + "env": { + "description": "Add additional env vars to wait-for-airflow-migrations init container.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": false + } + }, + "securityContexts": { + "description": "Security context definition for the wait for migrations. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the wait for migrations.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "env": { + "description": "Add additional env vars to webserver.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] + } + }, + "required": [ + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } + ], + "additionalProperties": false + } + } + } + }, + "flower": { + "description": "Flower settings.", + "type": "object", + "x-docsSection": "Flower", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable Flower.", + "type": "boolean", + "default": false + }, + "livenessProbe": { + "description": "Liveness probe configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "Flower Liveness probe initial delay.", + "type": "integer", + "default": 10 + }, + "timeoutSeconds": { + "description": "Flower Liveness probe timeout seconds.", + "type": "integer", + "default": 5 + }, + "failureThreshold": { + "description": "Flower Liveness probe failure threshold.", + "type": "integer", + "default": 10 + }, + "periodSeconds": { + "description": "Flower Liveness probe period seconds.", + "type": "integer", + "default": 5 + } + } + }, + "readinessProbe": { + "description": "Readiness probe configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "Flower Readiness probe initial delay.", + "type": "integer", + "default": 10 + }, + "timeoutSeconds": { + "description": "Flower Readiness probe timeout seconds.", + "type": "integer", + "default": 5 + }, + "failureThreshold": { + "description": "Flower Readiness probe failure threshold.", + "type": "integer", + "default": 10 + }, + "periodSeconds": { + "description": "Flower Readiness probe period seconds.", + "type": "integer", + "default": 5 + } + } + }, + "startupProbe": { + "description": "Startup probe configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "Flower Startup probe initial delay seconds.", + "type": "integer", + "default": 0 + }, + "timeoutSeconds": { + "description": "Flower Startup probe timeout seconds.", + "type": "integer", + "default": 20 + }, + "failureThreshold": { + "description": "Flower Startup probe failure threshold.", + "type": "integer", + "default": 6 + }, + "periodSeconds": { + "description": "Flower Startup probe period seconds.", + "type": "integer", + "default": 10 + } + } + }, + "revisionHistoryLimit": { + "description": "Number of old ReplicaSets to retain.", + "type": [ + "integer", + "null" + ], + "default": null, + "x-docsSection": "Kubernetes" + }, + "command": { + "description": "Command to use when running flower (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running flower (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "bash", + "-c", + "exec \\\nairflow celery flower" + ] + }, + "extraNetworkPolicies": { + "description": "Additional NetworkPolicies as needed (Deprecated - renamed to `flower.networkPolicy.ingress.from`).", + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyPeer" + }, + "default": [] + }, + "networkPolicy": { + "description": "Flower NetworkPolicyconfiguration", + "type": "object", + "additionalProperties": false, + "properties": { + "ingress": { + "description": "Flower NetworkPolicyingress configuration", + "type": "object", + "additionalProperties": false, + "properties": { + "from": { + "description": "Peers for flower NetworkPolicyingress.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyPeer" + } + }, + "ports": { + "description": "Ports for flower NetworkPolicyingress (if `from` is set).", + "type": "array", + "items": { + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyPort" + }, + "default": [ + { + "port": "{{ .Values.ports.flowerUI }}" + } + ], + "examples": [ + { + "port": 5565 + } + ] + } + } + } + } + }, + "resources": { + "description": "Resources for Flower pods.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "secretName": { + "description": "A secret containing the user and password pair.", + "type": [ + "string", + "null" + ], + "default": null + }, + "secretAnnotations": { + "description": "Annotations to add to the flower secret.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "username": { + "description": "Username use to access Flower.", + "type": [ + "string", + "null" + ], + "default": null + }, + "password": { + "description": "Password use to access Flower.", + "type": [ + "string", + "null" + ], + "default": null + }, + "service": { + "description": "Flower Service configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "description": "Flower Service type.", + "type": "string", + "default": "ClusterIP" + }, + "annotations": { + "description": "Annotations for the flower Service.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "ports": { + "description": "Ports for the flower Service.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "port": { + "type": [ + "string", + "integer" + ] + }, + "targetPort": { + "type": [ + "string", + "integer" + ] + }, + "protocol": { + "type": "string" + } + } + }, + "default": [ + { + "name": "flower-ui", + "port": "{{ .Values.ports.flowerUI }}" + } + ], + "examples": [ + { + "name": "flower-ui", + "port": 8080, + "targetPort": "flower-ui" + } + ] + }, + "loadBalancerIP": { + "description": "Flower Service loadBalancerIP.", + "type": [ + "string", + "null" + ], + "default": null + }, + "loadBalancerSourceRanges": { + "description": "Flower Service ``loadBalancerSourceRanges``.", + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "examples": [ + "10.123.0.0/16" + ] + } + } + }, + "serviceAccount": { + "description": "Create ServiceAccount.", + "type": "object", + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods.", + "type": "boolean", + "default": true + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created.", + "type": "boolean", + "default": true + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the worker Kubernetes ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "extraContainers": { + "description": "Launch additional containers into the flower pods.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraVolumes": { + "description": "Mount additional volumes into the flower pods.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "extraVolumeMounts": { + "description": "Mount additional volumes into the flower pods.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "nodeSelector": { + "description": "Select certain nodes for Flower pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "priorityClassName": { + "description": "Specify priority for Flower pods.", + "type": [ + "string", + "null" + ], + "default": null + }, + "affinity": { + "description": "Specify scheduling constraints for Flower pods.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for Flower pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for Flower pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "annotations": { + "description": "Annotations to add to the flower deployment", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "podAnnotations": { + "description": "Annotations to add to the Flower pods (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "description": "Labels to add to the flower objects and pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "securityContext": { + "description": "Security context for the flower pod (deprecated, use `securityContexts` instead). If not set, the values from `securityContext` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the network policy. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the network policy. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition for the network policy.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition for the network policy.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "env": { + "description": "Add additional env vars to flower.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] + } + }, + "required": [ + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } + ], + "additionalProperties": false + } + } + } + }, + "statsd": { + "description": "StatsD settings.", + "type": "object", + "x-docsSection": "StatsD", + "additionalProperties": false, + "properties": { + "configMapAnnotations": { + "description": "Extra annotations to apply to the statsd configmap.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "enabled": { + "description": "Enable StatsD.", + "type": "boolean", + "default": true + }, + "cache": { + "description": "StatsD cache configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "size": { + "description": "Maximum number of metric mappings to cache in memory. Higher values improve performance for frequently used metrics but consume more memory.", + "type": "integer", + "default": 1000 + }, + "type": { + "description": "Cache eviction strategy for metric mappings. `lru` (Least Recently Used) evicts oldest accessed items, 'random' evicts randomly selected items.", + "type": "string", + "default": "lru" + }, + "ttl": { + "description": "Time-to-live for cached metric mappings. Determines how long mappings remain in cache before expiring. Set to '0s' to disable expiration.", + "type": "string", + "default": "0s" + } + } + }, + "revisionHistoryLimit": { + "description": "Number of old ReplicaSets to retain.", + "type": [ + "integer", + "null" + ], + "default": null, + "x-docsSection": "Kubernetes" + }, + "extraNetworkPolicies": { + "description": "Additional NetworkPolicies as needed.", + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyPeer" + }, + "default": [] + }, + "resources": { + "description": "Resources for StatsD pods.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "service": { + "description": "StatsD Service configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "extraAnnotations": { + "description": "Extra annotations for the StatsD Service.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "terminationGracePeriodSeconds": { + "description": "Grace period for statsd to finish after SIGTERM is sent from Kubernetes.", + "type": "integer", + "default": 30 + }, + "serviceAccount": { + "description": "Create ServiceAccount.", + "type": "object", + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods.", + "type": "boolean", + "default": true + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created.", + "type": "boolean", + "default": true + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the StatsD Kubernetes ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "uid": { + "description": "StatsD run as user parameter.", + "type": "integer", + "default": 65534 + }, + "nodeSelector": { + "description": "Select certain nodes for StatsD pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "priorityClassName": { + "description": "Specify priority for StatsD pods.", + "type": [ + "string", + "null" + ], + "default": null + }, + "affinity": { + "description": "Specify scheduling constraints for StatsD pods.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for StatsD pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for StatsD pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "extraMappings": { + "description": "Additional mappings for StatsD exporter.If set, will merge default mapping and extra mappings, default mapping has higher priority. So, if you want to change some default mapping, please use `overrideMappings`", + "type": "array", + "default": [] + }, + "overrideMappings": { + "description": "Override mappings for StatsD exporter.If set, will ignore setting item in default and `extraMappings`. So, If you use it, ensure all mapping item contains in it.", + "type": "array", + "default": [] + }, + "securityContext": { + "description": "Security context for the StatsD pod (deprecated, use `securityContexts` instead).", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the statsd. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the statsd.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition for the statsd.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition for the statsd.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "podAnnotations": { + "description": "Annotations to add to the StatsD pods (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "annotations": { + "description": "Annotations to add to the StatsD deployment.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "args": { + "description": "Args to use when running statsd-exporter (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "--statsd.mapping-config=/etc/statsd-exporter/mappings.yml" + ] + }, + "labels": { + "description": "Labels specific to statsd objects and pods", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "env": { + "description": "Add additional env vars to statsd container.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": false + } + } + } + }, + "pgbouncer": { + "description": "PgBouncer settings.", + "type": "object", + "x-docsSection": "PgBouncer", + "additionalProperties": false, + "properties": { + "env": { + "description": "Add additional env vars to `pgbouncer` container.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": false + } + }, + "labels": { + "description": "Labels to add to the PgBouncer objects and pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "enabled": { + "description": "Enable PgBouncer.", + "type": "boolean", + "x-docsSection": "Common", + "default": false + }, + "annotations": { + "description": "Annotations to add to the PgBouncer deployment", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "certificatesSecretAnnotations": { + "description": "Annotations to add to the PgBouncer certificates secret.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "replicas": { + "description": "Number of PgBouncer replicas to run in Deployment.", + "type": "integer", + "default": 1 + }, + "revisionHistoryLimit": { + "description": "Number of old ReplicaSets to retain.", + "type": [ + "integer", + "null" + ], + "default": null, + "x-docsSection": "Kubernetes" + }, + "command": { + "description": "Command to use for PgBouncer (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "pgbouncer", + "-u", + "nobody", + "/etc/pgbouncer/pgbouncer.ini" + ] + }, + "args": { + "description": "Args to use for PgBouncer (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "mountConfigSecret": { + "description": "Whether to mount the config secret files under `/etc/pgbouncer/` by default.", + "type": "boolean", + "x-docsSection": "Common", + "default": true + }, + "extraNetworkPolicies": { + "description": "Additional NetworkPolicies as needed.", + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyPeer" + }, + "default": [] + }, + "metadataPoolSize": { + "description": "Metadata pool size.", + "type": "integer", + "default": 10 + }, + "resultBackendPoolSize": { + "description": "Result backend pool size.", + "type": "integer", + "default": 5 + }, + "maxClientConn": { + "description": "Maximum clients that can connect to PgBouncer (higher = more file descriptors).", + "type": "integer", + "default": 100 + }, + "configSecretName": { + "description": "The PgBouncer config Secret name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "configSecretAnnotations": { + "description": "Annotations to add to the PgBouncer config secret.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "podAnnotations": { + "description": "Add annotations for the PgBouncer Pod (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "podDisruptionBudget": { + "description": "PgBouncer PodDisruptionBudget.", + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "description": "Enabled PodDistributionBudget.", + "type": "boolean", + "default": false + }, + "config": { + "description": "Pod distribution configuration.", + "type": "object", + "additionalProperties": true, + "properties": { + "maxUnavailable": { + "description": "Max unavailable pods for PgBouncer.", + "type": [ + "integer", + "string" + ], + "default": 1 + }, + "minAvailable": { + "description": "Min available pods for PgBouncer.", + "type": [ + "integer", + "string" + ], + "default": 1 + } + } + } + } + }, + "resources": { + "description": "Resources for the PgBouncer pods.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the PgBouncer. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": { + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "killall -INT pgbouncer && sleep 120" + ] + } + } + }, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the PgBouncer.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition for the PgBouncer.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 65534, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition for the PgBouncer.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "service": { + "description": "PgBouncer Service configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "extraAnnotations": { + "description": "Extra annotations for the PgBouncer Service.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "clusterIp": { + "description": "Specific ClusterIP for the PgBouncer Service.", + "type": [ + "string", + "null" + ], + "default": null + } + } + }, + "verbose": { + "description": "Increase PgBouncer verbosity.", + "type": "integer", + "default": 0 + }, + "auth_type": { + "description": "Method of authenticating users", + "type": "string", + "default": "scram-sha-256" + }, + "auth_file": { + "description": "The name of the file to load user names and passwords from", + "type": "string", + "default": "/etc/pgbouncer/users.txt" + }, + "logDisconnections": { + "description": "Log disconnections with reasons.", + "type": "integer", + "default": 0 + }, + "logConnections": { + "description": "Log successful logins.", + "type": "integer", + "default": 0 + }, + "sslmode": { + "description": "SSL mode for PgBouncer.", + "type": "string", + "enum": [ + "disable", + "allow", + "prefer", + "require", + "verify-ca", + "verify-full" + ], + "default": "prefer" + }, + "ciphers": { + "description": "The allowed ciphers, might be 'fast', 'normal' or list ciphers separated with ':'.", + "type": "string", + "default": "normal" + }, + "ssl": { + "description": "SSL certificates for PgBouncer connection.", + "type": "object", + "properties": { + "ca": { + "description": "Certificate Authority for server side", + "type": [ + "string", + "null" + ], + "default": null + }, + "cert": { + "description": "Server Certificate for server side", + "type": [ + "string", + "null" + ], + "default": null + }, + "key": { + "description": "Private key used to authenticate with the server", + "type": [ + "string", + "null" + ], + "default": null + } + } + }, + "extraIniMetadata": { + "description": "Add extra metadata database specific PgBouncer ini configuration: https://www.pgbouncer.org/config.html#section-databases", + "type": [ + "string", + "null" + ], + "default": null + }, + "extraIniResultBackend": { + "description": "Add extra result backend database specific PgBouncer ini configuration: https://www.pgbouncer.org/config.html#section-databases", + "type": [ + "string", + "null" + ], + "default": null + }, + "extraIni": { + "description": "Add extra general PgBouncer ini configuration: https://www.pgbouncer.org/config.html", + "type": [ + "string", + "null" + ], + "default": null + }, + "extraVolumes": { + "description": "Mount additional volumes into PgBouncer.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "extraVolumeMounts": { + "description": "Mount additional volumes into PgBouncer.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "extraContainers": { + "description": "Launch additional containers into `pgbouncer`.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "serviceAccount": { + "description": "Create ServiceAccount.", + "type": "object", + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods.", + "type": "boolean", + "default": true + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created.", + "type": "boolean", + "default": true + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the worker Kubernetes ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "nodeSelector": { + "description": "Select certain nodes for PgBouncer pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "priorityClassName": { + "description": "Specify priority for PgBouncer pods.", + "type": [ + "string", + "null" + ], + "default": null + }, + "affinity": { + "description": "Specify scheduling constraints for PgBouncer pods.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for PgBouncer pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for PgBouncer pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "uid": { + "description": "PgBouncer run as user parameter.", + "type": "integer", + "default": 65534 + }, + "metricsExporterSidecar": { + "description": "PgBouncer - metrics exporter settings.", + "type": "object", + "x-docsSection": "PgBouncer", + "additionalProperties": false, + "properties": { + "resources": { + "description": "Resources for the PgBouncer metric exporter.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "statsSecretName": { + "description": "Name of an existing Secrets object containing PgBouncer Metrics secrets.", + "type": [ + "string", + "null" + ], + "default": null + }, + "statsSecretKey": { + "description": "Key referencing the PGBouncer Metrics connection URI within an existing Secrets object. Defaults to `connection` if left null.", + "type": [ + "string", + "null" + ], + "default": null + }, + "statsSecretAnnotations": { + "description": "Annotations to add to the PgBouncer stats secret.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "sslmode": { + "description": "SSL mode for ``metricsExporterSidecar``", + "type": "string", + "enum": [ + "disable", + "require", + "verify-ca", + "verify-full" + ], + "default": "disable" + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the metrics exporter sidecar. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the metrics exporter sidecar. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the metrics exporter sidecar.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "livenessProbe": { + "description": "LivenessProbe configurations for ``metricsExporterSidecar``", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "Metrics Exporter liveness probe initial delay", + "type": "integer", + "default": 10 + }, + "periodSeconds": { + "description": "Metrics Exporter liveness probe frequency", + "type": "integer", + "default": 10 + }, + "timeoutSeconds": { + "description": "Metrics Exporter liveness probe command timeout", + "type": "integer", + "default": 1 + } + } + }, + "readinessProbe": { + "description": "ReadinessProbe configurations for ``metricsExporterSidecar``", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "Metrics Exporter readiness probe initial delay", + "type": "integer", + "default": 10 + }, + "periodSeconds": { + "description": "Metrics Exporter readiness probe frequency", + "type": "integer", + "default": 10 + }, + "timeoutSeconds": { + "description": "Metrics Exporter readiness probe command timeout", + "type": "integer", + "default": 1 + } + } + }, + "extraVolumeMounts": { + "description": "Mount additional volumes into PgBouncer Metrics Exporter.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + } + } + } + } + }, + "redis": { + "description": "Configuration for the Redis provisioned by the chart.", + "type": "object", + "x-docsSection": "Redis", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable the Redis provisioned by the chart (you can also use an external Redis instance with `data.brokerUrl` or `data.brokerUrlSecretName`).", + "type": "boolean", + "default": true + }, + "terminationGracePeriodSeconds": { + "description": "Grace period for Redis to exit after SIGTERM is sent from Kubernetes.", + "type": "integer", + "default": 600 + }, + "service": { + "description": "service configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "description": "Service type.", + "enum": [ + "ClusterIP", + "NodePort", + "LoadBalancer" + ], + "type": "string", + "default": "ClusterIP" + }, + "clusterIP": { + "description": "If using `ClusterIP` service type, custom IP address can be specified.", + "type": [ + "string", + "null" + ], + "default": null + }, + "nodePort": { + "description": "If using `NodePort` service type, custom node port can be specified.", + "type": [ + "integer", + "null" + ], + "default": null + } + } + }, + "persistence": { + "description": "Persistence configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable persistent volumes.", + "type": "boolean", + "default": true + }, + "size": { + "description": "Volume size for Redis StatefulSet.", + "type": "string", + "default": "1Gi" + }, + "storageClassName": { + "description": "If using a custom StorageClass, pass name ref to all StatefulSets here (templated).", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to redis volumes.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "existingClaim": { + "description": "The name of an existing PVC to use.", + "type": [ + "string", + "null" + ], + "default": null + }, + "persistentVolumeClaimRetentionPolicy": { + "$ref": "#/definitions/persistentVolumeClaimRetentionPolicy", + "description": "PersistentVolumeClaim retention policy to be used in the lifecycle of a StatefulSet." + } + } + }, + "emptyDirConfig": { + "description": "Configuration for redis empty dir volume.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.EmptyDirVolumeSource", + "default": null + }, + "resources": { + "description": "Resources for the Redis pods", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "passwordSecretName": { + "description": "Redis password secret.", + "type": [ + "string", + "null" + ], + "default": null + }, + "password": { + "description": "If password is set, create secret with it, else generate a new one on install (can only be set during install, not upgrade).", + "type": [ + "string", + "null" + ], + "default": null + }, + "passwordSecretAnnotations": { + "description": "Annotations to add to the redis password secret.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "safeToEvict": { + "description": "This setting tells Kubernetes that its ok to evict when it wants to scale a node down.", + "type": "boolean", + "default": true + }, + "nodeSelector": { + "description": "Select certain nodes for Redis pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "affinity": { + "description": "Specify scheduling constraints for Redis pods.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for Redis pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for Redis pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "priorityClassName": { + "description": "Specify priority for redis pods.", + "type": [ + "string", + "null" + ], + "default": null + }, + "serviceAccount": { + "description": "Create ServiceAccount.", + "type": "object", + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods", + "type": "boolean", + "default": true + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created.", + "type": "boolean", + "default": true + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the worker Kubernetes ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "securityContext": { + "description": "Security context for the cleanup job pod (deprecated, use `securityContexts` instead). If not set, the values from `securityContext` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the redis. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the redis.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition for the redis.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 999, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition for the redis.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "labels": { + "description": "Labels to add to the redis objects and pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "uid": { + "description": "Redis run as user parameter.", + "type": "integer", + "default": 0 + }, + "podAnnotations": { + "description": "Annotations to add to the redis pods (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "annotations": { + "description": "Annotations for the redis.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "registry": { + "description": "Auth secret for a private registry. This is used if pulling Airflow images from a private registry.", + "type": "object", + "x-docsSection": "Kubernetes", + "additionalProperties": false, + "properties": { + "secretName": { + "description": "Name of the Kubernetes secret containing Base64 encoded credentials to connect to a private registry (will get passed to imagePullSecrets) (Deprecated - renamed to `registry.secretNames`).", + "type": [ + "string", + "null" + ], + "default": null + }, + "connection": { + "description": "Credentials to connect to a private registry, these will get Base64 encoded and stored in a secret (will get passed to imagePullSecrets) (create manually the credentials secret and add to `imagePullSecrets` instead).", + "type": "object", + "default": {}, + "additionalProperties": false, + "properties": { + "user": { + "description": "Username", + "type": "string", + "default": "" + }, + "pass": { + "description": "Password", + "type": "string", + "default": "" + }, + "host": { + "description": "Registry Server URL (e.g. https://index.docker.io/v1/ for DockerHub)", + "type": "string", + "default": "" + }, + "email": { + "description": "Email Address", + "type": "string", + "default": "" + } + }, + "examples": [ + { + "user": "...", + "pass": "...", + "host": "...", + "email": "..." + } + ] + } + } + }, + "elasticsearch": { + "description": "Elasticsearch logging configuration.", + "type": "object", + "x-docsSection": "Airflow", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable Elasticsearch task logging.", + "type": "boolean", + "default": false + }, + "secretName": { + "description": "A secret containing the connection string.", + "type": [ + "string", + "null" + ], + "default": null + }, + "secretAnnotations": { + "description": "Extra annotations to apply to the elasticsearch secret.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "connection": { + "description": "Elasticsearch connection configuration.", + "type": "object", + "default": {}, + "additionalProperties": false, + "properties": { + "scheme": { + "description": "Scheme", + "type": "string", + "default": "http" + }, + "user": { + "description": "Username", + "type": "string", + "default": "" + }, + "pass": { + "description": "Password", + "type": "string", + "default": "" + }, + "host": { + "description": "Host", + "type": "string", + "default": "" + }, + "port": { + "description": "Port", + "type": "number", + "default": 80 + } + }, + "examples": [ + { + "scheme": "https", + "user": "...", + "pass": "...", + "host": "...", + "port": "..." + } + ] + } + } + }, + "opensearch": { + "description": "OpenSearch logging configuration.", + "type": "object", + "x-docsSection": "Airflow", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable OpenSearch task logging.", + "type": "boolean", + "default": false + }, + "secretName": { + "description": "A secret containing the connection string.", + "type": [ + "string", + "null" + ], + "default": null + }, + "connection": { + "description": "OpenSearch connection configuration.", + "type": "object", + "default": {}, + "additionalProperties": false, + "properties": { + "scheme": { + "description": "Scheme", + "type": "string", + "default": "http" + }, + "user": { + "description": "Username", + "type": "string", + "default": "" + }, + "pass": { + "description": "Password", + "type": "string", + "default": "" + }, + "host": { + "description": "Host", + "type": "string", + "default": "" + }, + "port": { + "description": "Port", + "type": "number", + "default": 80 + } + }, + "examples": [ + { + "scheme": "https", + "user": "...", + "pass": "...", + "host": "...", + "port": "..." + } + ] + } + } + }, + "ports": { + "description": "All ports used by chart.", + "type": "object", + "x-docsSection": "Ports", + "additionalProperties": false, + "properties": { + "flowerUI": { + "description": "Flower UI port.", + "type": "integer", + "default": 5555 + }, + "airflowUI": { + "description": "Airflow UI port.", + "type": "integer", + "default": 8080 + }, + "apiServer": { + "description": "API server port.", + "type": "integer", + "default": 8080 + }, + "workerLogs": { + "description": "Worker logs port.", + "type": "integer", + "default": 8793 + }, + "triggererLogs": { + "description": "Triggerer logs port.", + "type": "integer", + "default": 8794 + }, + "redisDB": { + "description": "Redis port.", + "type": "integer", + "default": 6379 + }, + "statsdIngest": { + "description": "StatsD ingest port.", + "type": "integer", + "default": 9125 + }, + "statsdScrape": { + "description": "StatsD scrape port.", + "type": "integer", + "default": 9102 + }, + "pgbouncer": { + "description": "PgBouncer port.", + "type": "integer", + "default": 6543 + }, + "pgbouncerScrape": { + "description": "PgBouncer scrape port.", + "type": "integer", + "default": 9127 + } + } + }, + "quotas": { + "description": "Define any ResourceQuotas for namespace.", + "x-docsSection": "Kubernetes", + "default": {}, + "additionalProperties": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + }, + "limits": { + "description": "Define default/max/min values for pods and containers in namespace.", + "type": "array", + "x-docsSection": "Kubernetes", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.LimitRangeItem" + } + }, + "cleanup": { + "description": "This runs as a CronJob to cleanup old pods.", + "type": "object", + "x-docsSection": "Jobs", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable cleanup.", + "type": "boolean", + "default": false + }, + "schedule": { + "description": "Cleanup schedule (templated).", + "type": "string", + "default": "*/15 * * * *" + }, + "command": { + "description": "Command to use when running the cleanup cronjob (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running the cleanup cronjob (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "bash", + "-c", + "exec airflow kubernetes cleanup-pods --namespace={{ .Release.Namespace }}" + ] + }, + "jobAnnotations": { + "description": "Annotations to add to the cleanup cronjob.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "nodeSelector": { + "description": "Select certain nodes for cleanup pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "affinity": { + "description": "Specify scheduling constraints for cleanup pods.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for cleanup pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for cleanup pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "priorityClassName": { + "description": "Specify priority for cleanup pods.", + "type": [ + "string", + "null" + ], + "default": null + }, + "podAnnotations": { + "description": "Annotations to add to cleanup pods (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "description": "labels to add to cleanup pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "resources": { + "description": "Resources for cleanup pods", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ] + }, + "serviceAccount": { + "description": "Create ServiceAccount.", + "type": "object", + "additionalProperties": false, + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods", + "type": "boolean", + "default": true + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created.", + "type": "boolean", + "default": true + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the cleanup CronJob Kubernetes ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "securityContext": { + "description": "Security context for the cleanup job pod (deprecated, use `securityContexts` instead). If not set, the values from `securityContext` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the cleanup. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the cleanup. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition for the cleanup.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition for the cleanup.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "env": { + "description": "Add additional env vars to cleanup.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] + } + }, + "required": [ + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } + ], + "additionalProperties": false + } + }, + "failedJobsHistoryLimit": { + "description": "The failed jobs history limit specifies the number of failed jobs to retain.", + "type": [ + "integer", + "null" + ], + "default": null, + "x-docsSection": "Kubernetes" + }, + "successfulJobsHistoryLimit": { + "description": "The successful jobs history limit specifies the number of finished jobs to retain.", + "type": [ + "integer", + "null" + ], + "default": null, + "x-docsSection": "Kubernetes" + } + } + }, + "databaseCleanup": { + "description": "This runs as a CronJob to cleanup the database after the retention period.", + "type": "object", + "x-docsSection": "Jobs", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable database cleanup.", + "type": "boolean", + "default": false + }, + "applyCustomEnv": { + "description": "Specify if you want additional configured env vars applied to database cleanup job", + "type": "boolean", + "default": true + }, + "schedule": { + "description": "Database cleanup schedule (templated).", + "type": "string", + "default": "0 0 * * 0" + }, + "retentionDays": { + "description": "Number of days to retain records in the metadata database.", + "type": "integer", + "default": 90, + "minimum": 1 + }, + "skipArchive": { + "description": "Don't preserve purged records in an archive table.", + "type": "boolean", + "default": false + }, + "tables": { + "description": "Table names to perform maintenance on. Supported values in: https://airflow.apache.org/docs/apache-airflow/stable/cli-and-env-variables-ref.html#clean", + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "default": [] + }, + "batchSize": { + "description": "Maximum number of rows to delete or archive in a single transaction.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "verbose": { + "description": "Make logging output more verbose.", + "type": "boolean", + "default": true + }, + "command": { + "description": "Command to use when running the database cleanup cronjob (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running the database cleanup cronjob (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "bash", + "-c", + "CLEAN_TS=$(date -d \"-{{ .Values.databaseCleanup.retentionDays }} days\" +\"%Y-%m-%dT%H:%M:%S\"); echo \"Cleaning up metadata DB entries older than ${CLEAN_TS}\"; exec airflow db clean --clean-before-timestamp \"${CLEAN_TS}\" --yes {{- if .Values.databaseCleanup.skipArchive }} --skip-archive{{ end }} {{- if .Values.databaseCleanup.verbose }} --verbose{{ end }} {{- with .Values.databaseCleanup.batchSize }} --batch-size {{ . }}{{ end }} {{- with .Values.databaseCleanup.tables }} --tables {{ . | join \",\" }}{{ end }}" + ] + }, + "jobAnnotations": { + "description": "Annotations to add to the database cleanup cronjob.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "nodeSelector": { + "description": "Select certain nodes for database cleanup pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "affinity": { + "description": "Specify scheduling constraints for database cleanup pods.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for database cleanup pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for database cleanup pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "priorityClassName": { + "description": "Specify priority for database cleanup pods.", + "type": [ + "string", + "null" + ], + "default": null + }, + "podAnnotations": { + "description": "Annotations to add to database cleanup pods (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "description": "labels to add to database cleanup pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "resources": { + "description": "Resources for database cleanup pods", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ] + }, + "serviceAccount": { + "description": "Create ServiceAccount.", + "type": "object", + "additionalProperties": false, + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods", + "type": "boolean", + "default": true + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created.", + "type": "boolean", + "default": true + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the database cleanup CronJob Kubernetes ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the database cleanup. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the database cleanup. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition for the database cleanup.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition for the database cleanup.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "env": { + "description": "Add additional env vars to database cleanup.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] + } + }, + "required": [ + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } + ], + "additionalProperties": false + } + }, + "failedJobsHistoryLimit": { + "description": "The failed jobs history limit specifies the number of failed jobs to retain.", + "type": [ + "integer", + "null" + ], + "default": 1, + "x-docsSection": "Kubernetes" + }, + "successfulJobsHistoryLimit": { + "description": "The successful jobs history limit specifies the number of finished jobs to retain.", + "type": [ + "integer", + "null" + ], + "default": 1, + "x-docsSection": "Kubernetes" + }, + "ttlSecondsAfterFinished": { + "description": "The number of seconds after a finished Job is eligible to be automatically deleted.", + "type": [ + "integer", + "null" + ], + "default": null, + "x-docsSection": "Kubernetes" + } + } + }, + "postgresql": { + "description": "Configuration for PostgreSQL subchart.", + "type": "object", + "x-docsSection": "Database", + "properties": { + "enabled": { + "description": "Enable PostgreSQL subchart.", + "type": "boolean", + "default": true + }, + "auth": { + "description": "PostgreSQL authentication values.", + "type": "object", + "additionalProperties": true, + "properties": { + "enablePostgresUser": { + "description": "Assign a password to the 'postgres' admin user. Otherwise, remote access will be blocked for this user", + "type": "boolean", + "default": true + }, + "postgresPassword": { + "description": "Password for the 'postgres' admin user.", + "type": [ + "string", + "null" + ], + "default": "postgres" + }, + "username": { + "description": "Name for a custom user to create", + "type": [ + "string", + "null" + ], + "default": "" + }, + "password": { + "description": "Password for the custom user to create.", + "type": [ + "string", + "null" + ], + "default": "" + } + } + }, + "image": { + "description": "PostgreSQL image configuration.", + "type": "object", + "additionalProperties": true, + "properties": { + "repository": { + "description": "The PostgreSQL image repository.", + "type": "string", + "default": "bitnamilegacy/postgresql" + }, + "tag": { + "description": "The PostgreSQL image tag.", + "type": "string", + "default": "16.1.0-debian-11-r15" + } + } + } + } + }, + "config": { + "description": "Settings to go into the mounted airflow.cfg", + "type": "object", + "x-docsSection": "Common", + "default": "See values.yaml", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": [ + "boolean", + "integer", + "number", + "string" + ] + } + } + }, + "multiNamespaceMode": { + "description": "Whether Airflow can launch workers and/or pods in multiple namespaces. If true, it creates ``ClusterRole``/``ClusterRolebinding`` (with access to entire cluster)", + "x-docsSection": "Airflow", + "type": "boolean", + "default": false + }, + "podTemplate": { + "description": "The content of ``pod_template_file.yaml`` used for KubernetesExecutor workers (templated). The default (see ``files/pod-template-file.kubernetes-helm-yaml``) already takes into account normal ``workers`` configuration parameters (e.g. ``workers.resources``), so you normally won't need to override this directly.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Airflow", + "default": null, + "examples": [ + "apiVersion: v1\nkind: Pod\nmetadata:\n name: placeholder-name\n labels:\n tier: airflow\n component: worker\n release: {{ .Release.Name }}\nspec:\n priorityClassName: high-priority\n containers:\n - name: base\n ..." + ] + }, + "dags": { + "description": "DAGs settings.", + "type": "object", + "x-docsSection": "Airflow", + "additionalProperties": false, + "properties": { + "mountPath": { + "description": "Where dags volume will be mounted. Works for both `persistence` and `gitSync`. If not specified, dags mount path will be set to `$AIRFLOW_HOME/dags`", + "type": [ + "string", + "null" + ], + "default": null + }, + "persistence": { + "description": "Persistence configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable persistent volume for storing dags.", + "type": "boolean", + "default": false + }, + "size": { + "description": "Volume size for dags.", + "type": "string", + "default": "1Gi" + }, + "storageClassName": { + "description": "If using a custom StorageClass, pass name here (templated).", + "type": [ + "string", + "null" + ], + "default": null + }, + "accessMode": { + "description": "Access mode of the persistent volume.", + "type": "string", + "enum": [ + "ReadWriteOnce", + "ReadOnlyMany", + "ReadWriteMany" + ], + "default": "ReadWriteOnce" + }, + "existingClaim": { + "description": "The name of an existing PVC to use.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations for the dag PVC", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "subPath": { + "description": "Subpath within the PVC where dags are located.", + "type": [ + "string", + "null" + ], + "default": null + } + } + }, + "gitSync": { + "description": "Git sync settings.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable Git sync.", + "type": "boolean", + "default": false + }, + "repo": { + "description": "Git repository.", + "type": "string", + "default": "https://github.com/apache/airflow.git" + }, + "branch": { + "description": "Git branch", + "type": "string", + "default": "v2-2-stable" + }, + "rev": { + "description": "Git revision.", + "type": "string", + "default": "HEAD" + }, + "ref": { + "description": "Git revision branch, tag, or hash.", + "type": "string", + "default": "v2-2-stable" + }, + "depth": { + "description": "Repository depth.", + "type": "integer", + "default": 1 + }, + "maxFailures": { + "description": "The number of consecutive failures allowed before aborting.", + "type": "integer", + "default": 0 + }, + "subPath": { + "description": "Subpath within the repo where dags are located.", + "type": "string", + "default": "tests/dags" + }, + "wait": { + "description": "Interval between git sync attempts in seconds. High values are more likely to cause DAGs to become out of sync between different components. Low values cause more traffic to the remote git repository.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "period": { + "description": "Interval between git sync attempts in Go-style duration string. High values are more likely to cause DAGs to become out of sync between different components. Low values cause more traffic to the remote git repository.", + "type": "string", + "default": "5s" + }, + "containerName": { + "description": "Git sync container name.", + "type": "string", + "default": "git-sync" + }, + "securityContext": { + "description": "Security context for the `gitSync` container (deprecated, use `securityContexts` instead). If not set, the values from `securityContext` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0 + } + ] + }, + "httpPort": { + "description": "Git-Sync liveness service http bind port.", + "type": "integer", + "default": 1234 + }, + "recommendedProbeSetting": { + "description": "Setting this to true, will remove readiness probe usage and configure liveness probe to use a dedicated Git-Sync liveness service.", + "type": "boolean", + "default": false + }, + "startupProbe": { + "description": "Startup probe configuration for git sync container.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable GitSync Kubernetes Startup Probe.", + "type": "boolean", + "default": true + }, + "timeoutSeconds": { + "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds.", + "type": "integer", + "default": 1 + }, + "initialDelaySeconds": { + "description": "Number of seconds after the container has started before startup probe is initiated.", + "type": "integer", + "default": 0 + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe. Minimum value is 1.", + "type": "integer", + "default": 5 + }, + "failureThreshold": { + "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1.", + "type": "integer", + "default": 10 + } + } + }, + "livenessProbe": { + "description": "Liveness probe configuration for git sync container.", + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "description": "Enable GitSync Kubernetes Liveness Probe.", + "type": "boolean", + "default": true + }, + "timeoutSeconds": { + "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds.", + "type": "integer", + "default": 1 + }, + "initialDelaySeconds": { + "description": "Number of seconds after the container has started before startup probe is initiated.", + "type": "integer", + "default": 0 + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe. Minimum value is 1.", + "type": "integer", + "default": 5 + }, + "failureThreshold": { + "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1.", + "type": "integer", + "default": 10 + } + } + }, + "readinessProbe": { + "description": "Readiness probe configuration for git sync container. As Git-Sync is not service-type object, the usage of the section was removed. Section itself will be removed in future release as to not break setups during upgrades.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Probe" + }, + "emptyDirConfig": { + "description": "Configuration for dags empty dir volume.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.EmptyDirVolumeSource", + "default": null + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the git sync sidecar. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the git sync sidecar. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the git sync sidecar.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "uid": { + "description": "Git sync container run as user parameter.", + "type": "integer", + "default": 65533 + }, + "extraVolumeMounts": { + "description": "Mount additional volumes into git sync container.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "credentialsSecret": { + "description": "Name of a Secret containing the repo `GIT_SYNC_USERNAME` and `GIT_SYNC_PASSWORD`.", + "type": [ + "string", + "null" + ], + "default": null + }, + "sshKey": { + "description": "SSH private key", + "type": [ + "string", + "null" + ], + "default": null + }, + "sshKeySecret": { + "description": "Name of a Secret containing the repo `sshKeySecret`.", + "type": [ + "string", + "null" + ], + "default": null + }, + "knownHosts": { + "description": "When using a ssh private key, the contents of your `known_hosts` file.", + "type": [ + "string", + "null" + ], + "default": null, + "examples": [ + ", \n, ", + ", " + ] + }, + "env": { + "description": "Environment variables for git sync container.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar" + }, + "examples": [ + { + "name": "GIT_SYNC_TIMEOUT", + "value": "60" + }, + { + "name": "GIT_SYNC_USERNAME", + "valueFrom": { + "secretKeyRef": { + "name": "git-secret", + "key": "username" + } + } + } + ] + }, + "envFrom": { + "description": "Extra envFrom 'items' that will be added to the definition of Airflow gitSync containers; a string or array are expected (templated).", + "type": [ + "null", + "string" + ], + "default": null, + "examples": [ + "- secretRef:\n name: 'proxy-config", + "- configMapRef:\n name: 'proxy-config" + ] + }, + "resources": { + "description": "Resources on workers git-sync sidecar", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + } + } + } + } + }, + "logs": { + "description": "Logs settings.", + "type": "object", + "x-docsSection": "Airflow", + "additionalProperties": false, + "properties": { + "persistence": { + "description": "Persistence configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable persistent volume for storing logs.", + "type": "boolean", + "default": false + }, + "size": { + "description": "Volume size for logs.", + "type": "string", + "default": "100Gi" + }, + "storageClassName": { + "description": "If using a custom StorageClass, pass name here (templated).", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to logs PVC", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "existingClaim": { + "description": "The name of an existing PVC to use.", + "type": [ + "string", + "null" + ], + "default": null + }, + "subPath": { + "description": "The subpath of the existing PVC to use.", + "type": [ + "string", + "null" + ], + "default": null + } + } + }, + "emptyDirConfig": { + "description": "Configuration for logs empty dir volume.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.EmptyDirVolumeSource", + "default": null + } + } + } + }, + "definitions": { + "io.k8s.api.apps.v1.DeploymentStrategy": { + "description": "DeploymentStrategy describes how to replace existing pods with new ones.", + "properties": { + "rollingUpdate": { + "$ref": "#/definitions/io.k8s.api.apps.v1.RollingUpdateDeployment", + "description": "Rolling update config params. Present only if DeploymentStrategyType = RollingUpdate." + }, + "type": { + "description": "Type of deployment. Can be \"Recreate\" or \"RollingUpdate\". Default is RollingUpdate.", + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.apps.v1.RollingUpdateDeployment": { + "description": "Spec to control the desired behavior of rolling update.", + "properties": { + "maxSurge": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.util.intstr.IntOrString", + "description": "The maximum number of pods that can be scheduled above the desired number of pods. Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). This can not be 0 if MaxUnavailable is 0. Absolute number is calculated from percentage by rounding up. Defaults to 25%. Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when the rolling update starts, such that the total number of old and new pods do not exceed 130% of desired pods. Once old pods have been killed, new ReplicaSet can be scaled up further, ensuring that total number of pods running at any time during the update is at most 130% of desired pods." + }, + "maxUnavailable": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.util.intstr.IntOrString", + "description": "The maximum number of pods that can be unavailable during the update. Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). Absolute number is calculated from percentage by rounding down. This can not be 0 if MaxSurge is 0. Defaults to 25%. Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods immediately when the rolling update starts. Once new pods are ready, old ReplicaSet can be scaled down further, followed by scaling up the new ReplicaSet, ensuring that the total number of pods available at all times during the update is at least 70% of desired pods." + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.autoscaling.v2.ContainerResourceMetricSource": { + "description": "ContainerResourceMetricSource indicates how to scale on a resource metric known to Kubernetes, as specified in requests and limits, describing each pod in the current scale target (e.g. CPU or memory). The values will be averaged together before being compared to the target. Such metrics are built in to Kubernetes, and have special scaling options on top of those available to normal per-pod metrics using the \"pods\" source. Only one \"target\" type should be set.", + "properties": { + "container": { + "description": "container is the name of the container in the pods of the scaling target", + "type": "string" + }, + "name": { + "description": "name is the name of the resource in question.", + "type": "string" + }, + "target": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricTarget", + "description": "target specifies the target value for the given metric" + } + }, + "required": [ + "name", + "target", + "container" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.autoscaling.v2.CrossVersionObjectReference": { + "description": "CrossVersionObjectReference contains enough information to let you identify the referred resource.", + "properties": { + "apiVersion": { + "description": "apiVersion is the API version of the referent", + "type": "string" + }, + "kind": { + "description": "kind is the kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + "type": "string" + }, + "name": { + "description": "name is the name of the referent; More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + } + }, + "required": [ + "kind", + "name" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.autoscaling.v2.ExternalMetricSource": { + "description": "ExternalMetricSource indicates how to scale on a metric not associated with any Kubernetes object (for example length of queue in cloud messaging service, or QPS from loadbalancer running outside of cluster).", + "properties": { + "metric": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricIdentifier", + "description": "metric identifies the target metric by name and selector" + }, + "target": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricTarget", + "description": "target specifies the target value for the given metric" + } + }, + "required": [ + "metric", + "target" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.autoscaling.v2.HPAScalingPolicy": { + "description": "HPAScalingPolicy is a single policy which must hold true for a specified past interval.", + "properties": { + "periodSeconds": { + "description": "periodSeconds specifies the window of time for which the policy should hold true. PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min).", + "format": "int32", + "type": "integer" + }, + "type": { + "description": "type is used to specify the scaling policy.", + "type": "string" + }, + "value": { + "description": "value contains the amount of change which is permitted by the policy. It must be greater than zero", + "format": "int32", + "type": "integer" + } + }, + "required": [ + "type", + "value", + "periodSeconds" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.autoscaling.v2.HPAScalingRules": { + "description": "HPAScalingRules configures the scaling behavior for one direction. These Rules are applied after calculating DesiredReplicas from metrics for the HPA. They can limit the scaling velocity by specifying scaling policies. They can prevent flapping by specifying the stabilization window, so that the number of replicas is not set instantly, instead, the safest value from the stabilization window is chosen.", + "properties": { + "policies": { + "description": "policies is a list of potential scaling polices which can be used during scaling. At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid", + "items": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HPAScalingPolicy" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "selectPolicy": { + "description": "selectPolicy is used to specify which policy should be used. If not set, the default value Max is used.", + "type": "string" + }, + "stabilizationWindowSeconds": { + "description": "stabilizationWindowSeconds is the number of seconds for which past recommendations should be considered while scaling up or scaling down. StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). If not set, use the default values: - For scale up: 0 (i.e. no stabilization is done). - For scale down: 300 (i.e. the stabilization window is 300 seconds long).", + "format": "int32", + "type": "integer" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.autoscaling.v2.HorizontalPodAutoscalerBehavior": { + "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target in both Up and Down directions (scaleUp and scaleDown fields respectively).", + "properties": { + "scaleDown": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HPAScalingRules", + "description": "scaleDown is scaling policy for scaling Down. If not set, the default value is to allow to scale down to minReplicas pods, with a 300 second stabilization window (i.e., the highest recommendation for the last 300sec is used)." + }, + "scaleUp": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HPAScalingRules", + "description": "scaleUp is scaling policy for scaling Up. If not set, the default value is the higher of:\n * increase no more than 4 pods per 60 seconds\n * double the number of pods per 60 seconds\nNo stabilization is used." + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.autoscaling.v2.MetricIdentifier": { + "description": "MetricIdentifier defines the name and optionally selector for a metric", + "properties": { + "name": { + "description": "name is the name of the given metric", + "type": "string" + }, + "selector": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector", + "description": "selector is the string-encoded form of a standard kubernetes label selector for the given metric When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. When unset, just the metricName will be used to gather metrics." + } + }, + "required": [ + "name" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.autoscaling.v2.MetricSpec": { + "description": "MetricSpec specifies how to scale based on a single metric (only `type` and one other matching field should be set at once).", + "properties": { + "containerResource": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.ContainerResourceMetricSource", + "description": "containerResource refers to a resource metric (such as those specified in requests and limits) known to Kubernetes describing a single container in each pod of the current scale target (e.g. CPU or memory). Such metrics are built in to Kubernetes, and have special scaling options on top of those available to normal per-pod metrics using the \"pods\" source. This is an alpha feature and can be enabled by the HPAContainerMetrics feature flag." + }, + "external": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.ExternalMetricSource", + "description": "external refers to a global metric that is not associated with any Kubernetes object. It allows autoscaling based on information coming from components running outside of cluster (for example length of queue in cloud messaging service, or QPS from loadbalancer running outside of cluster)." + }, + "object": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.ObjectMetricSource", + "description": "object refers to a metric describing a single kubernetes object (for example, hits-per-second on an Ingress object)." + }, + "pods": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.PodsMetricSource", + "description": "pods refers to a metric describing each pod in the current scale target (for example, transactions-processed-per-second). The values will be averaged together before being compared to the target value." + }, + "resource": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.ResourceMetricSource", + "description": "resource refers to a resource metric (such as those specified in requests and limits) known to Kubernetes describing each pod in the current scale target (e.g. CPU or memory). Such metrics are built in to Kubernetes, and have special scaling options on top of those available to normal per-pod metrics using the \"pods\" source." + }, + "type": { + "description": "type is the type of metric source. It should be one of \"ContainerResource\", \"External\", \"Object\", \"Pods\" or \"Resource\", each mapping to a matching field in the object. Note: \"ContainerResource\" type is available on when the feature-gate HPAContainerMetrics is enabled", + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.autoscaling.v2.MetricTarget": { + "description": "MetricTarget defines the target value, average value, or average utilization of a specific metric", + "properties": { + "averageUtilization": { + "description": "averageUtilization is the target value of the average of the resource metric across all relevant pods, represented as a percentage of the requested value of the resource for the pods. Currently only valid for Resource metric source type", + "format": "int32", + "type": "integer" + }, + "averageValue": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity", + "description": "averageValue is the target value of the average of the metric across all relevant pods (as a quantity)" + }, + "type": { + "description": "type represents whether the metric type is Utilization, Value, or AverageValue", + "type": "string" + }, + "value": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity", + "description": "value is the target value of the metric (as a quantity)." + } + }, + "required": [ + "type" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.autoscaling.v2.ObjectMetricSource": { + "description": "ObjectMetricSource indicates how to scale on a metric describing a kubernetes object (for example, hits-per-second on an Ingress object).", + "properties": { + "describedObject": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.CrossVersionObjectReference", + "description": "describedObject specifies the descriptions of a object,such as kind,name apiVersion" + }, + "metric": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricIdentifier", + "description": "metric identifies the target metric by name and selector" + }, + "target": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricTarget", + "description": "target specifies the target value for the given metric" + } + }, + "required": [ + "describedObject", + "target", + "metric" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.autoscaling.v2.PodsMetricSource": { + "description": "PodsMetricSource indicates how to scale on a metric describing each pod in the current scale target (for example, transactions-processed-per-second). The values will be averaged together before being compared to the target value.", + "properties": { + "metric": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricIdentifier", + "description": "metric identifies the target metric by name and selector" + }, + "target": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricTarget", + "description": "target specifies the target value for the given metric" + } + }, + "required": [ + "metric", + "target" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.autoscaling.v2.ResourceMetricSource": { + "description": "ResourceMetricSource indicates how to scale on a resource metric known to Kubernetes, as specified in requests and limits, describing each pod in the current scale target (e.g. CPU or memory). The values will be averaged together before being compared to the target. Such metrics are built in to Kubernetes, and have special scaling options on top of those available to normal per-pod metrics using the \"pods\" source. Only one \"target\" type should be set.", + "properties": { + "name": { + "description": "name is the name of the resource in question.", + "type": "string" + }, + "target": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricTarget", + "description": "target specifies the target value for the given metric" + } + }, + "required": [ + "name", + "target" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.AWSElasticBlockStoreVolumeSource": { + "description": "Represents a Persistent Disk resource in AWS.\n\nAn AWS EBS disk must exist before mounting to a container. The disk must also be in the same AWS zone as the kubelet. An AWS EBS disk can only be mounted as read/write once. AWS EBS volumes support ownership management and SELinux relabeling.", + "properties": { + "fsType": { + "description": "fsType is the filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore", + "type": "string" + }, + "partition": { + "description": "partition is the partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as \"1\". Similarly, the volume partition for /dev/sda is \"0\" (or you can leave the property empty).", + "format": "int32", + "type": "integer" + }, + "readOnly": { + "description": "readOnly value true will force the readOnly setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore", + "type": "boolean" + }, + "volumeID": { + "description": "volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore", + "type": "string" + } + }, + "required": [ + "volumeID" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.Affinity": { + "description": "Affinity is a group of affinity scheduling rules.", + "properties": { + "nodeAffinity": { + "$ref": "#/definitions/io.k8s.api.core.v1.NodeAffinity", + "description": "Describes node affinity scheduling rules for the pod." + }, + "podAffinity": { + "$ref": "#/definitions/io.k8s.api.core.v1.PodAffinity", + "description": "Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s))." + }, + "podAntiAffinity": { + "$ref": "#/definitions/io.k8s.api.core.v1.PodAntiAffinity", + "description": "Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s))." + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.AppArmorProfile": { + "description": "AppArmorProfile defines a pod or container's AppArmor settings.", + "properties": { + "localhostProfile": { + "description": "localhostProfile indicates a profile loaded on the node that should be used. The profile must be preconfigured on the node to work. Must match the loaded name of the profile. Must be set if and only if type is \"Localhost\".", + "type": "string" + }, + "type": { + "description": "type indicates which kind of AppArmor profile will be applied. Valid options are:\n Localhost - a profile pre-loaded on the node.\n RuntimeDefault - the container runtime's default profile.\n Unconfined - no AppArmor enforcement.", + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object", + "x-kubernetes-unions": [ + { + "discriminator": "type", + "fields-to-discriminateBy": { + "localhostProfile": "LocalhostProfile" + } + } + ], + "additionalProperties": false + }, + "io.k8s.api.core.v1.AzureDiskVolumeSource": { + "description": "AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod.", + "properties": { + "cachingMode": { + "description": "cachingMode is the Host Caching mode: None, Read Only, Read Write.", + "type": "string" + }, + "diskName": { + "description": "diskName is the Name of the data disk in the blob storage", + "type": "string" + }, + "diskURI": { + "description": "diskURI is the URI of data disk in the blob storage", + "type": "string" + }, + "fsType": { + "description": "fsType is Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.", + "type": "string" + }, + "kind": { + "description": "kind expected values are Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared", + "type": "string" + }, + "readOnly": { + "description": "readOnly Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", + "type": "boolean" + } + }, + "required": [ + "diskName", + "diskURI" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.AzureFileVolumeSource": { + "description": "AzureFile represents an Azure File Service mount on the host and bind mount to the pod.", + "properties": { + "readOnly": { + "description": "readOnly defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", + "type": "boolean" + }, + "secretName": { + "description": "secretName is the name of secret that contains Azure Storage Account Name and Key", + "type": "string" + }, + "shareName": { + "description": "shareName is the azure share Name", + "type": "string" + } + }, + "required": [ + "secretName", + "shareName" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.CSIVolumeSource": { + "description": "Represents a source location of a volume to mount, managed by an external CSI driver", + "properties": { + "driver": { + "description": "driver is the name of the CSI driver that handles this volume. Consult with your admin for the correct name as registered in the cluster.", + "type": "string" + }, + "fsType": { + "description": "fsType to mount. Ex. \"ext4\", \"xfs\", \"ntfs\". If not provided, the empty value is passed to the associated CSI driver which will determine the default filesystem to apply.", + "type": "string" + }, + "nodePublishSecretRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.LocalObjectReference", + "description": "nodePublishSecretRef is a reference to the secret object containing sensitive information to pass to the CSI driver to complete the CSI NodePublishVolume and NodeUnpublishVolume calls. This field is optional, and may be empty if no secret is required. If the secret object contains more than one secret, all secret references are passed." + }, + "readOnly": { + "description": "readOnly specifies a read-only configuration for the volume. Defaults to false (read/write).", + "type": "boolean" + }, + "volumeAttributes": { + "additionalProperties": { + "type": "string" + }, + "description": "volumeAttributes stores driver-specific properties that are passed to the CSI driver. Consult your driver's documentation for supported values.", + "type": "object" + } + }, + "required": [ + "driver" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.Capabilities": { + "description": "Adds and removes POSIX capabilities from running containers.", + "properties": { + "add": { + "description": "Added capabilities", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "drop": { + "description": "Removed capabilities", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.CephFSVolumeSource": { + "description": "Represents a Ceph Filesystem mount that lasts the lifetime of a pod Cephfs volumes do not support ownership management or SELinux relabeling.", + "properties": { + "monitors": { + "description": "monitors is Required: Monitors is a collection of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "path": { + "description": "path is Optional: Used as the mounted root, rather than the full Ceph tree, default is /", + "type": "string" + }, + "readOnly": { + "description": "readOnly is Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it", + "type": "boolean" + }, + "secretFile": { + "description": "secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it", + "type": "string" + }, + "secretRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.LocalObjectReference", + "description": "secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it" + }, + "user": { + "description": "user is optional: User is the rados user name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it", + "type": "string" + } + }, + "required": [ + "monitors" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.CinderVolumeSource": { + "description": "Represents a cinder volume resource in Openstack. A Cinder volume must exist before mounting to a container. The volume must also be in the same region as the kubelet. Cinder volumes support ownership management and SELinux relabeling.", + "properties": { + "fsType": { + "description": "fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md", + "type": "string" + }, + "readOnly": { + "description": "readOnly defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md", + "type": "boolean" + }, + "secretRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.LocalObjectReference", + "description": "secretRef is optional: points to a secret object containing parameters used to connect to OpenStack." + }, + "volumeID": { + "description": "volumeID used to identify the volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md", + "type": "string" + } + }, + "required": [ + "volumeID" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ClusterTrustBundleProjection": { + "description": "ClusterTrustBundleProjection describes how to select a set of ClusterTrustBundle objects and project their contents into the pod filesystem.", + "properties": { + "labelSelector": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector", + "description": "Select all ClusterTrustBundles that match this label selector. Only has effect if signerName is set. Mutually-exclusive with name. If unset, interpreted as \"match nothing\". If set but empty, interpreted as \"match everything\"." + }, + "name": { + "description": "Select a single ClusterTrustBundle by object name. Mutually-exclusive with signerName and labelSelector.", + "type": "string" + }, + "optional": { + "description": "If true, don't block pod startup if the referenced ClusterTrustBundle(s) aren't available. If using name, then the named ClusterTrustBundle is allowed not to exist. If using signerName, then the combination of signerName and labelSelector is allowed to match zero ClusterTrustBundles.", + "type": "boolean" + }, + "path": { + "description": "Relative path from the volume root to write the bundle.", + "type": "string" + }, + "signerName": { + "description": "Select all ClusterTrustBundles that match this signer name. Mutually-exclusive with name. The contents of all selected ClusterTrustBundles will be unified and deduplicated.", + "type": "string" + } + }, + "required": [ + "path" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ConfigMapEnvSource": { + "description": "ConfigMapEnvSource selects a ConfigMap to populate the environment variables with.\n\nThe contents of the target ConfigMap's Data field will represent the key-value pairs as environment variables.", + "properties": { + "name": { + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + }, + "optional": { + "description": "Specify whether the ConfigMap must be defined", + "type": "boolean" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ConfigMapKeySelector": { + "description": "Selects a key from a ConfigMap.", + "properties": { + "key": { + "description": "The key to select.", + "type": "string" + }, + "name": { + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + }, + "optional": { + "description": "Specify whether the ConfigMap or its key must be defined", + "type": "boolean" + } + }, + "required": [ + "key" + ], + "type": "object", + "x-kubernetes-map-type": "atomic", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ConfigMapProjection": { + "description": "Adapts a ConfigMap into a projected volume.\n\nThe contents of the target ConfigMap's Data field will be presented in a projected volume as files using the keys in the Data field as the file names, unless the items element is populated with specific mappings of keys to paths. Note that this is identical to a configmap volume source without the default mode.", + "properties": { + "items": { + "description": "items if unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.KeyToPath" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "name": { + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + }, + "optional": { + "description": "optional specify whether the ConfigMap or its keys must be defined", + "type": "boolean" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ConfigMapVolumeSource": { + "description": "Adapts a ConfigMap into a volume.\n\nThe contents of the target ConfigMap's Data field will be presented in a volume as files using the keys in the Data field as the file names, unless the items element is populated with specific mappings of keys to paths. ConfigMap volumes support ownership management and SELinux relabeling.", + "properties": { + "defaultMode": { + "description": "defaultMode is optional: mode bits used to set permissions on created files by default. Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", + "format": "int32", + "type": "integer" + }, + "items": { + "description": "items if unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.KeyToPath" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "name": { + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + }, + "optional": { + "description": "optional specify whether the ConfigMap or its keys must be defined", + "type": "boolean" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.Container": { + "description": "A single application container that you want to run within a pod.", + "properties": { + "args": { + "description": "Arguments to the entrypoint. The container image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "command": { + "description": "Entrypoint array. Not executed within a shell. The container image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "env": { + "description": "List of environment variables to set in the container. Cannot be updated.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar" + }, + "type": "array", + "x-kubernetes-list-map-keys": [ + "name" + ], + "x-kubernetes-list-type": "map", + "x-kubernetes-patch-merge-key": "name", + "x-kubernetes-patch-strategy": "merge" + }, + "envFrom": { + "description": "List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.EnvFromSource" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "image": { + "description": "Container image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.", + "type": "string" + }, + "imagePullPolicy": { + "description": "Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images", + "type": "string" + }, + "lifecycle": { + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "description": "Actions that the management system should take in response to container lifecycle events. Cannot be updated." + }, + "livenessProbe": { + "$ref": "#/definitions/io.k8s.api.core.v1.Probe", + "description": "Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes" + }, + "name": { + "description": "Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.", + "type": "string" + }, + "ports": { + "description": "List of ports to expose from the container. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \"0.0.0.0\" address inside a container will be accessible from the network. Modifying this array with strategic merge patch may corrupt the data. For more information See https://github.com/kubernetes/kubernetes/issues/108255. Cannot be updated.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.ContainerPort" + }, + "type": "array", + "x-kubernetes-list-map-keys": [ + "containerPort", + "protocol" + ], + "x-kubernetes-list-type": "map", + "x-kubernetes-patch-merge-key": "containerPort", + "x-kubernetes-patch-strategy": "merge" + }, + "readinessProbe": { + "$ref": "#/definitions/io.k8s.api.core.v1.Probe", + "description": "Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes" + }, + "resizePolicy": { + "description": "Resources resize policy for the container.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.ContainerResizePolicy" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "resources": { + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements", + "description": "Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/" + }, + "restartPolicy": { + "description": "RestartPolicy defines the restart behavior of individual containers in a pod. This field may only be set for init containers, and the only allowed value is \"Always\". For non-init containers or when this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. Setting the RestartPolicy as \"Always\" for the init container will have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy \"Always\" will be shut down. This lifecycle differs from normal init containers and is often referred to as a \"sidecar\" container. Although this init container still starts in the init container sequence, it does not wait for the container to complete before proceeding to the next init container. Instead, the next init container starts immediately after this init container is started, or after any startupProbe has successfully completed.", + "type": "string" + }, + "securityContext": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "description": "SecurityContext defines the security options the container should be run with. If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/" + }, + "startupProbe": { + "$ref": "#/definitions/io.k8s.api.core.v1.Probe", + "description": "StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes" + }, + "stdin": { + "description": "Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.", + "type": "boolean" + }, + "stdinOnce": { + "description": "Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false", + "type": "boolean" + }, + "terminationMessagePath": { + "description": "Optional: Path at which the file to which the container's termination message will be written is mounted into the container's filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.", + "type": "string" + }, + "terminationMessagePolicy": { + "description": "Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.", + "type": "string" + }, + "tty": { + "description": "Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.", + "type": "boolean" + }, + "volumeDevices": { + "description": "volumeDevices is the list of block devices to be used by the container.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeDevice" + }, + "type": "array", + "x-kubernetes-list-map-keys": [ + "devicePath" + ], + "x-kubernetes-list-type": "map", + "x-kubernetes-patch-merge-key": "devicePath", + "x-kubernetes-patch-strategy": "merge" + }, + "volumeMounts": { + "description": "Pod volumes to mount into the container's filesystem. Cannot be updated.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + }, + "type": "array", + "x-kubernetes-list-map-keys": [ + "mountPath" + ], + "x-kubernetes-list-type": "map", + "x-kubernetes-patch-merge-key": "mountPath", + "x-kubernetes-patch-strategy": "merge" + }, + "workingDir": { + "description": "Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.", + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ContainerPort": { + "description": "ContainerPort represents a network port in a single container.", + "properties": { + "containerPort": { + "description": "Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536.", + "format": "int32", + "type": "integer" + }, + "hostIP": { + "description": "What host IP to bind the external port to.", + "type": "string" + }, + "hostPort": { + "description": "Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.", + "format": "int32", + "type": "integer" + }, + "name": { + "description": "If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.", + "type": "string" + }, + "protocol": { + "description": "Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \"TCP\".", + "type": "string" + } + }, + "required": [ + "containerPort" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ContainerResizePolicy": { + "description": "ContainerResizePolicy represents resource resize policy for the container.", + "properties": { + "resourceName": { + "description": "Name of the resource to which this resource resize policy applies. Supported values: cpu, memory.", + "type": "string" + }, + "restartPolicy": { + "description": "Restart policy to apply when specified resource is resized. If not specified, it defaults to NotRequired.", + "type": "string" + } + }, + "required": [ + "resourceName", + "restartPolicy" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.DownwardAPIProjection": { + "description": "Represents downward API info for projecting into a projected volume. Note that this is identical to a downwardAPI volume source without the default mode.", + "properties": { + "items": { + "description": "Items is a list of DownwardAPIVolume file", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.DownwardAPIVolumeFile" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.DownwardAPIVolumeFile": { + "description": "DownwardAPIVolumeFile represents information to create the file containing the pod field", + "properties": { + "fieldRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ObjectFieldSelector", + "description": "Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported." + }, + "mode": { + "description": "Optional: mode bits used to set permissions on this file, must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", + "format": "int32", + "type": "integer" + }, + "path": { + "description": "Required: Path is the relative path name of the file to be created. Must not be absolute or contain the '..' path. Must be utf-8 encoded. The first item of the relative path must not start with '..'", + "type": "string" + }, + "resourceFieldRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceFieldSelector", + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported." + } + }, + "required": [ + "path" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.DownwardAPIVolumeSource": { + "description": "DownwardAPIVolumeSource represents a volume containing downward API info. Downward API volumes support ownership management and SELinux relabeling.", + "properties": { + "defaultMode": { + "description": "Optional: mode bits to use on created files by default. Must be a Optional: mode bits used to set permissions on created files by default. Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", + "format": "int32", + "type": "integer" + }, + "items": { + "description": "Items is a list of downward API volume file", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.DownwardAPIVolumeFile" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.EmptyDirVolumeSource": { + "description": "Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.", + "properties": { + "medium": { + "description": "medium represents what type of storage medium should back this directory. The default is \"\" which means to use the node's default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir", + "type": "string" + }, + "sizeLimit": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity", + "description": "sizeLimit is the total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.EnvFromSource": { + "description": "EnvFromSource represents the source of a set of ConfigMaps", + "properties": { + "configMapRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapEnvSource", + "description": "The ConfigMap to select from" + }, + "prefix": { + "description": "An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.", + "type": "string" + }, + "secretRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretEnvSource", + "description": "The Secret to select from" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.EnvVar": { + "description": "EnvVar represents an environment variable present in a Container.", + "properties": { + "name": { + "description": "Name of the environment variable. Must be a C_IDENTIFIER.", + "type": "string" + }, + "value": { + "description": "Variable references $(VAR_NAME) are expanded using the previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".", + "type": "string" + }, + "valueFrom": { + "$ref": "#/definitions/io.k8s.api.core.v1.EnvVarSource", + "description": "Source for the environment variable's value. Cannot be used if value is not empty." + } + }, + "required": [ + "name" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.EnvVarSource": { + "description": "EnvVarSource represents a source for the value of an EnvVar.", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "fieldRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ObjectFieldSelector", + "description": "Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs." + }, + "resourceFieldRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceFieldSelector", + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.EphemeralVolumeSource": { + "description": "Represents an ephemeral volume that is handled by a normal storage driver.", + "properties": { + "volumeClaimTemplate": { + "$ref": "#/definitions/io.k8s.api.core.v1.PersistentVolumeClaimTemplate", + "description": "Will be used to create a stand-alone PVC to provision the volume. The pod in which this EphemeralVolumeSource is embedded will be the owner of the PVC, i.e. the PVC will be deleted together with the pod. The name of the PVC will be `-` where `` is the name from the `PodSpec.Volumes` array entry. Pod validation will reject the pod if the concatenated name is not valid for a PVC (for example, too long).\n\nAn existing PVC with that name that is not owned by the pod will *not* be used for the pod to avoid using an unrelated volume by mistake. Starting the pod is then blocked until the unrelated PVC is removed. If such a pre-created PVC is meant to be used by the pod, the PVC has to updated with an owner reference to the pod once the pod exists. Normally this should not be necessary, but it may be useful when manually reconstructing a broken cluster.\n\nThis field is read-only and no changes will be made by Kubernetes to the PVC after it has been created.\n\nRequired, must not be nil." + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ExecAction": { + "description": "ExecAction describes a \"run in container\" action.", + "properties": { + "command": { + "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.FCVolumeSource": { + "description": "Represents a Fibre Channel volume. Fibre Channel volumes can only be mounted as read/write once. Fibre Channel volumes support ownership management and SELinux relabeling.", + "properties": { + "fsType": { + "description": "fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.", + "type": "string" + }, + "lun": { + "description": "lun is Optional: FC target lun number", + "format": "int32", + "type": "integer" + }, + "readOnly": { + "description": "readOnly is Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", + "type": "boolean" + }, + "targetWWNs": { + "description": "targetWWNs is Optional: FC target worldwide names (WWNs)", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "wwids": { + "description": "wwids Optional: FC volume world wide identifiers (wwids) Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously.", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.FlexVolumeSource": { + "description": "FlexVolume represents a generic volume resource that is provisioned/attached using an exec based plugin.", + "properties": { + "driver": { + "description": "driver is the name of the driver to use for this volume.", + "type": "string" + }, + "fsType": { + "description": "fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". The default filesystem depends on FlexVolume script.", + "type": "string" + }, + "options": { + "additionalProperties": { + "type": "string" + }, + "description": "options is Optional: this field holds extra command options if any.", + "type": "object" + }, + "readOnly": { + "description": "readOnly is Optional: defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", + "type": "boolean" + }, + "secretRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.LocalObjectReference", + "description": "secretRef is Optional: secretRef is reference to the secret object containing sensitive information to pass to the plugin scripts. This may be empty if no secret object is specified. If the secret object contains more than one secret, all secrets are passed to the plugin scripts." + } + }, + "required": [ + "driver" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.FlockerVolumeSource": { + "description": "Represents a Flocker volume mounted by the Flocker agent. One and only one of datasetName and datasetUUID should be set. Flocker volumes do not support ownership management or SELinux relabeling.", + "properties": { + "datasetName": { + "description": "datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker should be considered as deprecated", + "type": "string" + }, + "datasetUUID": { + "description": "datasetUUID is the UUID of the dataset. This is unique identifier of a Flocker dataset", + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.GCEPersistentDiskVolumeSource": { + "description": "Represents a Persistent Disk resource in Google Compute Engine.\n\nA GCE PD must exist before mounting to a container. The disk must also be in the same GCE project and zone as the kubelet. A GCE PD can only be mounted as read/write once or read-only many times. GCE PDs support ownership management and SELinux relabeling.", + "properties": { + "fsType": { + "description": "fsType is filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk", + "type": "string" + }, + "partition": { + "description": "partition is the partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as \"1\". Similarly, the volume partition for /dev/sda is \"0\" (or you can leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk", + "format": "int32", + "type": "integer" + }, + "pdName": { + "description": "pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk", + "type": "string" + }, + "readOnly": { + "description": "readOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk", + "type": "boolean" + } + }, + "required": [ + "pdName" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.GRPCAction": { + "properties": { + "port": { + "description": "Port number of the gRPC service. Number must be in the range 1 to 65535.", + "format": "int32", + "type": "integer" + }, + "service": { + "description": "Service is the name of the service to place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).\n\nIf this is not specified, the default behavior is defined by gRPC.", + "type": "string" + } + }, + "required": [ + "port" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.GitRepoVolumeSource": { + "description": "Represents a volume that is populated with the contents of a git repository. Git repo volumes do not support ownership management. Git repo volumes support SELinux relabeling.\n\nDEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir into the Pod's container.", + "properties": { + "directory": { + "description": "directory is the target directory name. Must not contain or start with '..'. If '.' is supplied, the volume directory will be the git repository. Otherwise, if specified, the volume will contain the git repository in the subdirectory with the given name.", + "type": "string" + }, + "repository": { + "description": "repository is the URL", + "type": "string" + }, + "revision": { + "description": "revision is the commit hash for the specified revision.", + "type": "string" + } + }, + "required": [ + "repository" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.GlusterfsVolumeSource": { + "description": "Represents a Glusterfs mount that lasts the lifetime of a pod. Glusterfs volumes do not support ownership management or SELinux relabeling.", + "properties": { + "endpoints": { + "description": "endpoints is the endpoint name that details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod", + "type": "string" + }, + "path": { + "description": "path is the Glusterfs volume path. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod", + "type": "string" + }, + "readOnly": { + "description": "readOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod", + "type": "boolean" + } + }, + "required": [ + "endpoints", + "path" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.HTTPGetAction": { + "description": "HTTPGetAction describes an action based on HTTP Get requests.", + "properties": { + "host": { + "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", + "type": "string" + }, + "httpHeaders": { + "description": "Custom headers to set in the request. HTTP allows repeated headers.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.HTTPHeader" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "path": { + "description": "Path to access on the HTTP server.", + "type": "string" + }, + "port": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.util.intstr.IntOrString", + "description": "Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME." + }, + "scheme": { + "description": "Scheme to use for connecting to the host. Defaults to HTTP.", + "type": "string" + } + }, + "required": [ + "port" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.HTTPHeader": { + "description": "HTTPHeader describes a custom header to be used in HTTP probes", + "properties": { + "name": { + "description": "The header field name. This will be canonicalized upon output, so case-variant names will be understood as the same header.", + "type": "string" + }, + "value": { + "description": "The header field value", + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.HostAlias": { + "description": "HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the pod's hosts file.", + "properties": { + "hostnames": { + "description": "Hostnames for the above IP address.", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "ip": { + "description": "IP address of the host file entry.", + "type": "string" + } + }, + "required": [ + "ip" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.HostPathVolumeSource": { + "description": "Represents a host path mapped into a pod. Host path volumes do not support ownership management or SELinux relabeling.", + "properties": { + "path": { + "description": "path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", + "type": "string" + }, + "type": { + "description": "type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", + "type": "string" + } + }, + "required": [ + "path" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ISCSIVolumeSource": { + "description": "Represents an ISCSI disk. ISCSI volumes can only be mounted as read/write once. ISCSI volumes support ownership management and SELinux relabeling.", + "properties": { + "chapAuthDiscovery": { + "description": "chapAuthDiscovery defines whether support iSCSI Discovery CHAP authentication", + "type": "boolean" + }, + "chapAuthSession": { + "description": "chapAuthSession defines whether support iSCSI Session CHAP authentication", + "type": "boolean" + }, + "fsType": { + "description": "fsType is the filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi", + "type": "string" + }, + "initiatorName": { + "description": "initiatorName is the custom iSCSI Initiator Name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface : will be created for the connection.", + "type": "string" + }, + "iqn": { + "description": "iqn is the target iSCSI Qualified Name.", + "type": "string" + }, + "iscsiInterface": { + "description": "iscsiInterface is the interface Name that uses an iSCSI transport. Defaults to 'default' (tcp).", + "type": "string" + }, + "lun": { + "description": "lun represents iSCSI Target Lun number.", + "format": "int32", + "type": "integer" + }, + "portals": { + "description": "portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260).", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "readOnly": { + "description": "readOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false.", + "type": "boolean" + }, + "secretRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.LocalObjectReference", + "description": "secretRef is the CHAP Secret for iSCSI target and initiator authentication" + }, + "targetPortal": { + "description": "targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260).", + "type": "string" + } + }, + "required": [ + "targetPortal", + "iqn", + "lun" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ImageVolumeSource": { + "description": "ImageVolumeSource represents a image volume resource.", + "properties": { + "pullPolicy": { + "description": "Policy for pulling OCI objects. Possible values are: Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise.", + "type": "string" + }, + "reference": { + "description": "Required: Image or artifact reference to be used. Behaves in the same way as pod.spec.containers[*].image. Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.", + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.KeyToPath": { + "description": "Maps a string key to a path within a volume.", + "properties": { + "key": { + "description": "key is the key to project.", + "type": "string" + }, + "mode": { + "description": "mode is Optional: mode bits used to set permissions on this file. Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", + "format": "int32", + "type": "integer" + }, + "path": { + "description": "path is the relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.", + "type": "string" + } + }, + "required": [ + "key", + "path" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.Lifecycle": { + "description": "Lifecycle describes actions that the management system should take in response to container lifecycle events. For the PostStart and PreStop lifecycle handlers, management of the container blocks until the action is complete, unless the container process fails, in which case the handler is aborted.", + "properties": { + "postStart": { + "$ref": "#/definitions/io.k8s.api.core.v1.LifecycleHandler", + "description": "PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks" + }, + "preStop": { + "$ref": "#/definitions/io.k8s.api.core.v1.LifecycleHandler", + "description": "PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The Pod's termination grace period countdown begins before the PreStop hook is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod's termination grace period (unless delayed by finalizers). Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.LifecycleHandler": { + "description": "LifecycleHandler defines a specific action that should be taken in a lifecycle hook. One and only one of the fields, except TCPSocket must be specified.", + "properties": { + "exec": { + "$ref": "#/definitions/io.k8s.api.core.v1.ExecAction", + "description": "Exec specifies the action to take." + }, + "httpGet": { + "$ref": "#/definitions/io.k8s.api.core.v1.HTTPGetAction", + "description": "HTTPGet specifies the http request to perform." + }, + "sleep": { + "$ref": "#/definitions/io.k8s.api.core.v1.SleepAction", + "description": "Sleep represents the duration that the container should sleep before being terminated." + }, + "tcpSocket": { + "$ref": "#/definitions/io.k8s.api.core.v1.TCPSocketAction", + "description": "Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept for the backward compatibility. There are no validation of this field and lifecycle hooks will fail in runtime when tcp handler is specified." + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.LimitRangeItem": { + "description": "LimitRangeItem defines a min/max usage limit for any resource that matches on kind.", + "properties": { + "default": { + "additionalProperties": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + }, + "description": "Default resource requirement limit value by resource name if resource limit is omitted.", + "type": "object" + }, + "defaultRequest": { + "additionalProperties": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + }, + "description": "DefaultRequest is the default resource requirement request value by resource name if resource request is omitted.", + "type": "object" + }, + "max": { + "additionalProperties": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + }, + "description": "Max usage constraints on this kind by resource name.", + "type": "object" + }, + "maxLimitRequestRatio": { + "additionalProperties": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + }, + "description": "MaxLimitRequestRatio if specified, the named resource must have a request and limit that are both non-zero where limit divided by request is less than or equal to the enumerated value; this represents the max burst for the named resource.", + "type": "object" + }, + "min": { + "additionalProperties": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + }, + "description": "Min usage constraints on this kind by resource name.", + "type": "object" + }, + "type": { + "description": "Type of resource that this limit applies to.", + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.LocalObjectReference": { + "description": "LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.", + "properties": { + "name": { + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + } + }, + "type": "object", + "x-kubernetes-map-type": "atomic", + "additionalProperties": false + }, + "io.k8s.api.core.v1.NFSVolumeSource": { + "description": "Represents an NFS mount that lasts the lifetime of a pod. NFS volumes do not support ownership management or SELinux relabeling.", + "properties": { + "path": { + "description": "path that is exported by the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs", + "type": "string" + }, + "readOnly": { + "description": "readOnly here will force the NFS export to be mounted with read-only permissions. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs", + "type": "boolean" + }, + "server": { + "description": "server is the hostname or IP address of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs", + "type": "string" + } + }, + "required": [ + "server", + "path" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.NodeAffinity": { + "description": "Node affinity is a group of node affinity scheduling rules.", + "properties": { + "preferredDuringSchedulingIgnoredDuringExecution": { + "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.PreferredSchedulingTerm" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "requiredDuringSchedulingIgnoredDuringExecution": { + "$ref": "#/definitions/io.k8s.api.core.v1.NodeSelector", + "description": "If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node." + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.NodeSelector": { + "description": "A node selector represents the union of the results of one or more label queries over a set of nodes; that is, it represents the OR of the selectors represented by the node selector terms.", + "properties": { + "nodeSelectorTerms": { + "description": "Required. A list of node selector terms. The terms are ORed.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.NodeSelectorTerm" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + } + }, + "required": [ + "nodeSelectorTerms" + ], + "type": "object", + "x-kubernetes-map-type": "atomic", + "additionalProperties": false + }, + "io.k8s.api.core.v1.NodeSelectorRequirement": { + "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", + "properties": { + "key": { + "description": "The label key that the selector applies to.", + "type": "string" + }, + "operator": { + "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", + "type": "string" + }, + "values": { + "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + } + }, + "required": [ + "key", + "operator" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.NodeSelectorTerm": { + "description": "A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.", + "properties": { + "matchExpressions": { + "description": "A list of node selector requirements by node's labels.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.NodeSelectorRequirement" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "matchFields": { + "description": "A list of node selector requirements by node's fields.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.NodeSelectorRequirement" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + } + }, + "type": "object", + "x-kubernetes-map-type": "atomic", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ObjectFieldSelector": { + "description": "ObjectFieldSelector selects an APIVersioned field of an object.", + "properties": { + "apiVersion": { + "description": "Version of the schema the FieldPath is written in terms of, defaults to \"v1\".", + "type": "string" + }, + "fieldPath": { + "description": "Path of the field to select in the specified API version.", + "type": "string" + } + }, + "required": [ + "fieldPath" + ], + "type": "object", + "x-kubernetes-map-type": "atomic", + "additionalProperties": false + }, + "io.k8s.api.core.v1.PersistentVolumeClaimSpec": { + "description": "PersistentVolumeClaimSpec describes the common attributes of storage devices and allows a Source for provider-specific attributes", + "properties": { + "accessModes": { + "description": "accessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "dataSource": { + "$ref": "#/definitions/io.k8s.api.core.v1.TypedLocalObjectReference", + "description": "dataSource field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. If the namespace is specified, then dataSourceRef will not be copied to dataSource." + }, + "dataSourceRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.TypedObjectReference", + "description": "dataSourceRef specifies the object from which to populate the volume with data, if a non-empty volume is desired. This may be any object from a non-empty API group (non core object) or a PersistentVolumeClaim object. When this field is specified, volume binding will only succeed if the type of the specified object matches some installed volume populator or dynamic provisioner. This field will replace the functionality of the dataSource field and as such if both fields are non-empty, they must have the same value. For backwards compatibility, when namespace isn't specified in dataSourceRef, both fields (dataSource and dataSourceRef) will be set to the same value automatically if one of them is empty and the other is non-empty. When namespace is specified in dataSourceRef, dataSource isn't set to the same value and must be empty. There are three important differences between dataSource and dataSourceRef: * While dataSource only allows two specific types of objects, dataSourceRef\n allows any non-core object, as well as PersistentVolumeClaim objects.\n* While dataSource ignores disallowed values (dropping them), dataSourceRef\n preserves all values, and generates an error if a disallowed value is\n specified.\n* While dataSource only allows local objects, dataSourceRef allows objects\n in any namespaces.\n(Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled." + }, + "resources": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeResourceRequirements", + "description": "resources represents the minimum resources the volume should have. If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements that are lower than previous value but must still be higher than capacity recorded in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources" + }, + "selector": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector", + "description": "selector is a label query over volumes to consider for binding." + }, + "storageClassName": { + "description": "storageClassName is the name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1", + "type": "string" + }, + "volumeAttributesClassName": { + "description": "volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass will be applied to the claim but it's not allowed to reset this field to empty string once it is set. If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass will be set by the persistentvolume controller if it exists. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default).", + "type": "string" + }, + "volumeMode": { + "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec.", + "type": "string" + }, + "volumeName": { + "description": "volumeName is the binding reference to the PersistentVolume backing this claim.", + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.PersistentVolumeClaimTemplate": { + "description": "PersistentVolumeClaimTemplate is used to produce PersistentVolumeClaim objects as part of an EphemeralVolumeSource.", + "properties": { + "metadata": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta", + "description": "May contain labels and annotations that will be copied into the PVC when creating it. No other fields are allowed and will be rejected during validation." + }, + "spec": { + "$ref": "#/definitions/io.k8s.api.core.v1.PersistentVolumeClaimSpec", + "description": "The specification for the PersistentVolumeClaim. The entire content is copied unchanged into the PVC that gets created from this template. The same fields as in a PersistentVolumeClaim are also valid here." + } + }, + "required": [ + "spec" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.PersistentVolumeClaimVolumeSource": { + "description": "PersistentVolumeClaimVolumeSource references the user's PVC in the same namespace. This volume finds the bound PV and mounts that volume for the pod. A PersistentVolumeClaimVolumeSource is, essentially, a wrapper around another type of volume that is owned by someone else (the system).", + "properties": { + "claimName": { + "description": "claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims", + "type": "string" + }, + "readOnly": { + "description": "readOnly Will force the ReadOnly setting in VolumeMounts. Default false.", + "type": "boolean" + } + }, + "required": [ + "claimName" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.PhotonPersistentDiskVolumeSource": { + "description": "Represents a Photon Controller persistent disk resource.", + "properties": { + "fsType": { + "description": "fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.", + "type": "string" + }, + "pdID": { + "description": "pdID is the ID that identifies Photon Controller persistent disk", + "type": "string" + } + }, + "required": [ + "pdID" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.PodAffinity": { + "description": "Pod affinity is a group of inter pod affinity scheduling rules.", + "properties": { + "preferredDuringSchedulingIgnoredDuringExecution": { + "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.WeightedPodAffinityTerm" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "requiredDuringSchedulingIgnoredDuringExecution": { + "description": "If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.PodAffinityTerm" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.PodAffinityTerm": { + "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running", + "properties": { + "labelSelector": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector", + "description": "A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods." + }, + "matchLabelKeys": { + "description": "MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "mismatchLabelKeys": { + "description": "MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "namespaceSelector": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector", + "description": "A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means \"this pod's namespace\". An empty selector ({}) matches all namespaces." + }, + "namespaces": { + "description": "namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "topologyKey": { + "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", + "type": "string" + } + }, + "required": [ + "topologyKey" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.PodAntiAffinity": { + "description": "Pod anti affinity is a group of inter pod anti affinity scheduling rules.", + "properties": { + "preferredDuringSchedulingIgnoredDuringExecution": { + "description": "The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.WeightedPodAffinityTerm" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "requiredDuringSchedulingIgnoredDuringExecution": { + "description": "If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.PodAffinityTerm" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.PodSecurityContext": { + "description": "PodSecurityContext holds pod-level security attributes and common container settings. Some fields are also present in container.securityContext. Field values of container.securityContext take precedence over field values of PodSecurityContext.", + "properties": { + "appArmorProfile": { + "$ref": "#/definitions/io.k8s.api.core.v1.AppArmorProfile", + "description": "appArmorProfile is the AppArmor options to use by the containers in this pod. Note that this field cannot be set when spec.os.name is windows." + }, + "fsGroup": { + "description": "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod:\n\n1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw----\n\nIf unset, the Kubelet will not modify the ownership and permissions of any volume. Note that this field cannot be set when spec.os.name is windows.", + "format": "int64", + "type": "integer" + }, + "fsGroupChangePolicy": { + "description": "fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are \"OnRootMismatch\" and \"Always\". If not specified, \"Always\" is used. Note that this field cannot be set when spec.os.name is windows.", + "type": "string" + }, + "runAsGroup": { + "description": "The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows.", + "format": "int64", + "type": "integer" + }, + "runAsNonRoot": { + "description": "Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", + "type": "boolean" + }, + "runAsUser": { + "description": "The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows.", + "format": "int64", + "type": "integer" + }, + "seLinuxOptions": { + "$ref": "#/definitions/io.k8s.api.core.v1.SELinuxOptions", + "description": "The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows." + }, + "seccompProfile": { + "$ref": "#/definitions/io.k8s.api.core.v1.SeccompProfile", + "description": "The seccomp options to use by the containers in this pod. Note that this field cannot be set when spec.os.name is windows." + }, + "supplementalGroups": { + "description": "A list of groups applied to the first process run in each container, in addition to the container's primary GID and fsGroup (if specified). If the SupplementalGroupsPolicy feature is enabled, the supplementalGroupsPolicy field determines whether these are in addition to or instead of any group memberships defined in the container image. If unspecified, no additional groups are added, though group memberships defined in the container image may still be used, depending on the supplementalGroupsPolicy field. Note that this field cannot be set when spec.os.name is windows.", + "items": { + "format": "int64", + "type": "integer" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "supplementalGroupsPolicy": { + "description": "Defines how supplemental groups of the first container processes are calculated. Valid values are \"Merge\" and \"Strict\". If not specified, \"Merge\" is used. (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled and the container runtime must implement support for this feature. Note that this field cannot be set when spec.os.name is windows.", + "type": "string" + }, + "sysctls": { + "description": "Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch. Note that this field cannot be set when spec.os.name is windows.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Sysctl" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "windowsOptions": { + "$ref": "#/definitions/io.k8s.api.core.v1.WindowsSecurityContextOptions", + "description": "The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux." + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.PortworxVolumeSource": { + "description": "PortworxVolumeSource represents a Portworx volume resource.", + "properties": { + "fsType": { + "description": "fSType represents the filesystem type to mount Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\". Implicitly inferred to be \"ext4\" if unspecified.", + "type": "string" + }, + "readOnly": { + "description": "readOnly defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", + "type": "boolean" + }, + "volumeID": { + "description": "volumeID uniquely identifies a Portworx volume", + "type": "string" + } + }, + "required": [ + "volumeID" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.PreferredSchedulingTerm": { + "description": "An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).", + "properties": { + "preference": { + "$ref": "#/definitions/io.k8s.api.core.v1.NodeSelectorTerm", + "description": "A node selector term, associated with the corresponding weight." + }, + "weight": { + "description": "Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.", + "format": "int32", + "type": "integer" + } + }, + "required": [ + "weight", + "preference" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.Probe": { + "description": "Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.", + "properties": { + "exec": { + "$ref": "#/definitions/io.k8s.api.core.v1.ExecAction", + "description": "Exec specifies the action to take." + }, + "failureThreshold": { + "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.", + "format": "int32", + "type": "integer" + }, + "grpc": { + "$ref": "#/definitions/io.k8s.api.core.v1.GRPCAction", + "description": "GRPC specifies an action involving a GRPC port." + }, + "httpGet": { + "$ref": "#/definitions/io.k8s.api.core.v1.HTTPGetAction", + "description": "HTTPGet specifies the http request to perform." + }, + "initialDelaySeconds": { + "description": "Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", + "format": "int32", + "type": "integer" + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.", + "format": "int32", + "type": "integer" + }, + "successThreshold": { + "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.", + "format": "int32", + "type": "integer" + }, + "tcpSocket": { + "$ref": "#/definitions/io.k8s.api.core.v1.TCPSocketAction", + "description": "TCPSocket specifies an action involving a TCP port." + }, + "terminationGracePeriodSeconds": { + "description": "Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.", + "format": "int64", + "type": "integer" + }, + "timeoutSeconds": { + "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", + "format": "int32", + "type": "integer" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ProjectedVolumeSource": { + "description": "Represents a projected volume source", + "properties": { + "defaultMode": { + "description": "defaultMode are the mode bits used to set permissions on created files by default. Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", + "format": "int32", + "type": "integer" + }, + "sources": { + "description": "sources is the list of volume projections. Each entry in this list handles one source.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeProjection" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.QuobyteVolumeSource": { + "description": "Represents a Quobyte mount that lasts the lifetime of a pod. Quobyte volumes do not support ownership management or SELinux relabeling.", + "properties": { + "group": { + "description": "group to map volume access to Default is no group", + "type": "string" + }, + "readOnly": { + "description": "readOnly here will force the Quobyte volume to be mounted with read-only permissions. Defaults to false.", + "type": "boolean" + }, + "registry": { + "description": "registry represents a single or multiple Quobyte Registry services specified as a string as host:port pair (multiple entries are separated with commas) which acts as the central registry for volumes", + "type": "string" + }, + "tenant": { + "description": "tenant owning the given Quobyte volume in the Backend Used with dynamically provisioned Quobyte volumes, value is set by the plugin", + "type": "string" + }, + "user": { + "description": "user to map volume access to Defaults to serivceaccount user", + "type": "string" + }, + "volume": { + "description": "volume is a string that references an already created Quobyte volume by name.", + "type": "string" + } + }, + "required": [ + "registry", + "volume" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.RBDVolumeSource": { + "description": "Represents a Rados Block Device mount that lasts the lifetime of a pod. RBD volumes support ownership management and SELinux relabeling.", + "properties": { + "fsType": { + "description": "fsType is the filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd", + "type": "string" + }, + "image": { + "description": "image is the rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", + "type": "string" + }, + "keyring": { + "description": "keyring is the path to key ring for RBDUser. Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", + "type": "string" + }, + "monitors": { + "description": "monitors is a collection of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "pool": { + "description": "pool is the rados pool name. Default is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", + "type": "string" + }, + "readOnly": { + "description": "readOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", + "type": "boolean" + }, + "secretRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.LocalObjectReference", + "description": "secretRef is name of the authentication secret for RBDUser. If provided overrides keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it" + }, + "user": { + "description": "user is the rados user name. Default is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", + "type": "string" + } + }, + "required": [ + "monitors", + "image" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ResourceClaim": { + "description": "ResourceClaim references one entry in PodSpec.ResourceClaims.", + "properties": { + "name": { + "description": "Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container.", + "type": "string" + }, + "request": { + "description": "Request is the name chosen for a request in the referenced claim. If empty, everything from the claim is made available, otherwise only the result of this request.", + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ResourceFieldSelector": { + "description": "ResourceFieldSelector represents container resources (cpu, memory) and their output format", + "properties": { + "containerName": { + "description": "Container name: required for volumes, optional for env vars", + "type": "string" + }, + "divisor": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity", + "description": "Specifies the output format of the exposed resources, defaults to \"1\"" + }, + "resource": { + "description": "Required: resource to select", + "type": "string" + } + }, + "required": [ + "resource" + ], + "type": "object", + "x-kubernetes-map-type": "atomic", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ResourceRequirements": { + "description": "ResourceRequirements describes the compute resource requirements.", + "properties": { + "claims": { + "description": "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container.\n\nThis is an alpha field and requires enabling the DynamicResourceAllocation feature gate.\n\nThis field is immutable. It can only be set for containers.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceClaim" + }, + "type": "array", + "x-kubernetes-list-map-keys": [ + "name" + ], + "x-kubernetes-list-type": "map" + }, + "limits": { + "additionalProperties": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + }, + "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/", + "type": "object" + }, + "requests": { + "additionalProperties": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + }, + "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/", + "type": "object" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.SELinuxOptions": { + "description": "SELinuxOptions are the labels to be applied to the container", + "properties": { + "level": { + "description": "Level is SELinux level label that applies to the container.", + "type": "string" + }, + "role": { + "description": "Role is a SELinux role label that applies to the container.", + "type": "string" + }, + "type": { + "description": "Type is a SELinux type label that applies to the container.", + "type": "string" + }, + "user": { + "description": "User is a SELinux user label that applies to the container.", + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ScaleIOVolumeSource": { + "description": "ScaleIOVolumeSource represents a persistent ScaleIO volume", + "properties": { + "fsType": { + "description": "fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Default is \"xfs\".", + "type": "string" + }, + "gateway": { + "description": "gateway is the host address of the ScaleIO API Gateway.", + "type": "string" + }, + "protectionDomain": { + "description": "protectionDomain is the name of the ScaleIO Protection Domain for the configured storage.", + "type": "string" + }, + "readOnly": { + "description": "readOnly Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", + "type": "boolean" + }, + "secretRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.LocalObjectReference", + "description": "secretRef references to the secret for ScaleIO user and other sensitive information. If this is not provided, Login operation will fail." + }, + "sslEnabled": { + "description": "sslEnabled Flag enable/disable SSL communication with Gateway, default false", + "type": "boolean" + }, + "storageMode": { + "description": "storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. Default is ThinProvisioned.", + "type": "string" + }, + "storagePool": { + "description": "storagePool is the ScaleIO Storage Pool associated with the protection domain.", + "type": "string" + }, + "system": { + "description": "system is the name of the storage system as configured in ScaleIO.", + "type": "string" + }, + "volumeName": { + "description": "volumeName is the name of a volume already created in the ScaleIO system that is associated with this volume source.", + "type": "string" + } + }, + "required": [ + "gateway", + "system", + "secretRef" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.SeccompProfile": { + "description": "SeccompProfile defines a pod/container's seccomp profile settings. Only one profile source may be set.", + "properties": { + "localhostProfile": { + "description": "localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is \"Localhost\". Must NOT be set for any other type.", + "type": "string" + }, + "type": { + "description": "type indicates which kind of seccomp profile will be applied. Valid options are:\n\nLocalhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied.", + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object", + "x-kubernetes-unions": [ + { + "discriminator": "type", + "fields-to-discriminateBy": { + "localhostProfile": "LocalhostProfile" + } + } + ], + "additionalProperties": false + }, + "io.k8s.api.core.v1.SecretEnvSource": { + "description": "SecretEnvSource selects a Secret to populate the environment variables with.\n\nThe contents of the target Secret's Data field will represent the key-value pairs as environment variables.", + "properties": { + "name": { + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + }, + "optional": { + "description": "Specify whether the Secret must be defined", + "type": "boolean" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.SecretKeySelector": { + "description": "SecretKeySelector selects a key of a Secret.", + "properties": { + "key": { + "description": "The key of the secret to select from. Must be a valid secret key.", + "type": "string" + }, + "name": { + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + }, + "optional": { + "description": "Specify whether the Secret or its key must be defined", + "type": "boolean" + } + }, + "required": [ + "key" + ], + "type": "object", + "x-kubernetes-map-type": "atomic", + "additionalProperties": false + }, + "io.k8s.api.core.v1.SecretProjection": { + "description": "Adapts a secret into a projected volume.\n\nThe contents of the target Secret's Data field will be presented in a projected volume as files using the keys in the Data field as the file names. Note that this is identical to a secret volume source without the default mode.", + "properties": { + "items": { + "description": "items if unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.KeyToPath" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "name": { + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + }, + "optional": { + "description": "optional field specify whether the Secret or its key must be defined", + "type": "boolean" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.SecretVolumeSource": { + "description": "Adapts a Secret into a volume.\n\nThe contents of the target Secret's Data field will be presented in a volume as files using the keys in the Data field as the file names. Secret volumes support ownership management and SELinux relabeling.", + "properties": { + "defaultMode": { + "description": "defaultMode is Optional: mode bits used to set permissions on created files by default. Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", + "format": "int32", + "type": "integer" + }, + "items": { + "description": "items If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.KeyToPath" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "optional": { + "description": "optional field specify whether the Secret or its keys must be defined", + "type": "boolean" + }, + "secretName": { + "description": "secretName is the name of the secret in the pod's namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret", + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.SecurityContext": { + "description": "SecurityContext holds security configuration that will be applied to a container. Some fields are present in both SecurityContext and PodSecurityContext. When both are set, the values in SecurityContext take precedence.", + "properties": { + "allowPrivilegeEscalation": { + "description": "AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN Note that this field cannot be set when spec.os.name is windows.", + "type": "boolean" + }, + "appArmorProfile": { + "$ref": "#/definitions/io.k8s.api.core.v1.AppArmorProfile", + "description": "appArmorProfile is the AppArmor options to use by this container. If set, this profile overrides the pod's appArmorProfile. Note that this field cannot be set when spec.os.name is windows." + }, + "capabilities": { + "$ref": "#/definitions/io.k8s.api.core.v1.Capabilities", + "description": "The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. Note that this field cannot be set when spec.os.name is windows." + }, + "privileged": { + "description": "Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. Note that this field cannot be set when spec.os.name is windows.", + "type": "boolean" + }, + "procMount": { + "description": "procMount denotes the type of proc mount to use for the containers. The default value is Default which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows.", + "type": "string" + }, + "readOnlyRootFilesystem": { + "description": "Whether this container has a read-only root filesystem. Default is false. Note that this field cannot be set when spec.os.name is windows.", + "type": "boolean" + }, + "runAsGroup": { + "description": "The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.", + "format": "int64", + "type": "integer" + }, + "runAsNonRoot": { + "description": "Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", + "type": "boolean" + }, + "runAsUser": { + "description": "The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.", + "format": "int64", + "type": "integer" + }, + "seLinuxOptions": { + "$ref": "#/definitions/io.k8s.api.core.v1.SELinuxOptions", + "description": "The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows." + }, + "seccompProfile": { + "$ref": "#/definitions/io.k8s.api.core.v1.SeccompProfile", + "description": "The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. Note that this field cannot be set when spec.os.name is windows." + }, + "windowsOptions": { + "$ref": "#/definitions/io.k8s.api.core.v1.WindowsSecurityContextOptions", + "description": "The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux." + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.ServiceAccountTokenProjection": { + "description": "ServiceAccountTokenProjection represents a projected service account token volume. This projection can be used to insert a service account token into the pods runtime filesystem for use against APIs (Kubernetes API Server or otherwise).", + "properties": { + "audience": { + "description": "audience is the intended audience of the token. A recipient of a token must identify itself with an identifier specified in the audience of the token, and otherwise should reject the token. The audience defaults to the identifier of the apiserver.", + "type": "string" + }, + "expirationSeconds": { + "description": "expirationSeconds is the requested duration of validity of the service account token. As the token approaches expiration, the kubelet volume plugin will proactively rotate the service account token. The kubelet will start trying to rotate the token if the token is older than 80 percent of its time to live or if the token is older than 24 hours.Defaults to 1 hour and must be at least 10 minutes.", + "format": "int64", + "type": "integer" + }, + "path": { + "description": "path is the path relative to the mount point of the file to project the token into.", + "type": "string" + } + }, + "required": [ + "path" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.SleepAction": { + "description": "SleepAction describes a \"sleep\" action.", + "properties": { + "seconds": { + "description": "Seconds is the number of seconds to sleep.", + "format": "int64", + "type": "integer" + } + }, + "required": [ + "seconds" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.StorageOSVolumeSource": { + "description": "Represents a StorageOS persistent volume resource.", + "properties": { + "fsType": { + "description": "fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.", + "type": "string" + }, + "readOnly": { + "description": "readOnly defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", + "type": "boolean" + }, + "secretRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.LocalObjectReference", + "description": "secretRef specifies the secret to use for obtaining the StorageOS API credentials. If not specified, default values will be attempted." + }, + "volumeName": { + "description": "volumeName is the human-readable name of the StorageOS volume. Volume names are only unique within a namespace.", + "type": "string" + }, + "volumeNamespace": { + "description": "volumeNamespace specifies the scope of the volume within StorageOS. If no namespace is specified then the Pod's namespace will be used. This allows the Kubernetes name scoping to be mirrored within StorageOS for tighter integration. Set VolumeName to any name to override the default behaviour. Set to \"default\" if you are not using namespaces within StorageOS. Namespaces that do not pre-exist within StorageOS will be created.", + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.Sysctl": { + "description": "Sysctl defines a kernel parameter to be set", + "properties": { + "name": { + "description": "Name of a property to set", + "type": "string" + }, + "value": { + "description": "Value of a property to set", + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.TCPSocketAction": { + "description": "TCPSocketAction describes an action based on opening a socket", + "properties": { + "host": { + "description": "Optional: Host name to connect to, defaults to the pod IP.", + "type": "string" + }, + "port": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.util.intstr.IntOrString", + "description": "Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME." + } + }, + "required": [ + "port" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.Toleration": { + "description": "The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator .", + "properties": { + "effect": { + "description": "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.", + "type": "string" + }, + "key": { + "description": "Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.", + "type": "string" + }, + "operator": { + "description": "Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.", + "type": "string" + }, + "tolerationSeconds": { + "description": "TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.", + "format": "int64", + "type": "integer" + }, + "value": { + "description": "Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.", + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.TopologySpreadConstraint": { + "description": "TopologySpreadConstraint specifies how to spread matching pods among the given topology.", + "properties": { + "labelSelector": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector", + "description": "LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain." + }, + "matchLabelKeys": { + "description": "MatchLabelKeys is a set of pod label keys to select the pods over which spreading will be calculated. The keys are used to lookup values from the incoming pod labels, those key-value labels are ANDed with labelSelector to select the group of existing pods over which spreading will be calculated for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. MatchLabelKeys cannot be set when LabelSelector isn't set. Keys that don't exist in the incoming pod labels will be ignored. A null or empty list means only match against labelSelector.\n\nThis is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default).", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "maxSkew": { + "description": "MaxSkew describes the degree to which pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference between the number of matching pods in the target topology and the global minimum. The global minimum is the minimum number of matching pods in an eligible domain or zero if the number of eligible domains is less than MinDomains. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 2/2/1: In this case, the global minimum is 1. | zone1 | zone2 | zone3 | | P P | P P | P | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence to topologies that satisfy it. It's a required field. Default value is 1 and 0 is not allowed.", + "format": "int32", + "type": "integer" + }, + "minDomains": { + "description": "MinDomains indicates a minimum number of eligible domains. When the number of eligible domains with matching topology keys is less than minDomains, Pod Topology Spread treats \"global minimum\" as 0, and then the calculation of Skew is performed. And when the number of eligible domains with matching topology keys equals or greater than minDomains, this value has no effect on scheduling. As a result, when the number of eligible domains is less than minDomains, scheduler won't schedule more than maxSkew Pods to those domains. If value is nil, the constraint behaves as if MinDomains is equal to 1. Valid values are integers greater than 0. When value is not nil, WhenUnsatisfiable must be DoNotSchedule.\n\nFor example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same labelSelector spread as 2/2/2: | zone1 | zone2 | zone3 | | P P | P P | P P | The number of domains is less than 5(MinDomains), so \"global minimum\" is treated as 0. In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew.", + "format": "int32", + "type": "integer" + }, + "nodeAffinityPolicy": { + "description": "NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector when calculating pod topology spread skew. Options are: - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations.\n\nIf this value is nil, the behavior is equivalent to the Honor policy. This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.", + "type": "string" + }, + "nodeTaintsPolicy": { + "description": "NodeTaintsPolicy indicates how we will treat node taints when calculating pod topology spread skew. Options are: - Honor: nodes without taints, along with tainted nodes for which the incoming pod has a toleration, are included. - Ignore: node taints are ignored. All nodes are included.\n\nIf this value is nil, the behavior is equivalent to the Ignore policy. This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.", + "type": "string" + }, + "topologyKey": { + "description": "TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each as a \"bucket\", and try to put balanced number of pods into each bucket. We define a domain as a particular instance of a topology. Also, we define an eligible domain as a domain whose nodes meet the requirements of nodeAffinityPolicy and nodeTaintsPolicy. e.g. If TopologyKey is \"kubernetes.io/hostname\", each Node is a domain of that topology. And, if TopologyKey is \"topology.kubernetes.io/zone\", each zone is a domain of that topology. It's a required field.", + "type": "string" + }, + "whenUnsatisfiable": { + "description": "WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it. - ScheduleAnyway tells the scheduler to schedule the pod in any location,\n but giving higher precedence to topologies that would help reduce the\n skew.\nA constraint is considered \"Unsatisfiable\" for an incoming pod if and only if every possible node assignment for that pod would violate \"MaxSkew\" on some topology. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won't make it *more* imbalanced. It's a required field.", + "type": "string" + } + }, + "required": [ + "maxSkew", + "topologyKey", + "whenUnsatisfiable" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.TypedLocalObjectReference": { + "description": "TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace.", + "properties": { + "apiGroup": { + "description": "APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.", + "type": "string" + }, + "kind": { + "description": "Kind is the type of resource being referenced", + "type": "string" + }, + "name": { + "description": "Name is the name of resource being referenced", + "type": "string" + } + }, + "required": [ + "kind", + "name" + ], + "type": "object", + "x-kubernetes-map-type": "atomic", + "additionalProperties": false + }, + "io.k8s.api.core.v1.TypedObjectReference": { + "properties": { + "apiGroup": { + "description": "APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.", + "type": "string" + }, + "kind": { + "description": "Kind is the type of resource being referenced", + "type": "string" + }, + "name": { + "description": "Name is the name of resource being referenced", + "type": "string" + }, + "namespace": { + "description": "Namespace is the namespace of resource being referenced Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled.", + "type": "string" + } + }, + "required": [ + "kind", + "name" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.Volume": { + "description": "Volume represents a named volume in a pod that may be accessed by any container in the pod.", + "properties": { + "awsElasticBlockStore": { + "$ref": "#/definitions/io.k8s.api.core.v1.AWSElasticBlockStoreVolumeSource", + "description": "awsElasticBlockStore represents an AWS Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore" + }, + "azureDisk": { + "$ref": "#/definitions/io.k8s.api.core.v1.AzureDiskVolumeSource", + "description": "azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod." + }, + "azureFile": { + "$ref": "#/definitions/io.k8s.api.core.v1.AzureFileVolumeSource", + "description": "azureFile represents an Azure File Service mount on the host and bind mount to the pod." + }, + "cephfs": { + "$ref": "#/definitions/io.k8s.api.core.v1.CephFSVolumeSource", + "description": "cephFS represents a Ceph FS mount on the host that shares a pod's lifetime" + }, + "cinder": { + "$ref": "#/definitions/io.k8s.api.core.v1.CinderVolumeSource", + "description": "cinder represents a cinder volume attached and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md" + }, + "configMap": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapVolumeSource", + "description": "configMap represents a configMap that should populate this volume" + }, + "csi": { + "$ref": "#/definitions/io.k8s.api.core.v1.CSIVolumeSource", + "description": "csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers (Beta feature)." + }, + "downwardAPI": { + "$ref": "#/definitions/io.k8s.api.core.v1.DownwardAPIVolumeSource", + "description": "downwardAPI represents downward API about the pod that should populate this volume" + }, + "emptyDir": { + "$ref": "#/definitions/io.k8s.api.core.v1.EmptyDirVolumeSource", + "description": "emptyDir represents a temporary directory that shares a pod's lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir" + }, + "ephemeral": { + "$ref": "#/definitions/io.k8s.api.core.v1.EphemeralVolumeSource", + "description": "ephemeral represents a volume that is handled by a cluster storage driver. The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, and deleted when the pod is removed.\n\nUse this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot or capacity\n tracking are needed,\nc) the storage driver is specified through a storage class, and d) the storage driver supports dynamic volume provisioning through\n a PersistentVolumeClaim (see EphemeralVolumeSource for more\n information on the connection between this volume type\n and PersistentVolumeClaim).\n\nUse PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of an individual pod.\n\nUse CSI for light-weight local ephemeral volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information.\n\nA pod can use both types of ephemeral volumes and persistent volumes at the same time." + }, + "fc": { + "$ref": "#/definitions/io.k8s.api.core.v1.FCVolumeSource", + "description": "fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod." + }, + "flexVolume": { + "$ref": "#/definitions/io.k8s.api.core.v1.FlexVolumeSource", + "description": "flexVolume represents a generic volume resource that is provisioned/attached using an exec based plugin." + }, + "flocker": { + "$ref": "#/definitions/io.k8s.api.core.v1.FlockerVolumeSource", + "description": "flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running" + }, + "gcePersistentDisk": { + "$ref": "#/definitions/io.k8s.api.core.v1.GCEPersistentDiskVolumeSource", + "description": "gcePersistentDisk represents a GCE Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk" + }, + "gitRepo": { + "$ref": "#/definitions/io.k8s.api.core.v1.GitRepoVolumeSource", + "description": "gitRepo represents a git repository at a particular revision. DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir into the Pod's container." + }, + "glusterfs": { + "$ref": "#/definitions/io.k8s.api.core.v1.GlusterfsVolumeSource", + "description": "glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md" + }, + "hostPath": { + "$ref": "#/definitions/io.k8s.api.core.v1.HostPathVolumeSource", + "description": "hostPath represents a pre-existing file or directory on the host machine that is directly exposed to the container. This is generally used for system agents or other privileged things that are allowed to see the host machine. Most containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + }, + "image": { + "$ref": "#/definitions/io.k8s.api.core.v1.ImageVolumeSource", + "description": "image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. The volume is resolved at pod startup depending on which PullPolicy value is provided:\n\n- Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails.\n\nThe volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. The volume will be mounted read-only (ro) and non-executable files (noexec). Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type." + }, + "iscsi": { + "$ref": "#/definitions/io.k8s.api.core.v1.ISCSIVolumeSource", + "description": "iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md" + }, + "name": { + "description": "name of the volume. Must be a DNS_LABEL and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + }, + "nfs": { + "$ref": "#/definitions/io.k8s.api.core.v1.NFSVolumeSource", + "description": "nfs represents an NFS mount on the host that shares a pod's lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs" + }, + "persistentVolumeClaim": { + "$ref": "#/definitions/io.k8s.api.core.v1.PersistentVolumeClaimVolumeSource", + "description": "persistentVolumeClaimVolumeSource represents a reference to a PersistentVolumeClaim in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims" + }, + "photonPersistentDisk": { + "$ref": "#/definitions/io.k8s.api.core.v1.PhotonPersistentDiskVolumeSource", + "description": "photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine" + }, + "portworxVolume": { + "$ref": "#/definitions/io.k8s.api.core.v1.PortworxVolumeSource", + "description": "portworxVolume represents a portworx volume attached and mounted on kubelets host machine" + }, + "projected": { + "$ref": "#/definitions/io.k8s.api.core.v1.ProjectedVolumeSource", + "description": "projected items for all in one resources secrets, configmaps, and downward API" + }, + "quobyte": { + "$ref": "#/definitions/io.k8s.api.core.v1.QuobyteVolumeSource", + "description": "quobyte represents a Quobyte mount on the host that shares a pod's lifetime" + }, + "rbd": { + "$ref": "#/definitions/io.k8s.api.core.v1.RBDVolumeSource", + "description": "rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md" + }, + "scaleIO": { + "$ref": "#/definitions/io.k8s.api.core.v1.ScaleIOVolumeSource", + "description": "scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes." + }, + "secret": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretVolumeSource", + "description": "secret represents a secret that should populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret" + }, + "storageos": { + "$ref": "#/definitions/io.k8s.api.core.v1.StorageOSVolumeSource", + "description": "storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes." + }, + "vsphereVolume": { + "$ref": "#/definitions/io.k8s.api.core.v1.VsphereVirtualDiskVolumeSource", + "description": "vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine" + } + }, + "required": [ + "name" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.VolumeDevice": { + "description": "volumeDevice describes a mapping of a raw block device within a container.", + "properties": { + "devicePath": { + "description": "devicePath is the path inside of the container that the device will be mapped to.", + "type": "string" + }, + "name": { + "description": "name must match the name of a persistentVolumeClaim in the pod", + "type": "string" + } + }, + "required": [ + "name", + "devicePath" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.VolumeMount": { + "description": "VolumeMount describes a mounting of a Volume within a container.", + "properties": { + "mountPath": { + "description": "Path within the container at which the volume should be mounted. Must not contain ':'.", + "type": "string" + }, + "mountPropagation": { + "description": "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified (which defaults to None).", + "type": "string" + }, + "name": { + "description": "This must match the Name of a Volume.", + "type": "string" + }, + "readOnly": { + "description": "Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.", + "type": "boolean" + }, + "recursiveReadOnly": { + "description": "RecursiveReadOnly specifies whether read-only mounts should be handled recursively.\n\nIf ReadOnly is false, this field has no meaning and must be unspecified.\n\nIf ReadOnly is true, and this field is set to Disabled, the mount is not made recursively read-only. If this field is set to IfPossible, the mount is made recursively read-only, if it is supported by the container runtime. If this field is set to Enabled, the mount is made recursively read-only if it is supported by the container runtime, otherwise the pod will not be started and an error will be generated to indicate the reason.\n\nIf this field is set to IfPossible or Enabled, MountPropagation must be set to None (or be unspecified, which defaults to None).\n\nIf this field is not specified, it is treated as an equivalent of Disabled.", + "type": "string" + }, + "subPath": { + "description": "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).", + "type": "string" + }, + "subPathExpr": { + "description": "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive.", + "type": "string" + } + }, + "required": [ + "name", + "mountPath" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.VolumeProjection": { + "description": "Projection that may be projected along with other supported volume types. Exactly one of these fields must be set.", + "properties": { + "clusterTrustBundle": { + "$ref": "#/definitions/io.k8s.api.core.v1.ClusterTrustBundleProjection", + "description": "ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field of ClusterTrustBundle objects in an auto-updating file.\n\nAlpha, gated by the ClusterTrustBundleProjection feature gate.\n\nClusterTrustBundle objects can either be selected by name, or by the combination of signer name and a label selector.\n\nKubelet performs aggressive normalization of the PEM contents written into the pod filesystem. Esoteric PEM features such as inter-block comments and block headers are stripped. Certificates are deduplicated. The ordering of certificates within the file is arbitrary, and Kubelet may change the order over time." + }, + "configMap": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapProjection", + "description": "configMap information about the configMap data to project" + }, + "downwardAPI": { + "$ref": "#/definitions/io.k8s.api.core.v1.DownwardAPIProjection", + "description": "downwardAPI information about the downwardAPI data to project" + }, + "secret": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretProjection", + "description": "secret information about the secret data to project" + }, + "serviceAccountToken": { + "$ref": "#/definitions/io.k8s.api.core.v1.ServiceAccountTokenProjection", + "description": "serviceAccountToken is information about the serviceAccountToken data to project" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.VolumeResourceRequirements": { + "description": "VolumeResourceRequirements describes the storage resource requirements for a volume.", + "properties": { + "limits": { + "additionalProperties": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + }, + "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/", + "type": "object" + }, + "requests": { + "additionalProperties": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + }, + "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/", + "type": "object" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.VsphereVirtualDiskVolumeSource": { + "description": "Represents a vSphere volume resource.", + "properties": { + "fsType": { + "description": "fsType is filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.", + "type": "string" + }, + "storagePolicyID": { + "description": "storagePolicyID is the storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName.", + "type": "string" + }, + "storagePolicyName": { + "description": "storagePolicyName is the storage Policy Based Management (SPBM) profile name.", + "type": "string" + }, + "volumePath": { + "description": "volumePath is the path that identifies vSphere volume vmdk", + "type": "string" + } + }, + "required": [ + "volumePath" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.WeightedPodAffinityTerm": { + "description": "The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)", + "properties": { + "podAffinityTerm": { + "$ref": "#/definitions/io.k8s.api.core.v1.PodAffinityTerm", + "description": "Required. A pod affinity term, associated with the corresponding weight." + }, + "weight": { + "description": "weight associated with matching the corresponding podAffinityTerm, in the range 1-100.", + "format": "int32", + "type": "integer" + } + }, + "required": [ + "weight", + "podAffinityTerm" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.core.v1.WindowsSecurityContextOptions": { + "description": "WindowsSecurityContextOptions contain Windows-specific options and credentials.", + "properties": { + "gmsaCredentialSpec": { + "description": "GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.", + "type": "string" + }, + "gmsaCredentialSpecName": { + "description": "GMSACredentialSpecName is the name of the GMSA credential spec to use.", + "type": "string" + }, + "hostProcess": { + "description": "HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true.", + "type": "boolean" + }, + "runAsUserName": { + "description": "The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.networking.v1.IPBlock": { + "description": "IPBlock describes a particular CIDR (Ex. \"192.168.1.0/24\",\"2001:db8::/64\") that is allowed to the pods matched by a NetworkPolicySpec's podSelector. The except entry describes CIDRs that should not be included within this rule.", + "properties": { + "cidr": { + "description": "cidr is a string representing the IPBlock Valid examples are \"192.168.1.0/24\" or \"2001:db8::/64\"", + "type": "string" + }, + "except": { + "description": "except is a slice of CIDRs that should not be included within an IPBlock Valid examples are \"192.168.1.0/24\" or \"2001:db8::/64\" Except values will be rejected if they are outside the cidr range", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + } + }, + "required": [ + "cidr" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.networking.v1.NetworkPolicyPeer": { + "description": "NetworkPolicyPeer describes a peer to allow traffic to/from. Only certain combinations of fields are allowed", + "properties": { + "ipBlock": { + "$ref": "#/definitions/io.k8s.api.networking.v1.IPBlock", + "description": "ipBlock defines policy on a particular IPBlock. If this field is set then neither of the other fields can be." + }, + "namespaceSelector": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector", + "description": "namespaceSelector selects namespaces using cluster-scoped labels. This field follows standard label selector semantics; if present but empty, it selects all namespaces.\n\nIf podSelector is also set, then the NetworkPolicyPeer as a whole selects the pods matching podSelector in the namespaces selected by namespaceSelector. Otherwise it selects all pods in the namespaces selected by namespaceSelector." + }, + "podSelector": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector", + "description": "podSelector is a label selector which selects pods. This field follows standard label selector semantics; if present but empty, it selects all pods.\n\nIf namespaceSelector is also set, then the NetworkPolicyPeer as a whole selects the pods matching podSelector in the Namespaces selected by NamespaceSelector. Otherwise it selects the pods matching podSelector in the policy's own namespace." + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.api.networking.v1.NetworkPolicyPort": { + "description": "NetworkPolicyPort describes a port to allow traffic on", + "properties": { + "endPort": { + "description": "endPort indicates that the range of ports from port to endPort if set, inclusive, should be allowed by the policy. This field cannot be defined if the port field is not defined or if the port field is defined as a named (string) port. The endPort must be equal or greater than port.", + "format": "int32", + "type": "integer" + }, + "port": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.util.intstr.IntOrString", + "description": "port represents the port on the given protocol. This can either be a numerical or named port on a pod. If this field is not provided, this matches all port names and numbers. If present, only traffic on the specified protocol AND port will be matched." + }, + "protocol": { + "description": "protocol represents the protocol (TCP, UDP, or SCTP) which traffic must match. If not specified, this field defaults to TCP.", + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.apimachinery.pkg.api.resource.Quantity": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + }, + "io.k8s.apimachinery.pkg.apis.meta.v1.FieldsV1": { + "description": "FieldsV1 stores a set of fields in a data structure like a Trie, in JSON format.\n\nEach key is either a '.' representing the field itself, and will always map to an empty set, or a string representing a sub-field or item. The string will follow one of these four formats: 'f:', where is the name of a field in a struct, or key in a map 'v:', where is the exact json formatted value of a list item 'i:', where is position of a item in a list 'k:', where is a map of a list item's key fields to their unique values If a key maps to an empty Fields value, the field that key represents is part of the set.\n\nThe exact format is defined in sigs.k8s.io/structured-merge-diff", + "type": "object" + }, + "io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector": { + "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", + "properties": { + "matchExpressions": { + "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", + "items": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelectorRequirement" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "matchLabels": { + "additionalProperties": { + "type": "string" + }, + "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", + "type": "object" + } + }, + "type": "object", + "x-kubernetes-map-type": "atomic", + "additionalProperties": false + }, + "io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelectorRequirement": { + "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", + "properties": { + "key": { + "description": "key is the label key that the selector applies to.", + "type": "string" + }, + "operator": { + "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", + "type": "string" + }, + "values": { + "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + } + }, + "required": [ + "key", + "operator" + ], + "type": "object", + "additionalProperties": false + }, + "io.k8s.apimachinery.pkg.apis.meta.v1.ManagedFieldsEntry": { + "description": "ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.", + "properties": { + "apiVersion": { + "description": "APIVersion defines the version of this resource that this field set applies to. The format is \"group/version\" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.", + "type": "string" + }, + "fieldsType": { + "description": "FieldsType is the discriminator for the different fields format and version. There is currently only one possible value: \"FieldsV1\"", + "type": "string" + }, + "fieldsV1": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.FieldsV1", + "description": "FieldsV1 holds the first JSON version format as described in the \"FieldsV1\" type." + }, + "manager": { + "description": "Manager is an identifier of the workflow managing these fields.", + "type": "string" + }, + "operation": { + "description": "Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.", + "type": "string" + }, + "subresource": { + "description": "Subresource is the name of the subresource used to update that object, or empty string if the object was updated through the main resource. The value of this field is used to distinguish between managers, even if they share the same name. For example, a status update will be distinct from a regular update using the same manager name. Note that the APIVersion field is not related to the Subresource field and it always corresponds to the version of the main resource.", + "type": "string" + }, + "time": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Time", + "description": "Time is the timestamp of when the ManagedFields entry was added. The timestamp will also be updated if a field is added, the manager changes any of the owned fields value or removes a field. The timestamp does not update when a field is removed from the entry because another manager took it over." + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta": { + "description": "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", + "properties": { + "annotations": { + "additionalProperties": { + "type": "string" + }, + "description": "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations", + "type": "object" + }, + "creationTimestamp": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Time", + "description": "CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.\n\nPopulated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata" + }, + "deletionGracePeriodSeconds": { + "description": "Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.", + "format": "int64", + "type": "integer" + }, + "deletionTimestamp": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Time", + "description": "DeletionTimestamp is RFC 3339 date and time at which this resource will be deleted. This field is set by the server when a graceful deletion is requested by the user, and is not directly settable by a client. The resource is expected to be deleted (no longer visible from resource lists, and not reachable by name) after the time in this field, once the finalizers list is empty. As long as the finalizers list contains items, deletion is blocked. Once the deletionTimestamp is set, this value may not be unset or be set further into the future, although it may be shortened or the resource may be deleted prior to this time. For example, a user may request that a pod is deleted in 30 seconds. The Kubelet will react by sending a graceful termination signal to the containers in the pod. After that 30 seconds, the Kubelet will send a hard termination signal (SIGKILL) to the container and after cleanup, remove the pod from the API. In the presence of network partitions, this object may still exist after this timestamp, until an administrator or automated process can determine the resource is fully terminated. If not set, graceful deletion of the object has not been requested.\n\nPopulated by the system when a graceful deletion is requested. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata" + }, + "finalizers": { + "description": "Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed. Finalizers may be processed and removed in any order. Order is NOT enforced because it introduces significant risk of stuck finalizers. finalizers is a shared field, any actor with permission can reorder it. If the finalizer list is processed in order, then this can lead to a situation in which the component responsible for the first finalizer in the list is waiting for a signal (field value, external system, or other) produced by a component responsible for a finalizer later in the list, resulting in a deadlock. Without enforced ordering finalizers are free to order amongst themselves and are not vulnerable to ordering changes in the list.", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "set", + "x-kubernetes-patch-strategy": "merge" + }, + "generateName": { + "description": "GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\n\nIf this field is specified and the generated name exists, the server will return a 409.\n\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#idempotency", + "type": "string" + }, + "generation": { + "description": "A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.", + "format": "int64", + "type": "integer" + }, + "labels": { + "additionalProperties": { + "type": "string" + }, + "description": "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels", + "type": "object" + }, + "managedFields": { + "description": "ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like \"ci-cd\". The set of fields is always in the version that the workflow used when modifying the object.", + "items": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ManagedFieldsEntry" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "name": { + "description": "Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#names", + "type": "string" + }, + "namespace": { + "description": "Namespace defines the space within which each name must be unique. An empty namespace is equivalent to the \"default\" namespace, but \"default\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\n\nMust be a DNS_LABEL. Cannot be updated. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces", + "type": "string" + }, + "ownerReferences": { + "description": "List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.", + "items": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.OwnerReference" + }, + "type": "array", + "x-kubernetes-list-map-keys": [ + "uid" + ], + "x-kubernetes-list-type": "map", + "x-kubernetes-patch-merge-key": "uid", + "x-kubernetes-patch-strategy": "merge" + }, + "resourceVersion": { + "description": "An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\n\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency", + "type": "string" + }, + "selfLink": { + "description": "Deprecated: selfLink is a legacy read-only field that is no longer populated by the system.", + "type": "string" + }, + "uid": { + "description": "UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\n\nPopulated by the system. Read-only. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#uids", + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + }, + "io.k8s.apimachinery.pkg.apis.meta.v1.OwnerReference": { + "description": "OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.", + "properties": { + "apiVersion": { + "description": "API version of the referent.", + "type": "string" + }, + "blockOwnerDeletion": { + "description": "If true, AND if the owner has the \"foregroundDeletion\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. See https://kubernetes.io/docs/concepts/architecture/garbage-collection/#foreground-deletion for how the garbage collector interacts with this field and enforces the foreground deletion. Defaults to false. To set this field, a user needs \"delete\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.", + "type": "boolean" + }, + "controller": { + "description": "If true, this reference points to the managing controller.", + "type": "boolean" + }, + "kind": { + "description": "Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + "type": "string" + }, + "name": { + "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#names", + "type": "string" + }, + "uid": { + "description": "UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#uids", + "type": "string" + } + }, + "required": [ + "apiVersion", + "kind", + "name", + "uid" + ], + "type": "object", + "x-kubernetes-map-type": "atomic", + "additionalProperties": false + }, + "io.k8s.apimachinery.pkg.apis.meta.v1.Time": { + "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", + "format": "date-time", + "type": "string" + }, + "io.k8s.apimachinery.pkg.util.intstr.IntOrString": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "logGroomerConfigType": { + "description": "Configuration for log groomer sidecar", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Whether to deploy the Airflow log groomer sidecar.", + "type": "boolean", + "default": true + }, + "command": { + "description": "Command to use when running the Airflow log groomer sidecar (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running the Airflow log groomer sidecar (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "bash", + "/clean-logs" + ] + }, + "retentionDays": { + "description": "Number of days to retain the logs when running the Airflow log groomer sidecar. Total retention time is ``retentionDays`` + ``retentionMinutes``.", + "type": "integer", + "default": 15 + }, + "retentionMinutes": { + "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar. Total retention time is ``retentionDays`` + ``retentionMinutes``.", + "type": "integer", + "default": 0 + }, + "frequencyMinutes": { + "description": "Number of minutes between attempts to groom the Airflow logs in log groomer sidecar.", + "type": "integer", + "default": 15 + }, + "maxSizeBytes": { + "description": "Max size of logs directory in bytes. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled.", + "type": "integer", + "default": 0, + "minimum": 0 + }, + "maxSizePercent": { + "description": "Max size of logs as a percentage of total disk space. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if ``maxSizeBytes`` is set.", + "type": "integer", + "default": 0, + "minimum": 0, + "maximum": 100 + }, + "env": { + "description": "Add additional env vars to log groomer sidecar container (templated).", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar" + }, + "type": "array", + "default": [], + "x-kubernetes-patch-merge-key": "name", + "x-kubernetes-patch-strategy": "merge" + }, + "resources": { + "description": "Resources for Airflow log groomer sidecar.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the log groomer sidecar. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the log groomer sidecar. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the log groomer sidecar.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "persistentVolumeClaimRetentionPolicy": { + "description": "PersistentVolumeClaim retention policy to be used in the lifecycle of a StatefulSet", + "type": [ + "object", + "null" + ], + "default": null, + "additionalProperties": false, + "properties": { + "whenDeleted": { + "description": "Whether to retain the PVC when the StatefulSet is deleted.", + "type": "string", + "default": "Retain" + }, + "whenScaled": { + "description": "Whether to retain the PVC when the StatefulSet is scaled.", + "type": "string", + "default": "Retain" + } + } + } + } +} diff --git a/charts/airflow/values.yaml b/charts/airflow/values.yaml new file mode 100644 index 0000000..aad52cb --- /dev/null +++ b/charts/airflow/values.yaml @@ -0,0 +1,4367 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +# Default values for Airflow. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Provide a name to substitute for the full names of resources +fullnameOverride: "" + +# Provide a name to substitute for the name of the chart +nameOverride: "" + +# Use standard naming for all resources using airflow.fullname template +# Consider removing this later and default it to true +# to make this chart follow standard naming conventions using the fullname template. +# For now this is an opt-in switch for backwards compatibility to leverage the standard naming convention +# and being able to use fully fullnameOverride and nameOverride in all resources +# For new installations - it is recommended to set it to True to follow standard naming conventions +# For existing installations, this will rename and redeploy your resources with the new names. Be aware that +# this will recreate your Deployment/StatefulSets along with their persistent volume claims and data storage +# migration may be needed to keep your old data +useStandardNaming: false + +# Max number of old replicasets to retain. Can be overridden by each Deployment's revisionHistoryLimit +revisionHistoryLimit: ~ + +# User and group of Airflow user +uid: 50000 +gid: 0 + +# Default security context for Airflow (deprecated, use `securityContexts` instead) +securityContext: {} +# runAsUser: 50000 +# fsGroup: 0 +# runAsGroup: 0 + +# Detailed default security context for Airflow Deployments +securityContexts: + pod: {} + containers: {} + +# Global container lifecycle hooks for Airflow containers +containerLifecycleHooks: {} + +# Airflow home directory +# Used for mount paths +airflowHome: /opt/airflow + +# Default Airflow repository -- overridden by all the specific images below +defaultAirflowRepository: apache/airflow + +# Default Airflow tag to deploy +defaultAirflowTag: "3.2.0" + +# Default Airflow digest. If specified, it takes precedence over tag +defaultAirflowDigest: ~ + +# Airflow version (Used to make some decisions based on Airflow Version being deployed) +# Version 2.11.0 and above is supported. +airflowVersion: "3.2.0" + +images: + airflow: + repository: ~ + tag: ~ + # Specifying digest takes precedence over tag. + digest: ~ + pullPolicy: IfNotPresent + # To avoid images with user code, you can turn this to 'true' and + # all the 'run-airflow-migrations' and 'wait-for-airflow-migrations' jobs/containers + # will use the images from 'defaultAirflowRepository:defaultAirflowTag' values + # to run and wait for DB migrations . + useDefaultImageForMigration: false + # timeout (in seconds) for airflow-migrations to complete + migrationsWaitTimeout: 60 + pod_template: + # Note that `images.pod_template.repository` and `images.pod_template.tag` parameters can be overridden + # in `config.kubernetes_executor` section. So for these parameters to have effect + # `config.kubernetes_executor.worker_container_repository` and + # `config.kubernetes_executor.worker_container_tag` must be not set . + repository: ~ + tag: ~ + pullPolicy: IfNotPresent + flower: + repository: ~ + tag: ~ + pullPolicy: IfNotPresent + statsd: + repository: quay.io/prometheus/statsd-exporter + tag: v0.29.0 + pullPolicy: IfNotPresent + redis: + repository: redis + # Redis is limited to 7.2-bookworm due to licencing change + # https://redis.io/blog/redis-adopts-dual-source-available-licensing/ + tag: 7.2-bookworm + pullPolicy: IfNotPresent + pgbouncer: + repository: apache/airflow + tag: airflow-pgbouncer-2025.03.05-1.23.1 + pullPolicy: IfNotPresent + pgbouncerExporter: + repository: apache/airflow + tag: airflow-pgbouncer-exporter-2025.03.05-0.18.0 + pullPolicy: IfNotPresent + gitSync: + repository: registry.k8s.io/git-sync/git-sync + tag: v4.4.2 + pullPolicy: IfNotPresent + +# Select certain nodes for Airflow pods. +nodeSelector: {} +affinity: {} +tolerations: [] +topologySpreadConstraints: [] +schedulerName: ~ + +# Add common labels to all objects and pods defined in this chart. +labels: {} + +# List of existing Kubernetes secrets containing Base64 encoded credentials to connect to private +# registries. Items can be either strings or {name: secret} objects. +imagePullSecrets: [] + +# Ingress configuration +ingress: + # Enable all ingress resources + # (deprecated, use + # `ingress.web.enabled`, + # `ingress.apiServer.enabled` and/or + # `ingress.flower.enabled` + # instead) + enabled: ~ + + # Configs for the Ingress of the API Server (Airflow 3+) + apiServer: + # Enable API Server ingress resource + enabled: false + + # Annotations for the API Server Ingress + annotations: {} + + # The path for the API Server Ingress + path: "/" + + # The pathType for the above path + pathType: "ImplementationSpecific" + + # The hostname for the API Server Ingress (deprecated, use `ingress.apiServer.hosts` instead) + host: "" + + # The hostnames or hosts configuration for the API Server Ingress (templated) + hosts: [] + # - name: "" + # # configs for API Server Ingress TLS + # tls: + # # Enable TLS termination for the API Server Ingress + # enabled: false + # # The name of a pre-created Secret containing a TLS private key and certificate + # secretName: "" + + # The Ingress Class for the API Server Ingress + ingressClassName: "" + + # Configs for API Server Ingress TLS (deprecated, use `ingress.apiServer.hosts[*].tls` instead) + tls: + # Enable TLS termination for the API Server Ingress + enabled: false + # The name of a pre-created Secret containing a TLS private key and certificate + secretName: "" + + # HTTP paths to add to the API Server Ingress before the default path + precedingPaths: [] + + # HTTP paths to add to the API Server Ingress after the default path + succeedingPaths: [] + + # Configs for the Ingress of the web Service (Airflow <3.0.0) + web: + # Enable web ingress resource + enabled: false + + # Annotations for the web Ingress + annotations: {} + + # The path for the web Ingress + path: "/" + + # The pathType for the above path + pathType: "ImplementationSpecific" + + # The hostname for the web Ingress (deprecated, use `ingress.web.hosts` instead) + host: "" + + # The hostnames or hosts configuration for the web Ingress (templated) + hosts: [] + # - name: "" + # # Configs for web Ingress TLS + # tls: + # # Enable TLS termination for the web Ingress + # enabled: false + # # The name of a pre-created Secret containing a TLS private key and certificate + # secretName: "" + + # The Ingress Class for the web Ingress + ingressClassName: "" + + # Configs for web Ingress TLS (deprecated, use `ingress.web.hosts[*].tls` instead) + tls: + # Enable TLS termination for the web Ingress + enabled: false + # The name of a pre-created Secret containing a TLS private key and certificate + secretName: "" + + # HTTP paths to add to the web Ingress before the default path + precedingPaths: [] + + # HTTP paths to add to the web Ingress after the default path + succeedingPaths: [] + + # Configs for the Ingress of the flower Service + flower: + # Enable web ingress resource + enabled: false + + # Annotations for the flower Ingress + annotations: {} + + # The path for the flower Ingress + path: "/" + + # The pathType for the above path + pathType: "ImplementationSpecific" + + # The hostname for the flower Ingress (deprecated, use `ingress.flower.hosts` instead) + host: "" + + # The hostnames or hosts configuration for the flower Ingress (templated) + hosts: [] + # - name: "" + # tls: + # # Enable TLS termination for the flower Ingress + # enabled: false + # # The name of a pre-created Secret containing a TLS private key and certificate + # secretName: "" + + # The Ingress Class for the flower Ingress + ingressClassName: "" + + # Configs for flower Ingress TLS (deprecated, use `ingress.flower.hosts[*].tls` instead) + tls: + # Enable TLS termination for the flower Ingress + enabled: false + # The name of a pre-created Secret containing a TLS private key and certificate + secretName: "" + + # Configs for the Ingress of the StatsD Service + statsd: + # Enable web ingress resource + enabled: false + + # Annotations for the StatsD Ingress + annotations: {} + + # The path for the StatsD Ingress + path: "/metrics" + + # The pathType for the above path + pathType: "ImplementationSpecific" + + # The hostname for the StatsD Ingress (deprecated, use `ingress.statsd.hosts` instead) + host: "" + + # The hostnames or hosts configuration for the StatsD Ingress (templated) + hosts: [] + # - name: "" + # tls: + # # Enable TLS termination for the StatsD Ingress + # enabled: false + # # The name of a pre-created Secret containing a TLS private key and certificate + # secretName: "" + + # The Ingress Class for the StatsD Ingress + ingressClassName: "" + + # Configs for the Ingress of the PgBouncer Service + pgbouncer: + # Enable web ingress resource + enabled: false + + # Annotations for the PgBouncer Ingress + annotations: {} + + # The path for the PgBouncer Ingress + path: "/metrics" + + # The pathType for the above path + pathType: "ImplementationSpecific" + + # The hostname for the PgBouncer Ingress (deprecated, use `ingress.pgbouncer.hosts` instead) + host: "" + + # The hostnames or hosts configuration for the PgBouncer Ingress (templated) + hosts: [] + # - name: "" + # tls: + # # Enable TLS termination for the PgBouncer Ingress + # enabled: false + # # The name of a pre-created Secret containing a TLS private key and certificate + # secretName: "" + + # The Ingress Class for the PgBouncer Ingress + ingressClassName: "" + +# Network policy configuration +networkPolicies: + # Enabled network policies + enabled: false + +# Extra annotations to apply to all Airflow pods (templated) +airflowPodAnnotations: {} + +# Extra annotations to apply to main Airflow ConfigMap +airflowConfigAnnotations: {} + +# 'airflow_local_settings' file as a string (templated) +airflowLocalSettings: |- + {{- if semverCompare "<3.0.0" .Values.airflowVersion }} + {{- if not (or .Values.webserverSecretKey .Values.webserverSecretKeySecretName) }} + from airflow.www.utils import UIAlert + + DASHBOARD_UIALERTS = [ + UIAlert( + 'Usage of a dynamic webserver secret key detected. We recommend a static webserver secret key instead.' + ' See the ' + 'Helm Chart Production Guide for more details.', + category="warning", + roles=["Admin"], + html=True, + ) + ] + {{- end }} + {{- end }} + +# Enable RBAC (default on most clusters these days) +rbac: + # Specifies whether RBAC resources should be created + create: true + createSCCRoleBinding: false + +# Airflow executor +# One or multiple of: LocalExecutor, CeleryExecutor, KubernetesExecutor +# For Airflow <3.0, LocalKubernetesExecutor and CeleryKubernetesExecutor are supported. +# Specify executors in a prioritized list to leverage multiple execution environments as needed: +# https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/executor/index.html#using-multiple-executors-concurrently +executor: "CeleryExecutor" + +# If this is true and using LocalExecutor/KubernetesExecutor/CeleryKubernetesExecutor, the scheduler's +# Service Account will have access to communicate with the api-server and launch pods/jobs. +# If this is true and using CeleryExecutor/KubernetesExecutor/CeleryKubernetesExecutor, the workers +# will be able to launch pods/jobs. +allowPodLaunching: true +allowJobLaunching: false + +# Environment variables for all Airflow containers +env: [] +# - name: "" +# value: "" + +# Volumes for all Airflow containers +volumes: [] + +# VolumeMounts for all Airflow containers +volumeMounts: [] + +# Secrets for all Airflow containers +secret: [] +# - envName: "" +# secretName: "" +# secretKey: "" + +# Enables selected built-in secrets that are set via environment variables by default. +# Those secrets are provided by the Helm Chart secrets by default but in some cases you +# might want to provide some of those variables with _CMD or _SECRET variable, and you should +# in this case disable setting of those variables by setting the relevant configuration to 'false'. +enableBuiltInSecretEnvVars: + AIRFLOW__CORE__FERNET_KEY: true + AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: true + AIRFLOW_CONN_AIRFLOW_DB: true + AIRFLOW__API__SECRET_KEY: true + AIRFLOW__API_AUTH__JWT_SECRET: true + AIRFLOW__WEBSERVER__SECRET_KEY: true + AIRFLOW__CELERY__RESULT_BACKEND: true + AIRFLOW__CELERY__BROKER_URL: true + AIRFLOW__ELASTICSEARCH__HOST: true + AIRFLOW__OPENSEARCH__HOST: true + +# Priority Classes that will be installed by charts. +# Ideally, there should be an entry for dagProcessor, flower, +# pgbouncer, scheduler, statsd, triggerer, webserver/api-server, worker. +# The format for priorityClasses is an array with each element having: +# * name is the name of the priorityClass. Ensure the same name is given to the respective section as well +# * preemptionPolicy for the priorityClass +# * value is the preemption value for the priorityClass +priorityClasses: [] +# - name: class1 (if this is for dagProcessor, ensure overriding `dagProcessor.priorityClass` too) +# preemptionPolicy: PreemptLowerPriority +# value: 10000 +# - name: class2 +# preemptionPolicy: Never +# value: 100000 + +# Extra secrets that will be managed by the chart +# (You can use them with `extraEnv` or `extraEnvFrom` or some of the `extraVolumes` values). +# The format for secret data is "key/value" where +# * key (templated) is the name of the secret that will be created +# * value: an object with the standard 'data' or 'stringData' key (or both). +# The value associated with those keys must be a string (templated) +extraSecrets: {} +# extraSecrets: +# '{{ .Release.Name }}-airflow-connections': +# type: 'Opaque' +# labels: +# my.custom.label/v1: my_custom_label_value_1 +# data: | +# AIRFLOW_CONN_GCP: 'base64_encoded_gcp_conn_string' +# AIRFLOW_CONN_AWS: 'base64_encoded_aws_conn_string' +# stringData: | +# AIRFLOW_CONN_OTHER: 'other_conn' +# '{{ .Release.Name }}-other-secret-name-suffix': +# data: | +# ... +# 'proxy-config': +# stringData: | +# HTTP_PROXY: http://proxy_user:proxy_password@192.168.0.10:2080 +# HTTPS_PROXY: http://proxy_user:proxy_password@192.168.0.10:2080 +# NO_PROXY: "localhost,127.0.0.1,.svc.cluster.local,kubernetes.default.svc" + +# Extra ConfigMaps that will be managed by the chart +# (You can use them with `extraEnv` or `extraEnvFrom` or some of the `extraVolumes` values). +# The format for ConfigMap data is "key/value" where +# * key (templated) is the name of the ConfigMap that will be created +# * value: an object with the standard 'data' key. +# The value associated with this keys must be a string (templated) +extraConfigMaps: {} +# extraConfigMaps: +# '{{ .Release.Name }}-airflow-variables': +# labels: +# my.custom.label/v2: my_custom_label_value_2 +# data: | +# AIRFLOW_VAR_HELLO_MESSAGE: "Hi!" +# AIRFLOW_VAR_KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}" + +# Extra env 'items' that will be added to the definition of Airflow containers +# a string is expected (templated). +# TODO: difference from `env`? This is a templated string. Probably should template `env` and remove this. +extraEnv: ~ +# extraEnv: | +# - name: AIRFLOW__CORE__LOAD_EXAMPLES +# value: 'True' + +# Extra envFrom 'items' that will be added to the definition of Airflow containers +# A string is expected (templated). +extraEnvFrom: ~ +# extraEnvFrom: | +# - secretRef: +# name: '{{ .Release.Name }}-airflow-connections' +# - configMapRef: +# name: '{{ .Release.Name }}-airflow-variables' + +# Airflow database & redis config +data: + # If secret name is provided, secret itself has to be created manually with 'connection' key like: + # + # kind: Secret + # apiVersion: v1 + # metadata: + # name: custom-airflow-metadata-secret + # type: Opaque + # data: + # connection: base64_encoded_connection_string + # + # The 'connection' key is base64-encoded SQLAlchemy connection string, e.g.: + # postgresql+psycopg2://airflow:password@postgres/airflow + metadataSecretName: ~ + + # If not set, falls back to metadataSecretName. The secret must contain 'connection' key which is + # a base64-encoded connection string, e.g.: + # postgresql+psycopg2://user:password@host/db + resultBackendSecretName: ~ + + brokerUrlSecretName: ~ + + # If `metadataSecretName` is not specified, pass connection values below + metadataConnection: + user: postgres + pass: postgres + protocol: postgresql + host: ~ + port: 5432 + db: postgres + sslmode: disable + # Add custom annotations to the metadata connection secret + secretAnnotations: {} + + # `resultBackendConnection` defaults to the same database as metadataConnection + resultBackendConnection: ~ + # or, you can use a different database like: + # resultBackendConnection: + # user: postgres + # pass: postgres + # protocol: postgresql + # host: ~ + # port: 5432 + # db: postgres + # sslmode: disable + + # Add custom annotations to the result backend connection secret + resultBackendConnectionSecretAnnotations: {} + + # Note: `brokerUrl` can only be set during 'helm install', not 'helm upgrade' command + brokerUrl: ~ + + # Add custom annotations to the broker url secret + brokerUrlSecretAnnotations: {} + +# Fernet key settings +# Note: `fernetKey` can only be set during 'helm install', not 'helm upgrade' command +fernetKey: ~ + +# If set, the secret must contain a 'fernet-key' key with a base64-encoded key value +fernetKeySecretName: ~ +# Fernet key secret example: +# kind: Secret +# apiVersion: v1 +# metadata: +# name: custom-fernet-key-secret +# type: Opaque +# data: +# fernet-key: + +# Add custom annotations to the fernet key secret +fernetKeySecretAnnotations: {} + +# Flask secret key for Airflow 3+ Api: '[api] secret_key' in airflow.cfg +apiSecretKey: ~ + +# Add custom annotations to the api secret +apiSecretAnnotations: {} + +# If set, the secret must contain a key 'api-secret-key' with a base64-encoded key value +apiSecretKeySecretName: ~ +# API secret key example: +# kind: Secret +# apiVersion: v1 +# metadata: +# name: custom-api-secret +# type: Opaque +# data: +# api-secret-key: + +# Secret key used to encode and decode JWTs: '[api_auth] jwt_secret' in airflow.cfg +# Note: It is not advised to use in production as during helm upgrade it will be changed +# which can cause dag failures during component rollouts +jwtSecret: ~ + +# Add custom annotations to the JWT secret +jwtSecretAnnotations: {} + +# If set, the secret must contain a key 'jwt-secret' with a base64-encoded key value +jwtSecretName: ~ +# JWT secret example: +# kind: Secret +# apiVersion: v1 +# metadata: +# name: custom-jwt-secret +# type: Opaque +# data: +# jwt-secret: + +# Flask secret key for Airflow <3 Webserver: '[webserver] secret_key' in airflow.cfg +# (deprecated, use `apiSecretKey` instead (Airflow 3+)) +webserverSecretKey: ~ + +# Add custom annotations to the webserver secret +# (deprecated, use `apiSecretAnnotations` instead (Airflow 3+)) +webserverSecretAnnotations: {} + +# If set, the secret must contain a key 'webserver-secret-key' with a base64-encoded key value +# (deprecated, use `apiSecretKeySecretName` instead (Airflow 3+)) +webserverSecretKeySecretName: ~ +# Webserver secret key secret example: +# kind: Secret +# apiVersion: v1 +# metadata: +# name: custom-webserver-secret +# type: Opaque +# data: +# webserver-secret-key: + +# In order to use kerberos you need to create secret containing the keytab file. +# The secret name should follow naming convention of the application where resources are +# name '{{ .Release.Name }}-'. In case of the keytab file, the '' is "kerberos-keytab". +# If your release is named "my-release" the name of the secret should be "my-release-kerberos-keytab". +# +# The Keytab content should be available in the "kerberos.keytab" key of the secret. +# apiVersion: v1 +# kind: Secret +# data: +# kerberos.keytab: +# type: Opaque +# +# If you have keytab file you can do it with similar: +# kubectl create secret generic {{ .Release.Name }}-kerberos-keytab --from-file=kerberos.keytab +# +# Alternatively, instead of manually creating the secret, it is possible to specify +# `kerberos.keytabBase64Content` parameter. This parameter should contain base64 encoded keytab. +kerberos: + enabled: false + ccacheMountPath: /var/kerberos-ccache + ccacheFileName: cache + configPath: /etc/krb5.conf + keytabBase64Content: ~ + keytabPath: /etc/airflow.keytab + principal: airflow@FOO.COM + reinitFrequency: 3600 + config: | + # This is an example config showing how you can use templating and how "example" config + # might look like. It works with the test kerberos server that we are using during integration + # testing at Apache Airflow (see 'scripts/ci/docker-compose/integration-kerberos.yml' but in + # order to make it production-ready you must replace it with your own configuration that + # Matches your kerberos deployment. Administrators of your Kerberos instance should + # provide the right configuration. + + [logging] + default = "FILE:{{ template "airflow_logs_no_quote" . }}/kerberos_libs.log" + kdc = "FILE:{{ template "airflow_logs_no_quote" . }}/kerberos_kdc.log" + admin_server = "FILE:{{ template "airflow_logs_no_quote" . }}/kadmind.log" + + [libdefaults] + default_realm = FOO.COM + ticket_lifetime = 10h + renew_lifetime = 7d + forwardable = true + + [realms] + FOO.COM = { + kdc = kdc-server.foo.com + admin_server = admin_server.foo.com + } + +# Airflow Worker Config +workers: + # Number of Airflow Celery workers (deprecated, use `workers.celery.replicas` instead) + replicas: 1 + + # Max number of old Airflow Celery workers ReplicaSets to retain + # (deprecated, use `workers.celery.revisionHistoryLimit` instead) + revisionHistoryLimit: ~ + + # Command to use when running Airflow Celery workers and using pod-template-file (templated) + # (deprecated, use `workers.celery.command` and/or `workers.kubernetes.command` instead) + command: ~ + + # Args to use when running Airflow Celery workers (templated) + # (deprecated, use `workers.celery.args` instead) + args: + - "bash" + - "-c" + # The format below is necessary to get `helm lint` happy + - |- + exec \ + airflow celery worker + {{- if and .Values.workers.queue (ne .Values.workers.queue "default") }} + {{- " -q " }}{{ .Values.workers.queue }} + {{- end }} + + # If the Airflow Celery worker stops responding for 5 minutes (5*60s) + # kill the worker and let Kubernetes restart it + # (deprecated, use `workers.celery.livenessProbe` section instead) + livenessProbe: + # (deprecated, use `workers.celery.livenessProbe.enabled` instead) + enabled: true + # (deprecated, use `workers.celery.livenessProbe.initialDelaySeconds` instead) + initialDelaySeconds: 10 + # (deprecated, use `workers.celery.livenessProbe.timeoutSeconds` instead) + timeoutSeconds: 20 + # (deprecated, use `workers.celery.livenessProbe.failureThreshold` instead) + failureThreshold: 5 + # (deprecated, use `workers.celery.livenessProbe.periodSeconds` instead) + periodSeconds: 60 + # (deprecated, use `workers.celery.livenessProbe.command` instead) + command: ~ + + # Update Strategy when Airflow Celery worker is deployed as a StatefulSet + # (deprecated, use `workers.celery.updateStrategy` instead) + updateStrategy: ~ + # Update Strategy when Airflow Celery worker is deployed as a Deployment + # (deprecated, use `workers.celery.strategy` instead) + strategy: + rollingUpdate: + maxSurge: "100%" + maxUnavailable: "50%" + + # Allow relaxing ordering guarantees for Airflow Celery worker while preserving its uniqueness and identity + # (deprecated, use `workers.celery.podManagementPolicy` instead) + # podManagementPolicy: Parallel + + # When not set, the values defined in the global securityContext will + # be used in Airflow Celery workers and pod-template-file + # (deprecated, use `workers.celery.securityContexts` and/or `workers.kubernetes.securityContexts` instead) + securityContext: {} + # runAsUser: 50000 + # fsGroup: 0 + # runAsGroup: 0 + + # Detailed default security context for the + # Airflow Celery workers and pod-template-file on container and pod level + # (deprecated, use `workers.celery.securityContexts` and/or `workers.kubernetes.securityContexts` instead) + securityContexts: + # (deprecated, use + # `workers.celery.securityContexts.pod` and/or + # `workers.kubernetes.securityContexts.pod` + # instead) + pod: {} + # (deprecated, use + # `workers.celery.securityContexts.container` and/or + # `workers.kubernetes.securityContexts.container` + # instead) + container: {} + + # Container level Lifecycle Hooks definition for + # Airflow Celery workers and pods created with pod-template-file + # (deprecated, use + # `workers.celery.containerLifecycleHooks` and/or + # `workers.kubernetes.containerLifecycleHooks` + # instead) + containerLifecycleHooks: {} + + # Airflow Celery workers pod disruption budget + # (deprecated, use `workers.celery.podDisruptionBudget` instead) + podDisruptionBudget: + # (deprecated, use `workers.celery.podDisruptionBudget.enabled` instead) + enabled: false + + # PDB configuration (`minAvailable` and `maxUnavailable` are mutually exclusive) + # (deprecated, use `workers.celery.podDisruptionBudget.config` instead) + config: + # (deprecated, use `workers.celery.podDisruptionBudget.config.maxUnavailable` instead) + maxUnavailable: 1 + + # (deprecated, use `workers.celery.podDisruptionBudget.config.minAvailable` instead) + # minAvailable: 1 + + # Create Service Account for Airflow Celery workers and pods created with pod-template-file + # (deprecated, use `workers.celery.serviceAccount` and/or `workers.kubernetes.serviceAccount` instead) + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + # (deprecated, use + # `workers.celery.serviceAccount.automountServiceAccountToken` and/or + # `workers.kubernetes.serviceAccount.automountServiceAccountToken` + # instead) + automountServiceAccountToken: true + + # Specifies whether a Service Account should be created + # (deprecated, use + # `workers.celery.serviceAccount.create` and/or + # `workers.kubernetes.serviceAccount.create` + # instead) + create: true + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + # (deprecated, use + # `workers.celery.serviceAccount.name` and/or + # `workers.kubernetes.serviceAccount.name` + # instead) + name: ~ + + # Annotations to add to worker Kubernetes Service Account. + # (deprecated, use + # `workers.celery.serviceAccount.annotations` and/or + # `workers.kubernetes.serviceAccount.annotations` + # instead) + annotations: {} + + # Allow KEDA autoscaling for Airflow Celery workers + # (deprecated, use `workers.celery.keda` instead) + keda: + # (deprecated, use `workers.celery.keda.enabled` instead) + enabled: false + + # (deprecated, use `workers.celery.keda.namespaceLabels` instead) + namespaceLabels: {} + + # How often KEDA polls the Airflow DB to report new scale requests to the HPA + # (deprecated, use `workers.celery.keda.pollingInterval` instead) + pollingInterval: 5 + + # How many seconds KEDA will wait before scaling to zero. + # Note: HPA has a separate cooldown period for scale-downs + # (deprecated, use `workers.celery.keda.cooldownPeriod` instead) + cooldownPeriod: 30 + + # Minimum number of Airflow Celery workers created by keda + # (deprecated, use `workers.celery.keda.minReplicaCount` instead) + minReplicaCount: 0 + + # Maximum number of Airflow Celery workers created by keda + # (deprecated, use `workers.celery.keda.maxReplicaCount` instead) + maxReplicaCount: 10 + + # Specify HPA related options + # (deprecated, use `workers.celery.keda.advanced` instead) + advanced: {} + # horizontalPodAutoscalerConfig: + # behavior: + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Percent + # value: 100 + # periodSeconds: 15 + + # Query to use for KEDA autoscaling. Must return a single integer. + # (deprecated, use `workers.celery.keda.query` instead) + query: >- + SELECT ceil(COUNT(*)::decimal / {{ .Values.config.celery.worker_concurrency }}) + FROM task_instance + WHERE (state='running' OR state='queued') + AND queue IN ( + {{- range $i, $q := splitList "," .Values.workers.queue -}} + {{- if $i }},{{ end }}'{{ $q | trim }}' + {{- end -}} + ) + {{- if contains "CeleryKubernetesExecutor" .Values.executor }} + AND queue != '{{ .Values.config.celery_kubernetes_executor.kubernetes_queue }}' + {{- else if contains "KubernetesExecutor" .Values.executor }} + AND executor IS DISTINCT FROM 'KubernetesExecutor' + {{- else if contains "airflow.providers.edge3.executors.EdgeExecutor" .Values.executor }} + AND executor IS DISTINCT FROM 'EdgeExecutor' + {{- end }} + + # Weather to use PGBouncer to connect to the database or not when it is enabled + # This configuration will be ignored if PGBouncer is not enabled + # (deprecated, use `workers.celery.keda.usePgbouncer` instead) + usePgbouncer: true + + # Allow HPA for Airflow Celery workers (KEDA must be disabled) + # (deprecated, use `workers.celery.hpa` instead) + hpa: + # (deprecated, use `workers.celery.hpa.enabled` instead) + enabled: false + + # Minimum number of Airflow Celery workers created by HPA + # (deprecated, use `workers.celery.hpa.minReplicaCount` instead) + minReplicaCount: 0 + + # Maximum number of Airflow Celery workers created by HPA + # (deprecated, use `workers.celery.hpa.maxReplicaCount` instead) + maxReplicaCount: 5 + + # Specifications for which to use to calculate the desired replica count + # (deprecated, use `workers.celery.hpa.metrics` instead) + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + + # Scaling behavior of the target in both Up and Down directions + # (deprecated, use `workers.celery.hpa.behavior` instead) + behavior: {} + + # Persistence volume configuration for Airflow Celery workers + # (deprecated, use `workers.celery.persistence` instead) + persistence: + # Enable persistent volumes (deprecated, use `workers.celery.persistence.enabled` instead) + enabled: true + + # This policy determines whether PVCs should be deleted when StatefulSet is scaled down or removed + # (deprecated, use `workers.celery.persistence.persistentVolumeClaimRetentionPolicy` instead) + persistentVolumeClaimRetentionPolicy: ~ + # persistentVolumeClaimRetentionPolicy: + # whenDeleted: Delete + # whenScaled: Delete + + # Volume size for Airflow Celery worker StatefulSet + # (deprecated, use `workers.celery.persistence.size` instead) + size: 100Gi + + # If using a custom storageClass, pass name ref to all StatefulSets here + # (deprecated, use `workers.celery.persistence.storageClassName` instead) + storageClassName: + + # Execute init container to chown log directory. + # This is currently only needed in kind, due to usage + # of local-path provisioner. + # (deprecated, use `workers.celery.persistence.fixPermissions` instead) + fixPermissions: false + + # Annotations to add to Airflow Celery worker volumes + # (deprecated, use `workers.celery.persistence.annotations` instead) + annotations: {} + + # Detailed default security context for persistence on container level + # (deprecated, use `workers.celery.persistence.securityContexts` instead) + securityContexts: + # (deprecated, use `workers.celery.persistence.securityContexts.container` instead) + container: {} + + # Kerberos sidecar configuration for Airflow Celery workers and pods created with pod-template-file + # (deprecated, use `workers.celery.kerberosSidecar` and/or `workers.kubernetes.kerberosSidecar` instead) + kerberosSidecar: + # Enable kerberos sidecar + # (deprecated, use + # `workers.celery.kerberosSidecar.enabled` and/or + # `workers.kubernetes.kerberosSidecar.enabled` + # instead) + enabled: false + + # (deprecated, use + # `workers.celery.kerberosSidecar.resources` and/or + # `workers.kubernetes.kerberosSidecar.resources` + # instead) + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Detailed default security context for kerberos sidecar on container level + # (deprecated, use + # `workers.celery.kerberosSidecar.securityContexts` and/or + # `workers.kubernetes.kerberosSidecar.securityContexts` + # instead) + securityContexts: + # (deprecated, use + # `workers.celery.kerberosSidecar.securityContexts.container` and/or + # `workers.kubernetes.kerberosSidecar.securityContexts.container` + # instead) + container: {} + + # Container level lifecycle hooks + # (deprecated, use + # `workers.celery.kerberosSidecar.containerLifecycleHooks` and/or + # `workers.kubernetes.kerberosSidecar.containerLifecycleHooks` + # instead) + containerLifecycleHooks: {} + + # Kerberos init container configuration for Airflow Celery workers and pods created with pod-template-file + # (deprecated, use + # `workers.celery.kerberosInitContainer` and/or + # `workers.kubernetes.kerberosInitContainer` + # instead) + kerberosInitContainer: + # Enable kerberos init container + # (deprecated, use + # `workers.celery.kerberosInitContainer.enabled` and/or + # `workers.kubernetes.kerberosInitContainer.enabled` + # instead) + enabled: false + + # (deprecated, use + # `workers.celery.kerberosInitContainer.resources` and/or + # `workers.kubernetes.kerberosInitContainer.resources` + # instead) + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Detailed default security context for kerberos init container + # (deprecated, use + # `workers.celery.kerberosInitContainer.securityContexts` and/or + # `workers.kubernetes.kerberosInitContainer.securityContexts` + # instead) + securityContexts: + # (deprecated, use + # `workers.celery.kerberosInitContainer.securityContexts.container` and/or + # `workers.kubernetes.kerberosInitContainer.securityContexts.container` + # instead) + container: {} + + # Container level lifecycle hooks + # (deprecated, use + # `workers.celery.kerberosInitContainer.containerLifecycleHooks` and/or + # `workers.kubernetes.kerberosInitContainer.containerLifecycleHooks` + # instead) + containerLifecycleHooks: {} + + # Resource configuration for Airflow Celery workers and pods created with pod-template-file + # (deprecated, use `workers.celery.resources` and/or `workers.kubernetes.resources` instead) + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Grace period for tasks to finish after SIGTERM is sent from Kubernetes. + # It is used by Airflow Celery workers and pod-template-file. + # (deprecated, use + # `workers.celery.terminationGracePeriodSeconds` and/or + # `workers.kubernetes.terminationGracePeriodSeconds` + # instead) + terminationGracePeriodSeconds: 600 + + # This setting tells Kubernetes that its ok to evict when it wants to scale a node down. + # It is used by Airflow Celery workers and pod-template-file. + # (deprecated, use + # `workers.celery.safeToEvict` and/or + # `workers.kubernetes.safeToEvict` + # instead) + safeToEvict: false + + # Launch additional containers into Airflow Celery worker + # and pods created with pod-template-file (templated). + # (deprecated, use + # `workers.celery.extraContainers` and/or + # `workers.kubernetes.extraContainers` + # instead) + # Note: If used with KubernetesExecutor, you are responsible for signaling sidecars to exit when the main + # container finishes so Airflow can continue the worker shutdown process! + extraContainers: [] + + # Add additional init containers into Airflow Celery workers + # and pods created with pod-template-file (templated). + # (deprecated, use + # `workers.celery.extraInitContainers` and/or + # `workers.kubernetes.extraInitContainers` + # instead) + extraInitContainers: [] + + # Additional volumes attached to the Airflow Celery workers + # and pods created with pod-template-file + # (deprecated, use `workers.celery.extraVolumes` and/or `workers.kubernetes.extraVolumes` instead) + extraVolumes: [] + # Mount additional volumes into workers pods. It can be templated like in the following example: + # extraVolumes: + # - name: my-templated-extra-volume + # secret: + # secretName: '{{ include "my_secret_template" . }}' + # defaultMode: 0640 + # optional: true + + # Additional volume mounts attached to the Airflow Celery workers + # and pods created with pod-template-file + # (deprecated, use + # `workers.celery.extraVolumeMounts` and/or + # `workers.kubernetes.extraVolumeMounts` + # instead) + extraVolumeMounts: [] + # Mount additional volumes into workers pods. It can be templated like in the following example: + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + + # Expose additional ports of Airflow Celery workers. These can be used for additional metric collection. + # (deprecated, use `workers.celery.extraPorts` instead) + extraPorts: [] + + # Select certain nodes for Airflow Celery worker pods and pods created with pod-template-file + # (deprecated, use `workers.celery.nodeSelector` and/or `workers.kubernetes.nodeSelector` instead) + nodeSelector: {} + + # (deprecated, use `workers.celery.runtimeClassName` and/or `workers.kubernetes.runtimeClassName` instead) + runtimeClassName: ~ + + # (deprecated, use `workers.celery.priorityClassName` and/or `workers.kubernetes.priorityClassName` instead) + priorityClassName: ~ + + # (deprecated, use `workers.celery.affinity` and/or `workers.kubernetes.affinity` instead) + affinity: {} + # Default Airflow Celery worker affinity is: + # podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - podAffinityTerm: + # labelSelector: + # matchLabels: + # component: worker + # topologyKey: kubernetes.io/hostname + # weight: 100 + + # (deprecated, use `workers.celery.tolerations` and/or `workers.kubernetes.tolerations` instead) + tolerations: [] + + # (deprecated, use + # `workers.celery.topologySpreadConstraints` and/or + # `workers.kubernetes.topologySpreadConstraints` + # instead) + topologySpreadConstraints: [] + + # hostAliases to use in Airflow Celery worker pods and pods created with pod-template-file + # (deprecated, use `workers.celery.hostAliases` and/or `workers.kubernetes.hostAliases` instead) + # See: + # https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ + hostAliases: [] + # - ip: "127.0.0.2" + # hostnames: + # - "test.hostname.one" + # - ip: "127.0.0.3" + # hostnames: + # - "test.hostname.two" + + # Annotations for the Airflow Celery worker resource + # (deprecated, use `workers.celery.annotations` instead) + annotations: {} + + # Pod annotations for the Airflow Celery workers and pods created with pod-template-file (templated) + # (deprecated, use `workers.celery.podAnnotations` and/or `workers.kubernetes.podAnnotations` instead) + podAnnotations: {} + + # Labels specific to Airflow Celery workers objects and pods created with pod-template-file + # (deprecated, use `workers.celery.labels` and/or `workers.kubernetes.labels` instead) + labels: {} + + # Log groomer configuration for Airflow Celery workers + # (deprecated, use `workers.celery.logGroomerSidecar` instead) + logGroomerSidecar: + # Whether to deploy the Airflow Celery worker log groomer sidecar + # (deprecated, use `workers.celery.logGroomerSidecar.enabled` instead) + enabled: true + + # Command to use when running the Airflow Celery worker log groomer sidecar (templated) + # (deprecated, use `workers.celery.logGroomerSidecar.command` instead) + command: ~ + + # Args to use when running the Airflow Celery worker log groomer sidecar (templated) + # (deprecated, use `workers.celery.logGroomerSidecar.args` instead) + args: ["bash", "/clean-logs"] + + # Number of days to retain logs + # (deprecated, use `workers.celery.logGroomerSidecar.retentionDays` instead) + retentionDays: 15 + + # Number of minutes to retain logs. + # This can be used for finer granularity than days. + # Total retention is `retentionDays` + `retentionMinutes`. + # (deprecated, use `workers.celery.logGroomerSidecar.retentionMinutes` instead) + retentionMinutes: 0 + + # Frequency to attempt to groom logs (in minutes) + # (deprecated, use `workers.celery.logGroomerSidecar.frequencyMinutes` instead) + frequencyMinutes: 15 + + # Max size of logs in bytes. 0 = disabled + # (deprecated, use `workers.celery.logGroomerSidecar.maxSizeBytes` instead) + maxSizeBytes: 0 + + # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if `maxSizeBytes` is set. + # (deprecated, use `workers.celery.logGroomerSidecar.maxSizePercent` instead) + maxSizePercent: 0 + + # (deprecated, use `workers.celery.logGroomerSidecar.resources` instead) + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Detailed default security context for `logGroomerSidecar` for container level + # (deprecated, use `workers.celery.logGroomerSidecar.securityContexts` instead) + securityContexts: + # (deprecated, use `workers.celery.logGroomerSidecar.securityContexts.container` instead) + container: {} + + # (deprecated, use `workers.celery.logGroomerSidecar.env` instead) + env: [] + + # Container level lifecycle hooks + # (deprecated, use `workers.celery.logGroomerSidecar.containerLifecycleHooks` instead) + containerLifecycleHooks: {} + + # Configuration of wait-for-airflow-migration init container for Airflow Celery workers + # (deprecated, use `workers.celery.waitForMigrations` instead) + waitForMigrations: + # Whether to create init container to wait for db migrations + # (deprecated, use `workers.celery.waitForMigrations.enabled` instead) + enabled: true + + # (deprecated, use `workers.celery.waitForMigrations.env` instead) + env: [] + + # Detailed default security context for wait-for-airflow-migrations container + # (deprecated, use `workers.celery.waitForMigrations.securityContexts` instead) + securityContexts: + # (deprecated, use `workers.celery.waitForMigrations.securityContexts.container` instead) + container: {} + + # Additional env variable configuration for Airflow Celery workers and pods created with pod-template-file + # (deprecated, use `workers.celery.env` and/or `workers.kubernetes.env` instead) + env: [] + + # Additional volume claim templates for Airflow Celery workers. + # Requires mounting of specified volumes under extraVolumeMounts. + # (deprecated, use `workers.celery.volumeClaimTemplates` instead) + volumeClaimTemplates: [] + # Volume Claim Templates example: + # volumeClaimTemplates: + # - metadata: + # name: data-volume-1 + # spec: + # storageClassName: "storage-class-1" + # accessModes: + # - "ReadWriteOnce" + # resources: + # requests: + # storage: "10Gi" + # - metadata: + # name: data-volume-2 + # spec: + # storageClassName: "storage-class-2" + # accessModes: + # - "ReadWriteOnce" + # resources: + # requests: + # storage: "20Gi" + + # (deprecated, use `workers.celery.schedulerName` and/or `workers.kubernetes.schedulerName` instead) + schedulerName: ~ + + celery: + # Number of Airflow Celery workers + replicas: ~ + + # Max number of old Airflow Celery workers ReplicaSets to retain + revisionHistoryLimit: ~ + + # Command to use when running Airflow Celery workers (templated) + command: ~ + + # Args to use when running Airflow Celery workers (templated) + args: ~ + + # If the Airflow Celery worker stops responding for 5 minutes (5*60s) + # kill the worker and let Kubernetes restart it + livenessProbe: + enabled: ~ + initialDelaySeconds: ~ + timeoutSeconds: ~ + failureThreshold: ~ + periodSeconds: ~ + command: ~ + + # Enable the default workers defined by the root `workers` and `workers.celery` + # configurations to be created. + # If false, only dedicated workers defined in 'sets' will be created. + enableDefault: true + + # Queue name for the default workers + queue: "default" + + # List of worker sets. Each item can overwrite values from the parent `workers` and `workers.celery` + # section. + sets: [] + # sets: + # - name: highcpu + # replicas: 2 + # queue: "highcpu" + # resources: + # requests: + # memory: "2Gi" + # cpu: "4000m" + # limits: + # memory: "4Gi" + # cpu: "8000m" + # - name: highmem + # replicas: 2 + # queue: "highmem" + # resources: + # requests: + # memory: "4Gi" + # cpu: "2000m" + # limits: + # memory: "8Gi" + # cpu: "4000m" + + # Update Strategy when Airflow Celery worker is deployed as a StatefulSet + updateStrategy: ~ + # Update Strategy when Airflow Celery worker is deployed as a Deployment + strategy: ~ + + # Allow relaxing ordering guarantees for Airflow Celery worker + # while preserving its uniqueness and identity + # podManagementPolicy: Parallel + + # Detailed default security context for Airflow Celery workers for container and pod level + # If not set, the values from `workers.securityContexts` section will be used. + securityContexts: + pod: {} + container: {} + + # Container level Lifecycle Hooks definition for Airflow Celery workers + containerLifecycleHooks: {} + + # Airflow Celery workers pod disruption budget + podDisruptionBudget: + enabled: ~ + + # PDB configuration (`minAvailable` and `maxUnavailable` are mutually exclusive) + config: + maxUnavailable: ~ + # minAvailable: ~ + + # Create Service Account for Airflow Celery workers + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + automountServiceAccountToken: ~ + + # Specifies whether a Service Account should be created + create: ~ + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + name: ~ + + # Annotations to add to worker Kubernetes Service Account. + annotations: {} + + # Allow KEDA autoscaling for Airflow Celery workers + keda: + enabled: ~ + + namespaceLabels: {} + + # How often KEDA polls the airflow DB to report new scale requests to the HPA + pollingInterval: ~ + + # How many seconds KEDA will wait before scaling to zero. + # Note: HPA has a separate cooldown period for scale-downs + cooldownPeriod: ~ + + # Minimum number of Airflow Celery workers created by KEDA + minReplicaCount: ~ + + # Maximum number of Airflow Celery workers created by KEDA + maxReplicaCount: ~ + + # Specify HPA related options + advanced: {} + # horizontalPodAutoscalerConfig: + # behavior: + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Percent + # value: 100 + # periodSeconds: 15 + + # Query to use for KEDA autoscaling. Must return a single integer + query: ~ + + # Weather to use PGBouncer to connect to the database or not when it is enabled + # This configuration will be ignored if PGBouncer is not enabled + usePgbouncer: ~ + + # Allow HPA for Airflow Celery workers (KEDA must be disabled) + hpa: + enabled: ~ + + # Minimum number of Airflow Celery workers created by HPA + minReplicaCount: ~ + + # Maximum number of Airflow Celery workers created by HPA + maxReplicaCount: ~ + + # Specifications for which to use to calculate the desired replica count + metrics: ~ + + # Scaling behavior of the target in both Up and Down directions + behavior: {} + + # Persistence volume configuration for Airflow Celery workers + persistence: + # Enable persistent volumes + enabled: ~ + + # This policy determines whether PVCs should be deleted when StatefulSet is scaled down or removed + persistentVolumeClaimRetentionPolicy: ~ + # persistentVolumeClaimRetentionPolicy: + # whenDeleted: Delete + # whenScaled: Delete + + # Volume size for Airflow Celery worker StatefulSet + size: ~ + + # If using a custom storageClass, pass name ref to all StatefulSets here + storageClassName: + + # Execute init container to chown log directory. + # This is currently only needed in kind, due to usage + # of local-path provisioner. + fixPermissions: ~ + + # Annotations to add to Airflow Celery worker volumes + annotations: {} + + # Detailed default security context for persistence on container level + securityContexts: + container: {} + + # Kerberos sidecar configuration for Airflow Celery workers + kerberosSidecar: + # Enable kerberos sidecar + enabled: ~ + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Detailed default security context for kerberos sidecar on container level + securityContexts: + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + # Kerberos init container configuration for Airflow Celery workers + # If not set, the values from `workers.kerberosInitContainer` section will be used. + kerberosInitContainer: + # Enable kerberos init container + # If `workers.kerberosInitContainer.enabled` is set to True, this flag has no effect + enabled: ~ + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Detailed default security context for kerberos init container + securityContexts: + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + # Resource configuration for Airflow Celery workers + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Grace period for tasks to finish after SIGTERM is sent from Kubernetes + terminationGracePeriodSeconds: ~ + + # This setting tells Kubernetes that its ok to evict when it wants to scale a node down + safeToEvict: ~ + + # Launch additional containers into Airflow Celery worker (templated) + extraContainers: [] + + # Add additional init containers into Airflow Celery workers (templated) + extraInitContainers: [] + + # Additional volumes attached to the Airflow Celery workers + extraVolumes: [] + # Mount additional volumes into workers pods. It can be templated like in the following example: + # extraVolumes: + # - name: my-templated-extra-volume + # secret: + # secretName: '{{ include "my_secret_template" . }}' + # defaultMode: 0640 + # optional: true + + # Additional volume mounts attached to the Airflow Celery workers + extraVolumeMounts: [] + # Mount additional volumes into workers pods. It can be templated like in the following example: + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + + # Expose additional ports of Airflow Celery workers. These can be used for additional metric collection. + extraPorts: [] + + # Select certain nodes for Airflow Celery worker pods + nodeSelector: {} + + runtimeClassName: ~ + + priorityClassName: ~ + + affinity: {} + # Default Airflow Celery worker affinity is: + # podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - podAffinityTerm: + # labelSelector: + # matchLabels: + # component: worker + # topologyKey: kubernetes.io/hostname + # weight: 100 + + tolerations: [] + + topologySpreadConstraints: [] + + # hostAliases to use in Airflow Celery worker pods + # See: + # https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ + hostAliases: [] + # - ip: "127.0.0.2" + # hostnames: + # - "test.hostname.one" + # - ip: "127.0.0.3" + # hostnames: + # - "test.hostname.two" + + # Annotations for the Airflow Celery worker resource + annotations: {} + + # Pod annotations for the Airflow Celery workers (templated) + podAnnotations: {} + + # Labels specific to Airflow Celery workers objects + labels: {} + + # Log groomer configuration for Airflow Celery workers + logGroomerSidecar: + # Whether to deploy the Airflow Celery worker log groomer sidecar + enabled: ~ + + # Command to use when running the Airflow Celery worker log groomer sidecar (templated) + command: ~ + + # Args to use when running the Airflow Celery worker log groomer sidecar (templated) + args: [] + + # Number of days to retain logs + retentionDays: ~ + + # Number of minutes to retain logs. + # This can be used for finer granularity than days. + # Total retention is `retentionDays` + `retentionMinutes`. + retentionMinutes: ~ + + # Frequency to attempt to groom logs (in minutes) + frequencyMinutes: ~ + + # Max size of logs in bytes. 0 = disabled + maxSizeBytes: ~ + + # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if `maxSizeBytes` is set. + maxSizePercent: ~ + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Detailed default security context for `logGroomerSidecar` for container level + securityContexts: + container: {} + + env: [] + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + # Configuration of wait-for-airflow-migration init container for Airflow Celery workers + waitForMigrations: + # Whether to create init container to wait for db migrations + enabled: ~ + + env: [] + + # Detailed default security context for wait-for-airflow-migrations container + securityContexts: + container: {} + + # Additional env variable configuration for Airflow Celery workers + env: [] + + # Additional volume claim templates for Airflow Celery workers. + # Requires mounting of specified volumes under extraVolumeMounts. + volumeClaimTemplates: [] + # Volume Claim Templates example: + # volumeClaimTemplates: + # - metadata: + # name: data-volume-1 + # spec: + # storageClassName: "storage-class-1" + # accessModes: + # - "ReadWriteOnce" + # resources: + # requests: + # storage: "10Gi" + # - metadata: + # name: data-volume-2 + # spec: + # storageClassName: "storage-class-2" + # accessModes: + # - "ReadWriteOnce" + # resources: + # requests: + # storage: "20Gi" + + schedulerName: ~ + + kubernetes: + # Command to use in pod-template-file (templated) + command: ~ + + # Detailed default security context for pod-template-file for container and pod level + # If not set, the values from `workers.securityContexts` section will be used. + securityContexts: + pod: {} + container: {} + + # Container level Lifecycle Hooks definition for pods created with pod-template-file + containerLifecycleHooks: {} + + # Create Service Account for pods created with pod-template-file + # When this section is specified, the Service Account is created from + # 'templates/workers/worker-kubernetes-serviceaccount.yaml' file + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + # If not specified, the `workers.serviceAccount.automountServiceAccountToken` value will be taken + automountServiceAccountToken: ~ + + # Specifies whether a Service Account should be created. + # If not specified, the Service Account will be generated and used from + # 'templates/workers/worker-serviceaccount.yaml' file if `workers.serviceAccount.create` + # will be 'true' + create: ~ + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + # with Kubernetes dedicated name + name: ~ + + # Annotations to add to worker Kubernetes Service Account. + # If not specified, the `workers.serviceAccount.annotations` value will be taken + annotations: {} + + # Kerberos sidecar configuration for pods created with pod-template-file + kerberosSidecar: + # Enable kerberos sidecar + enabled: ~ + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Detailed default security context for kerberos sidecar on container level + securityContexts: + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + # Kerberos init container configuration for pods created with pod-template-file + # If not set, the values from `workers.kerberosInitContainer` section will be used. + kerberosInitContainer: + # Enable kerberos init container + # If `workers.kerberosInitContainer.enabled` is set to True, this flag has no effect + enabled: ~ + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Detailed default security context for kerberos init container + securityContexts: + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + # Resource configuration for pods created with pod-template-file + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Grace period for tasks to finish after SIGTERM is sent from Kubernetes + terminationGracePeriodSeconds: ~ + + # This setting tells Kubernetes that its ok to evict when it wants to scale a node down + safeToEvict: ~ + + # Launch additional containers into pods created with pod-template-file (templated). + # Note: You are responsible for signaling sidecars to exit when the main + # container finishes so Airflow can continue the worker shutdown process! + extraContainers: [] + + # Add additional init containers into pods created with pod-template-file (templated) + extraInitContainers: [] + + # Additional volumes attached to the pods created with pod-template-file + extraVolumes: [] + # Mount additional volumes into workers pods. It can be templated like in the following example: + # extraVolumes: + # - name: my-templated-extra-volume + # secret: + # secretName: '{{ include "my_secret_template" . }}' + # defaultMode: 0640 + # optional: true + + # Additional volume mounts attached to the pods created with pod-template-file + extraVolumeMounts: [] + # Mount additional volumes into workers pods. It can be templated like in the following example: + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + + # Select certain nodes for pods created with pod-template-file + nodeSelector: {} + + runtimeClassName: ~ + + priorityClassName: ~ + + affinity: {} + + tolerations: [] + + topologySpreadConstraints: [] + + # hostAliases to use in pods created with pod-template-file + # See: + # https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ + hostAliases: [] + # - ip: "127.0.0.2" + # hostnames: + # - "test.hostname.one" + # - ip: "127.0.0.3" + # hostnames: + # - "test.hostname.two" + + # Pod annotations for the pods created with pod-template-file (templated) + podAnnotations: {} + + # Labels specific to pods created with pod-template-file + labels: {} + + # Additional env variable configuration for pods created with pod-template-file + env: [] + + schedulerName: ~ + +# Airflow scheduler settings +scheduler: + enabled: true + # hostAliases for the scheduler pod + hostAliases: [] + # - ip: "127.0.0.1" + # hostnames: + # - "foo.local" + # - ip: "10.1.2.3" + # hostnames: + # - "foo.remote" + + # If the scheduler stops heartbeating for 5 minutes (5*60s) kill the + # scheduler and let Kubernetes restart it + livenessProbe: + initialDelaySeconds: 10 + timeoutSeconds: 20 + failureThreshold: 5 + periodSeconds: 60 + command: ~ + + # Wait for at most 1 minute (6*10s) for the scheduler container to startup. + # LivenessProbe kicks in after the first successful startupProbe + startupProbe: + initialDelaySeconds: 0 + failureThreshold: 6 + periodSeconds: 10 + timeoutSeconds: 20 + command: ~ + + # Amount of scheduler replicas + replicas: 1 + # Max number of old replicasets to retain + revisionHistoryLimit: ~ + + # Command to use when running the Airflow scheduler (templated). + command: ~ + # Args to use when running the Airflow scheduler (templated). + args: ["bash", "-c", "exec airflow scheduler"] + + # Update Strategy when scheduler is deployed as a StatefulSet + # (when using LocalExecutor and `workers.persistence`) + updateStrategy: ~ + # Update Strategy when scheduler is deployed as a Deployment + # (when not using LocalExecutor and `workers.persistence`) + strategy: ~ + + # When not set, the values defined in the global `securityContext` will be used + # (deprecated, use `scheduler.securityContexts` instead) + securityContext: {} + # runAsUser: 50000 + # fsGroup: 0 + # runAsGroup: 0 + + # Detailed default security context for scheduler Deployments for container and pod level + securityContexts: + pod: {} + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + # Grace period for tasks to finish after SIGTERM is sent from Kubernetes + terminationGracePeriodSeconds: 10 + + # Create Service Account + serviceAccount: + # Affects all executors that launch pods + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + automountServiceAccountToken: true + + # Specifies whether a Service Account should be created + create: true + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + name: ~ + + # Annotations to add to scheduler Kubernetes Service Account. + annotations: {} + + # Service Account Token Volume configuration + # This is only used when `automountServiceAccountToken` is 'false' + # and allows manual configuration of the Service Account token volume + serviceAccountTokenVolume: + # Enable manual Service Account token volume configuration + enabled: false + + # Path where the Service Account token should be mounted + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + + # Name of the volume + volumeName: kube-api-access + + # Token expiration in seconds + expirationSeconds: 3600 + + # Audience for the token + audience: ~ + + # Scheduler pod disruption budget + podDisruptionBudget: + enabled: false + + # PDB configuration (`minAvailable` and `maxUnavailable` are mutually exclusive) + config: + maxUnavailable: 1 + # minAvailable: 1 + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # This setting tells Kubernetes that its ok to evict + # when it wants to scale a node down. + safeToEvict: true + + # Launch additional containers into scheduler (templated). + extraContainers: [] + # Add additional init containers into scheduler (templated). + extraInitContainers: [] + + # Mount additional volumes into scheduler. + extraVolumes: [] + extraVolumeMounts: [] + # It can be templated like in the following example: + # extraVolumes: + # - name: my-templated-extra-volume + # secret: + # secretName: '{{ include "my_secret_template" . }}' + # defaultMode: 0640 + # optional: true + # + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + + # Select certain nodes for Airflow scheduler pods. + nodeSelector: {} + affinity: {} + # default scheduler affinity is: + # podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - podAffinityTerm: + # labelSelector: + # matchLabels: + # component: scheduler + # topologyKey: kubernetes.io/hostname + # weight: 100 + + tolerations: [] + topologySpreadConstraints: [] + + priorityClassName: ~ + + # Annotations for scheduler Deployment + annotations: {} + + # Pod annotations for scheduler pods (templated) + podAnnotations: {} + + # Labels specific to scheduler objects and pods + labels: {} + + logGroomerSidecar: + # Whether to deploy the Airflow scheduler log groomer sidecar. + enabled: true + + # Command to use when running the Airflow scheduler log groomer sidecar (templated). + command: ~ + + # Args to use when running the Airflow scheduler log groomer sidecar (templated). + args: ["bash", "/clean-logs"] + + # Number of days to retain logs + retentionDays: 15 + + # Number of minutes to retain logs. + # This can be used for finer granularity than days. + # Total retention is `retentionDays` + `retentionMinutes`. + retentionMinutes: 0 + + # Frequency to attempt to groom logs, in minutes + frequencyMinutes: 15 + + # Max size of logs in bytes. 0 = disabled + maxSizeBytes: 0 + + # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if `maxSizeBytes` is set. + maxSizePercent: 0 + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Detailed default security context for `logGroomerSidecar` for container level + securityContexts: + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + env: [] + + waitForMigrations: + # Whether to create init container to wait for db migrations + enabled: true + + env: [] + + # Detailed default security context for waitForMigrations for container level + securityContexts: + container: {} + + env: [] + +# Airflow create user job settings +createUserJob: + # Whether the create user job should be created + enabled: true + + # Create initial user. + defaultUser: + role: Admin + username: admin + email: admin@example.com + firstName: admin + lastName: user + password: admin + + # Limit the lifetime of the job object after it finished execution. + ttlSecondsAfterFinished: 300 + + # Command to use when running the create user job (templated). + command: ~ + + # Args to use when running the create user job (templated). + args: + - "bash" + - "-c" + # The format below is necessary to get `helm lint` happy + - |- + exec \ + airflow users create "$@" + - -- + # yamllint disable rule:line-length + - "-r" + - "{{ if .Values.webserver.defaultUser }}{{ .Values.webserver.defaultUser.role }}{{ else }}{{ .Values.createUserJob.defaultUser.role }}{{ end }}" + - "-u" + - "{{ if .Values.webserver.defaultUser }}{{ .Values.webserver.defaultUser.username }}{{ else }}{{ .Values.createUserJob.defaultUser.username }}{{ end }}" + - "-e" + - "{{ if .Values.webserver.defaultUser }}{{ .Values.webserver.defaultUser.email }}{{ else }}{{ .Values.createUserJob.defaultUser.email }}{{ end }}" + - "-f" + - "{{ if .Values.webserver.defaultUser }}{{ .Values.webserver.defaultUser.firstName }}{{ else }}{{ .Values.createUserJob.defaultUser.firstName }}{{ end }}" + - "-l" + - "{{ if .Values.webserver.defaultUser }}{{ .Values.webserver.defaultUser.lastName }}{{ else }}{{ .Values.createUserJob.defaultUser.lastName }}{{ end }}" + - "-p" + - "{{ if .Values.webserver.defaultUser }}{{ .Values.webserver.defaultUser.password }}{{ else }}{{ .Values.createUserJob.defaultUser.password }}{{ end }}" + + # Annotations on the create user job pod (templated) + annotations: {} + + # `jobAnnotations` are annotations on the create user job + jobAnnotations: {} + + restartPolicy: OnFailure + + # Labels specific to `createUserJob` objects and pods + labels: {} + + # When not set, the values defined in the global `securityContext` will be used + # (deprecated, use `createUserJob.securityContexts` instead) + securityContext: {} + # runAsUser: 50000 + # fsGroup: 0 + # runAsGroup: 0 + + # Detailed default security context for `createUserJob` for container and pod level + securityContexts: + pod: {} + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + # Create Service Account + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + automountServiceAccountToken: true + + # Specifies whether a Service Account should be created + create: true + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + name: ~ + + # Annotations to add to create user Kubernetes Service Account. + annotations: {} + + # Launch additional containers into user creation job + extraContainers: [] + + # Add additional init containers into user creation job (templated). + extraInitContainers: [] + + # Mount additional volumes into user creation job. + extraVolumes: [] + extraVolumeMounts: [] + # It can be templated like in the following example: + # extraVolumes: + # - name: my-templated-extra-volume + # secret: + # secretName: '{{ include "my_secret_template" . }}' + # defaultMode: 0640 + # optional: true + # + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + + nodeSelector: {} + affinity: {} + tolerations: [] + topologySpreadConstraints: [] + priorityClassName: ~ + + # In case you need to disable the helm hooks that create the jobs after install. + # Disable this if you are e.g. using ArgoCD + useHelmHooks: true + + applyCustomEnv: true + + env: [] + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# Airflow database migration job settings +migrateDatabaseJob: + enabled: true + + # Limit the lifetime of the job object after it finished execution. + ttlSecondsAfterFinished: 300 + + # Command to use when running the migrate database job (templated). + command: ~ + + # Args to use when running the migrate database job (templated). + args: + - "bash" + - "-c" + - >- + exec \ + + airflow db migrate + + # Annotations on the database migration pod (templated) + annotations: {} + + # `jobAnnotations` are annotations on the database migration job + jobAnnotations: {} + + restartPolicy: OnFailure + + # Labels specific to migrate database job objects and pods + labels: {} + + # When not set, the values defined in the global `securityContext` will be used + # (deprecated, use `migrateDatabaseJob.securityContexts` instead) + securityContext: {} + # runAsUser: 50000 + # fsGroup: 0 + # runAsGroup: 0 + + # Detailed default security context for `migrateDatabaseJob` for container and pod level + securityContexts: + pod: {} + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + # Create Service Account + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + automountServiceAccountToken: true + + # Specifies whether a Service Account should be created + create: true + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + name: ~ + + # Annotations to add to migrate database job Kubernetes Service Account. + annotations: {} + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Launch additional containers into database migration job + extraContainers: [] + + # Add additional init containers into migrate database job (templated). + extraInitContainers: [] + + # Mount additional volumes into database migration job. + extraVolumes: [] + extraVolumeMounts: [] + # It can be templated like in the following example: + # extraVolumes: + # - name: my-templated-extra-volume + # secret: + # secretName: '{{ include "my_secret_template" . }}' + # defaultMode: 0640 + # optional: true + # + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + + nodeSelector: {} + affinity: {} + tolerations: [] + topologySpreadConstraints: [] + priorityClassName: ~ + + # In case you need to disable the helm hooks that create the jobs after install. + # Disable this if you are using ArgoCD for example + useHelmHooks: true + + applyCustomEnv: true + env: [] + +apiServer: + enabled: true + + # Number of Airflow API servers in the Deployment. + # Omitted from the Deployment, when HPA is enabled. + replicas: 1 + + # Max number of old ReplicaSets to retain + revisionHistoryLimit: ~ + + # Labels specific to Airflow API server objects and pods + labels: {} + + # Command to use when running the Airflow API server (templated). + command: ~ + + # Args to use when running the Airflow API server (templated). + args: ["bash", "-c", "exec airflow api-server"] + # Example: To enable proxy headers support when running behind a reverse proxy: + # args: ["bash", "-c", "exec airflow api-server --proxy-headers"] + + allowPodLogReading: true + + # Environment variables for the Airflow API server. + env: [] + # Example: To configure FORWARDED_ALLOW_IPS when running behind a reverse proxy: + # env: + # - name: FORWARDED_ALLOW_IPS + # value: "*" # Use "*" for trusted environments, or specify proxy IP ranges for production + + # Allow Horizontal Pod Autoscaler (HPA) configuration for api-server. (optional) + # HPA automatically scales the number of api-server pods based on observed metrics. + # HPA automatically adjusts api-server replicas between `minReplicaCount` and `maxReplicaCount` based on metrics. + hpa: + enabled: false + + # Minimum number of api-servers created by HPA + minReplicaCount: 1 + + # Maximum number of api-servers created by HPA + maxReplicaCount: 5 + + # Specifications for which to use to calculate the desired replica count + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 + + # Scaling behavior of the target in both Up and Down directions + behavior: {} + + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + automountServiceAccountToken: true + + # Specifies whether a Service Account should be created + create: true + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + name: ~ + + # Annotations to add to Airflow API server Kubernetes Service Account. + annotations: {} + + service: + type: ClusterIP + + # Service annotations + annotations: {} + + ports: + - name: api-server + port: "{{ .Values.ports.apiServer }}" + + loadBalancerIP: ~ + + # Limit load balancer source ips to list of CIDRs + loadBalancerSourceRanges: [] + # loadBalancerSourceRanges: + # - "10.123.0.0/16" + + podDisruptionBudget: + enabled: false + + # PDB configuration (`minAvailable` and `maxUnavailable` are mutually exclusive) + config: + maxUnavailable: 1 + # minAvailable: 1 + + # Allow overriding Update Strategy for API server + strategy: ~ + + # Detailed default security contexts for Airflow API server Deployments for container and pod level + securityContexts: + pod: {} + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + waitForMigrations: + # Whether to create init container to wait for db migrations + enabled: true + + env: [] + + # Detailed default security context for waitForMigrations for container level + securityContexts: + container: {} + + # Launch additional containers into the Airflow API server pods. + extraContainers: [] + + # Add additional init containers into API server (templated). + extraInitContainers: [] + + # Mount additional volumes into API server. + extraVolumes: [] + extraVolumeMounts: [] + # It can be templated like in the following example: + # extraVolumes: + # - name: my-templated-extra-volume + # secret: + # secretName: '{{ include "my_secret_template" . }}' + # defaultMode: 0640 + # optional: true + # + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + + # Select certain nodes for Airflow API server pods. + nodeSelector: {} + affinity: {} + tolerations: [] + topologySpreadConstraints: [] + + priorityClassName: ~ + + # hostAliases for API server pod + hostAliases: [] + + # Annotations for Airflow API server Deployment + annotations: {} + + # Pod annotations for API server pods (templated) + podAnnotations: {} + + networkPolicy: + ingress: + # Peers for Airflow API server NetworkPolicy ingress + from: [] + + # Ports for Airflow API server NetworkPolicy ingress (if `from` is set) + ports: + - port: "{{ .Values.ports.apiServer }}" + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Add custom annotations to the `apiServer` ConfigMap + configMapAnnotations: {} + + # This string (templated) will be mounted into the Airflow API Server + # as a custom webserver_config.py. You can bake a webserver_config.py into + # your image instead or specify a ConfigMap containing the + # webserver_config.py. + apiServerConfig: ~ + # apiServerConfig: | + # from airflow import configuration as conf + + # # The SQLAlchemy connection string. + # SQLALCHEMY_DATABASE_URI = conf.get('database', 'SQL_ALCHEMY_CONN') + + # # Flask-WTF flag for CSRF + # CSRF_ENABLED = True + + apiServerConfigConfigMapName: ~ + + livenessProbe: + initialDelaySeconds: 15 + timeoutSeconds: 5 + failureThreshold: 5 + periodSeconds: 10 + scheme: HTTP + + readinessProbe: + initialDelaySeconds: 15 + timeoutSeconds: 5 + failureThreshold: 5 + periodSeconds: 10 + scheme: HTTP + + startupProbe: + initialDelaySeconds: 0 + timeoutSeconds: 20 + failureThreshold: 6 + periodSeconds: 10 + scheme: HTTP + +# Airflow webserver settings (only Airflow<3.0) +webserver: + enabled: true + + # Add custom annotations to the webserver ConfigMap + configMapAnnotations: {} + + # hostAliases for the webserver pod + hostAliases: [] + # - ip: "127.0.0.1" + # hostnames: + # - "foo.local" + # - ip: "10.1.2.3" + # hostnames: + # - "foo.remote" + + allowPodLogReading: true + + livenessProbe: + initialDelaySeconds: 15 + timeoutSeconds: 5 + failureThreshold: 5 + periodSeconds: 10 + scheme: HTTP + + readinessProbe: + initialDelaySeconds: 15 + timeoutSeconds: 5 + failureThreshold: 5 + periodSeconds: 10 + scheme: HTTP + + # Wait for at most 1 minute (6*10s) for the webserver container to startup. + # LivenessProbe kicks in after the first successful startupProbe + startupProbe: + initialDelaySeconds: 0 + timeoutSeconds: 20 + failureThreshold: 6 + periodSeconds: 10 + scheme: HTTP + + # Number of webservers + replicas: 1 + + # Max number of old replicasets to retain + revisionHistoryLimit: ~ + + # Command to use when running the Airflow webserver (templated). + command: ~ + + # Args to use when running the Airflow webserver (templated). + args: ["bash", "-c", "exec airflow webserver"] + + # Grace period for webserver to finish after SIGTERM is sent from Kubernetes + terminationGracePeriodSeconds: 30 + + # Allow HPA + hpa: + enabled: false + + # Minimum number of webservers created by HPA + minReplicaCount: 1 + + # Maximum number of webservers created by HPA + maxReplicaCount: 5 + + # Specifications for which to use to calculate the desired replica count + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + + # Scaling behavior of the target in both Up and Down directions + behavior: {} + + # Create Service Account + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + automountServiceAccountToken: true + + # Specifies whether a Service Account should be created + create: true + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + name: ~ + + # Annotations to add to webserver Kubernetes Service Account. + annotations: {} + + # Webserver pod disruption budget + podDisruptionBudget: + enabled: false + + # PDB configuration (`minAvailable` and `maxUnavailable` are mutually exclusive) + config: + maxUnavailable: 1 + # minAvailable: 1 + + # Allow overriding Update Strategy for Webserver + strategy: ~ + + # When not set, the values defined in the global `securityContext` will be used + # (deprecated, use `webserver.securityContexts` instead) + securityContext: {} + # runAsUser: 50000 + # fsGroup: 0 + # runAsGroup: 0 + + # Detailed default security contexts for webserver Deployments for container and pod level + securityContexts: + pod: {} + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + # Additional network policies as needed (deprecated, use `webserver.networkPolicy.ingress.from` instead) + extraNetworkPolicies: [] + networkPolicy: + ingress: + # Peers for webserver NetworkPolicy ingress + from: [] + + # Ports for webserver NetworkPolicy ingress (if `from` is set) + ports: + - port: "{{ .Values.ports.airflowUI }}" + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Create initial user. (deprecated, use `createUserJob` section instead) + # defaultUser: + # enabled: true + # role: Admin + # username: admin + # email: admin@example.com + # firstName: admin + # lastName: user + # password: admin + + # Launch additional containers into webserver (templated). + extraContainers: [] + + # Add additional init containers into webserver (templated). + extraInitContainers: [] + + # Mount additional volumes into webserver. + extraVolumes: [] + extraVolumeMounts: [] + # It can be templated like in the following example: + # extraVolumes: + # - name: my-templated-extra-volume + # secret: + # secretName: '{{ include "my_secret_template" . }}' + # defaultMode: 0640 + # optional: true + # + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + + # This string (templated) will be mounted into the Airflow Webserver + # as a custom webserver_config.py. You can bake a webserver_config.py into + # your image instead or specify a ConfigMap containing the + # webserver_config.py. + webserverConfig: ~ + # webserverConfig: | + # from airflow import configuration as conf + + # # The SQLAlchemy connection string. + # SQLALCHEMY_DATABASE_URI = conf.get('database', 'SQL_ALCHEMY_CONN') + + # # Flask-WTF flag for CSRF + # CSRF_ENABLED = True + + webserverConfigConfigMapName: ~ + + service: + type: ClusterIP + + # Service annotations + annotations: {} + + ports: + - name: airflow-ui + port: "{{ .Values.ports.airflowUI }}" + # To change the port used to access the webserver: + # ports: + # - name: airflow-ui + # port: 80 + # targetPort: airflow-ui + # To only expose a sidecar, not the webserver directly: + # ports: + # - name: only_sidecar + # port: 80 + # targetPort: 8888 + # If you have a public IP, set NodePort to set an external port. + # Service type must be 'NodePort': + # ports: + # - name: airflow-ui + # port: 8080 + # targetPort: 8080 + # nodePort: 31151 + + loadBalancerIP: ~ + + # Limit load balancer source ips to list of CIDRs + loadBalancerSourceRanges: [] + # loadBalancerSourceRanges: + # - "10.123.0.0/16" + + # Select certain nodes for Airflow webserver pods. + nodeSelector: {} + priorityClassName: ~ + + affinity: {} + # default webserver affinity is: + # podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - podAffinityTerm: + # labelSelector: + # matchLabels: + # component: webserver + # topologyKey: kubernetes.io/hostname + # weight: 100 + + tolerations: [] + topologySpreadConstraints: [] + + # Annotations for webserver Deployment + annotations: {} + + # Pod annotations for webserver pods (templated) + podAnnotations: {} + + # Labels specific webserver app + labels: {} + + waitForMigrations: + # Whether to create init container to wait for db migrations + enabled: true + + env: [] + + # Detailed default security context for waitForMigrations for container level + securityContexts: + container: {} + + env: [] + +# Airflow Triggerer Config +triggerer: + enabled: true + + # Number of Airflow triggerers in the Deployment + replicas: 1 + + # Max number of old replicasets to retain + revisionHistoryLimit: ~ + + # Command to use when running Airflow triggerers (templated). + command: ~ + # Args to use when running Airflow triggerer (templated). + args: ["bash", "-c", "exec airflow triggerer"] + + # Update Strategy when triggerer is deployed as a StatefulSet + updateStrategy: ~ + # Update Strategy when triggerer is deployed as a Deployment + strategy: + rollingUpdate: + maxSurge: "100%" + maxUnavailable: "50%" + + # If the triggerer stops heartbeating for 5 minutes (5*60s) kill the + # triggerer and let Kubernetes restart it + livenessProbe: + initialDelaySeconds: 10 + timeoutSeconds: 20 + failureThreshold: 5 + periodSeconds: 60 + command: ~ + + # Create Service Account + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + automountServiceAccountToken: true + + # Specifies whether a Service Account should be created + create: true + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + name: ~ + + # Annotations to add to triggerer Kubernetes Service Account. + annotations: {} + + # When not set, the values defined in the global `securityContext` will be used + # (deprecated, use `triggerer.securityContexts` instead) + securityContext: {} + # runAsUser: 50000 + # fsGroup: 0 + # runAsGroup: 0 + + # Detailed default security context for triggerer for container and pod level + securityContexts: + pod: {} + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + persistence: + # Enable persistent volumes + enabled: true + + # This policy determines whether PVCs should be deleted when StatefulSet is scaled down or removed. + persistentVolumeClaimRetentionPolicy: ~ + + # Volume size for triggerer StatefulSet + size: 100Gi + + # If using a custom storageClass, pass name ref to all statefulSets here + storageClassName: + + # Execute init container to chown log directory. + # This is currently only needed in kind, due to usage + # of local-path provisioner. + fixPermissions: false + + # Annotations to add to triggerer volumes + annotations: {} + + # Triggerer pod disruption budget + podDisruptionBudget: + enabled: false + + # PDB configuration (`minAvailable` and `maxUnavailable` are mutually exclusive) + config: + maxUnavailable: 1 + # minAvailable: 1 + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Grace period for triggerer to finish after SIGTERM is sent from Kubernetes + terminationGracePeriodSeconds: 60 + + # This setting tells Kubernetes that its ok to evict + # when it wants to scale a node down. + safeToEvict: true + + # Launch additional containers into triggerer (templated). + extraContainers: [] + + # Add additional init containers into triggerers (templated). + extraInitContainers: [] + + # Mount additional volumes into triggerer. + extraVolumes: [] + extraVolumeMounts: [] + # It can be templated like in the following example: + # extraVolumes: + # - name: my-templated-extra-volume + # secret: + # secretName: '{{ include "my_secret_template" . }}' + # defaultMode: 0640 + # optional: true + # + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + + # Select certain nodes for Airflow triggerer pods. + nodeSelector: {} + + affinity: {} + # default triggerer affinity is: + # podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - podAffinityTerm: + # labelSelector: + # matchLabels: + # component: triggerer + # topologyKey: kubernetes.io/hostname + # weight: 100 + + tolerations: [] + topologySpreadConstraints: [] + + # hostAliases for the triggerer pod + hostAliases: [] + # - ip: "127.0.0.1" + # hostnames: + # - "foo.local" + # - ip: "10.1.2.3" + # hostnames: + # - "foo.remote" + + priorityClassName: ~ + + # Annotations for the triggerer Deployment + annotations: {} + + # Pod annotations for triggerer pods (templated) + podAnnotations: {} + + # Labels specific to triggerer objects and pods + labels: {} + + logGroomerSidecar: + # Whether to deploy the Airflow triggerer log groomer sidecar. + enabled: true + + # Command to use when running the Airflow triggerer log groomer sidecar (templated). + command: ~ + + # Args to use when running the Airflow triggerer log groomer sidecar (templated). + args: ["bash", "/clean-logs"] + + # Number of days to retain logs + retentionDays: 15 + + # Number of minutes to retain logs. + # This can be used for finer granularity than days. + # Total retention is `retentionDays` + `retentionMinutes`. + retentionMinutes: 0 + + # frequency to attempt to groom logs, in minutes + frequencyMinutes: 15 + + # Max size of logs in bytes. 0 = disabled + maxSizeBytes: 0 + + # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if `maxSizeBytes` is set. + maxSizePercent: 0 + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Detailed default security context for `logGroomerSidecar` for container level + securityContexts: + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + env: [] + + waitForMigrations: + # Whether to create init container to wait for db migrations + enabled: true + + env: [] + + # Detailed default security context for waitForMigrations for container level + securityContexts: + container: {} + + env: [] + + # Allow KEDA autoscaling. + keda: + enabled: false + namespaceLabels: {} + + # How often KEDA polls the Airflow DB to report new scale requests to the HPA + pollingInterval: 5 + + # How many seconds KEDA will wait before scaling to zero. + # Note that HPA has a separate cooldown period for scale-downs + cooldownPeriod: 30 + + # Minimum number of triggerers created by keda + minReplicaCount: 0 + + # Maximum number of triggerers created by keda + maxReplicaCount: 10 + + # Specify HPA related options + advanced: {} + # horizontalPodAutoscalerConfig: + # behavior: + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Percent + # value: 100 + # periodSeconds: 15 + + # Query to use for KEDA autoscaling. Must return a single integer. + query: >- + SELECT ceil(COUNT(*)::decimal / {{ include "triggerer.capacity" . }}) + FROM trigger + + # Whether to use PGBouncer to connect to the database or not when it is enabled + # This configuration will be ignored if PGBouncer is not enabled + usePgbouncer: false + +# Airflow Dag Processor Config +dagProcessor: + enabled: ~ + + # Dag Bundle Configuration + # Define Dag bundles in a structured YAML format. This will be automatically + # converted to JSON string format for `config.dag_processor.dag_bundle_config_list`. + dagBundleConfigList: + - name: dags-folder + classpath: "airflow.dag_processing.bundles.local.LocalDagBundle" + kwargs: {} + # Example: + # dagBundleConfigList: + # - name: bundle1 + # classpath: "airflow.providers.git.bundles.git.GitDagBundle" + # kwargs: + # git_conn_id: "GITHUB__repo1" + # subdir: "dags" + # tracking_ref: "main" + # refresh_interval: 60 + # - name: bundle2 + # classpath: "airflow.providers.git.bundles.git.GitDagBundle" + # kwargs: + # git_conn_id: "GITHUB__repo2" + # subdir: "dags" + # tracking_ref: "develop" + # refresh_interval: 120 + # - name: dags-folder + # classpath: "airflow.dag_processing.bundles.local.LocalDagBundle" + # kwargs: {} + + # Number of Airflow dag processors in the Deployment + replicas: 1 + + # Max number of old ReplicaSets to retain + revisionHistoryLimit: ~ + + # Command to use when running Airflow dag processors (templated). + command: ~ + + # Args to use when running Airflow dag processor (templated). + args: ["bash", "-c", "exec airflow dag-processor"] + + # Update Strategy for dag processors + strategy: + rollingUpdate: + maxSurge: "100%" + maxUnavailable: "50%" + + # If the dag processor stops heartbeating for 5 minutes (5*60s) kill the + # dag processor and let Kubernetes restart it + livenessProbe: + initialDelaySeconds: 10 + timeoutSeconds: 20 + failureThreshold: 5 + periodSeconds: 60 + command: ~ + + # Create Service Account + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + automountServiceAccountToken: true + + # Specifies whether a Service Account should be created + create: true + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + name: ~ + + # Annotations to add to dag processor Kubernetes Service Account. + annotations: {} + + # Dag processor pod disruption budget + podDisruptionBudget: + enabled: false + + # PDB configuration (`minAvailable` and `maxUnavailable` are mutually exclusive) + config: + maxUnavailable: 1 + # minAvailable: 1 + + # When not set, the values defined in the global `securityContext` will be used + # (deprecated, use `dagProcessor.securityContexts` instead) + securityContext: {} + # runAsUser: 50000 + # fsGroup: 0 + # runAsGroup: 0 + + # Detailed default security context for dagProcessor for container and pod level + securityContexts: + pod: {} + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Grace period for dag processor to finish after SIGTERM is sent from Kubernetes + terminationGracePeriodSeconds: 60 + + # This setting tells Kubernetes that its ok to evict + # when it wants to scale a node down. + safeToEvict: true + + # Launch additional containers into dag processor (templated). + extraContainers: [] + + # Add additional init containers into dag processors (templated). + extraInitContainers: [] + + # Mount additional volumes into dag processor. + extraVolumes: [] + extraVolumeMounts: [] + # It can be templated like in the following example: + # extraVolumes: + # - name: my-templated-extra-volume + # secret: + # secretName: '{{ include "my_secret_template" . }}' + # defaultMode: 0640 + # optional: true + # + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + + # Select certain nodes for Airflow dag processor pods. + nodeSelector: {} + + affinity: {} + # Default dag processor affinity is: + # podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - podAffinityTerm: + # labelSelector: + # matchLabels: + # component: dag-processor + # topologyKey: kubernetes.io/hostname + # weight: 100 + + tolerations: [] + topologySpreadConstraints: [] + + priorityClassName: ~ + + # Annotations for the dag processor Deployment + annotations: {} + + # Pod annotations for dag processor pods (templated) + podAnnotations: {} + + logGroomerSidecar: + # Whether to deploy the Airflow dag processor log groomer sidecar. + enabled: true + + # Command to use when running the Airflow dag processor log groomer sidecar (templated). + command: ~ + + # Args to use when running the Airflow dag processor log groomer sidecar (templated). + args: ["bash", "/clean-logs"] + + # Number of days to retain logs + retentionDays: 15 + + # Number of minutes to retain logs. + # This can be used for finer granularity than days. + # Total retention is `retentionDays` + `retentionMinutes`. + retentionMinutes: 0 + + # frequency to attempt to groom logs, in minutes + frequencyMinutes: 15 + + # Max size of logs in bytes. 0 = disabled + maxSizeBytes: 0 + + # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if `maxSizeBytes` is set. + maxSizePercent: 0 + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContexts: + container: {} + + env: [] + + waitForMigrations: + # Whether to create init container to wait for db migrations + enabled: true + + env: [] + + # Detailed default security context for waitForMigrations for container level + securityContexts: + container: {} + + # Labels specific to dag processor objects + labels: {} + + # Environment variables to add to dag processor container + env: [] + +# Flower settings +flower: + # Enable flower. + # If True, and using CeleryExecutor/CeleryKubernetesExecutor, will deploy flower app. + enabled: false + + livenessProbe: + initialDelaySeconds: 10 + timeoutSeconds: 5 + failureThreshold: 10 + periodSeconds: 5 + + readinessProbe: + initialDelaySeconds: 10 + timeoutSeconds: 5 + failureThreshold: 10 + periodSeconds: 5 + + # Wait for at most 1 minute (6*10s) for the flower container to startup. + # LivenessProbe kicks in after the first successful StartupProbe + startupProbe: + initialDelaySeconds: 0 + timeoutSeconds: 20 + failureThreshold: 6 + periodSeconds: 10 + + # Max number of old ReplicaSets to retain + revisionHistoryLimit: ~ + + # Command to use when running flower (templated). + command: ~ + + # Args to use when running flower (templated). + args: + - "bash" + - "-c" + # The format below is necessary to get `helm lint` happy + - |- + exec \ + airflow celery flower + + # Additional network policies as needed (deprecated, use `flower.networkPolicy.ingress.from` instead) + extraNetworkPolicies: [] + networkPolicy: + ingress: + # Peers for flower NetworkPolicy ingress + from: [] + + # Ports for flower NetworkPolicy ingress (if `from` is set) + ports: + - port: "{{ .Values.ports.flowerUI }}" + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # When not set, the values defined in the global `securityContext` will be used + # (deprecated, use `flower.securityContexts` instead) + securityContext: {} + # runAsUser: 50000 + # fsGroup: 0 + # runAsGroup: 0 + + # Detailed default security context for flower for container and pod level + securityContexts: + pod: {} + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + # Create Service Account + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + automountServiceAccountToken: true + + # Specifies whether a Service Account should be created + create: true + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + name: ~ + + # Annotations to add to worker Kubernetes Service Account. + annotations: {} + + + # If set, the secret must contain a base64-encoded 'connection' key with + # a Flower basic auth connection string user:password. + secretName: ~ + # Example secret: + # kind: Secret + # apiVersion: v1 + # metadata: + # name: custom-flower-secret + # type: Opaque + # data: + # connection: + + # Add custom annotations to the flower secret + secretAnnotations: {} + + # If `secretName` is not specified, set username and password (secret will be created automatically) + username: ~ + password: ~ + + service: + type: ClusterIP + + # Service annotations + annotations: {} + + ports: + - name: flower-ui + port: "{{ .Values.ports.flowerUI }}" + # To change the port used to access flower: + # ports: + # - name: flower-ui + # port: 8080 + # targetPort: flower-ui + + loadBalancerIP: ~ + + # Limit load balancer source ips to list of CIDRs + loadBalancerSourceRanges: [] + # loadBalancerSourceRanges: + # - "10.123.0.0/16" + + # Launch additional containers into the flower pods. + extraContainers: [] + + # Mount additional volumes into the flower pods. + extraVolumes: [] + extraVolumeMounts: [] + # It can be templated like in the following example: + # extraVolumes: + # - name: my-templated-extra-volume + # secret: + # secretName: '{{ include "my_secret_template" . }}' + # defaultMode: 0640 + # optional: true + # + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + + # Select certain nodes for Airflow flower pods. + nodeSelector: {} + affinity: {} + tolerations: [] + topologySpreadConstraints: [] + + priorityClassName: ~ + + # Annotations for the flower Deployment + annotations: {} + + # Pod annotations for flower pods (templated) + podAnnotations: {} + + # Labels specific to flower objects and pods + labels: {} + + env: [] + +# StatsD settings +statsd: + # Add custom annotations to the StatsD ConfigMap + configMapAnnotations: {} + + enabled: true + + # Max number of old ReplicaSets to retain + revisionHistoryLimit: ~ + + # Arguments for StatsD exporter command. + # By default contains path in the container to the mapping config file. + args: ["--statsd.mapping-config=/etc/statsd-exporter/mappings.yml"] + # If you ever need to fully override the entire `args` list, you can + # supply your own array here; if set, all below flag-specific values + # under `statsd.cache` section are ignored. + # args: + # - "--statsd.cache-size=1000" + # - "--statsd.cache-type=random" + # - "--ttl=10m" + + cache: + # Maximum number of metric‐mapping entries to keep in cache. + # When you send more distinct metric names than this, older entries + # will be evicted according to cacheType. + size: 1000 + + # Metrics Eviction policy for the mapping cache. + # - lru → Least‐Recently‐Used eviction + # - random → Random eviction + type: lru + + # Per‐metric time‐to‐live. When set to a non‐zero duration, any metric + # series that hasn't received an update in this interval will be dropped + # from the exported '/metrics' output. + # Format: Go duration string (e.g. "30s", "5m", "1h") + # Default: "0s" (disabled, never expires) + ttl: "0s" + + # Annotations to add to the StatsD Deployment. + annotations: {} + + # Grace period for StatsD to finish after SIGTERM is sent from Kubernetes + terminationGracePeriodSeconds: 30 + + # Create Service Account + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + automountServiceAccountToken: true + + # Specifies whether a Service Account should be created + create: true + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + name: ~ + + # Annotations to add to worker Kubernetes Service Account. + annotations: {} + + uid: 65534 + + # (deprecated, use `statsd.securityContexts` instead) + securityContext: {} + # runAsUser: 65534 + # fsGroup: 0 + # runAsGroup: 0 + + # Detailed default security context for StatsD Deployments for container and pod level + securityContexts: + pod: {} + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + # Additional network policies as needed + extraNetworkPolicies: [] + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + service: + extraAnnotations: {} + + # Select certain nodes for StatsD pods. + nodeSelector: {} + affinity: {} + tolerations: [] + topologySpreadConstraints: [] + + priorityClassName: ~ + + # Additional mappings for StatsD exporter. + # If set, will merge default mapping and extra mappings, where default mapping has higher priority. + # If you want to change some default mapping, please use `overrideMappings` setting. + extraMappings: [] + + # Override mappings for StatsD exporter. + # If set, will ignore setting item in default and `extraMappings`. + # If you use it, ensure that it contains all mapping items. + overrideMappings: [] + + # Pod annotations for StatsD pods (templated) + podAnnotations: {} + + # Labels specific to StatsD objects and pods + labels: {} + + # Environment variables to add to StatsD container + env: [] + +# PgBouncer settings +pgbouncer: + # Enable PgBouncer + enabled: false + + # Number of PgBouncer replicas to run in Deployment + replicas: 1 + + # Max number of old replicasets to retain + revisionHistoryLimit: ~ + + # Command to use for PgBouncer (templated). + command: ["pgbouncer", "-u", "nobody", "/etc/pgbouncer/pgbouncer.ini"] + + # Args to use for PgBouncer (templated). + args: ~ + + auth_type: scram-sha-256 + auth_file: /etc/pgbouncer/users.txt + + # Whether to mount the config secret files at a default location (/etc/pgbouncer/*). + # Can be skipped to allow for other means to get the values, e.g. secrets provider class. + mountConfigSecret: true + + # Annotations to be added to the PgBouncer Deployment + annotations: {} + + # Pod annotations for PgBouncer pods (templated) + podAnnotations: {} + + # Add custom annotations to the PgBouncer certificates secret + certificatesSecretAnnotations: {} + + # Create Service Account + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + automountServiceAccountToken: true + + # Specifies whether a Service Account should be created + create: true + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + name: ~ + + # Annotations to add to worker Kubernetes Service Account. + annotations: {} + + # Additional network policies as needed + extraNetworkPolicies: [] + + # Pool sizes + metadataPoolSize: 10 + resultBackendPoolSize: 5 + + # Maximum clients that can connect to PgBouncer (higher = more file descriptors) + maxClientConn: 100 + + # Supply the name of existing secret with 'pgbouncer.ini' and 'users.txt' defined + configSecretName: ~ + # Secret example: + # apiVersion: v1 + # kind: Secret + # metadata: + # name: pgbouncer-config-secret + # data: + # pgbouncer.ini: + # users.txt: + # type: Opaque + + # Add custom annotations to the PgBouncer config secret + configSecretAnnotations: {} + + # PgBouncer pod disruption budget + podDisruptionBudget: + enabled: false + + # PDB configuration (`minAvailable` and `maxUnavailable` are mutually exclusive) + config: + maxUnavailable: 1 + # minAvailable: 1 + + resources: {} + # resource: + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + service: + extraAnnotations: {} + clusterIp: ~ + + # https://www.pgbouncer.org/config.html + verbose: 0 + logDisconnections: 0 + logConnections: 0 + + sslmode: "prefer" + ciphers: "normal" + + ssl: + ca: ~ + cert: ~ + key: ~ + + # Add extra PgBouncer ini configuration in the databases section: + # https://www.pgbouncer.org/config.html#section-databases + extraIniMetadata: ~ + extraIniResultBackend: ~ + + # Add extra general PgBouncer ini configuration: https://www.pgbouncer.org/config.html + extraIni: ~ + + # Mount additional volumes into PgBouncer. + # Volumes apply to all PgBouncer containers, while volume mounts apply to the PgBouncer + # container itself. Metrics exporter container has its own mounts. + extraVolumes: [] + extraVolumeMounts: [] + # It can be templated like in the following example: + # extraVolumes: + # - name: my-templated-extra-volume + # secret: + # secretName: '{{ include "my_secret_template" . }}' + # defaultMode: 0640 + # optional: true + # + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + + # Launch additional containers into PgBouncer pod. + extraContainers: [] + + # Select certain nodes for PgBouncer pods. + nodeSelector: {} + affinity: {} + tolerations: [] + topologySpreadConstraints: [] + + priorityClassName: ~ + + uid: 65534 + + # Detailed default security context for PgBouncer for container level + securityContexts: + pod: {} + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: + preStop: + exec: + # Allow existing queries clients to complete within 120 seconds + command: ["/bin/sh", "-c", "killall -INT pgbouncer && sleep 120"] + + metricsExporterSidecar: + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + sslmode: "disable" + + # Supply the name of existing secret with PGBouncer connection URI containing + # stats user and password, where 'connection' key is base64-encoded value. + statsSecretName: ~ + # Secret example: + # apiVersion: v1 + # kind: Secret + # metadata: + # name: pgbouncer-stats-secret + # data: + # connection: postgresql://:@127.0.0.1:6543/pgbouncer? + # type: Opaque + + # Key containing the PGBouncer connection URI, defaults to 'connection' if not defined + statsSecretKey: ~ + + # Add custom annotations to the PgBouncer stats secret + statsSecretAnnotations: {} + + # Detailed default security context for metricsExporterSidecar for container level + securityContexts: + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + livenessProbe: + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + + readinessProbe: + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + + # Mount additional volumes into the metrics exporter. + extraVolumeMounts: [] + # It can be templated like in the following example: + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + + # Labels specific to PgBouncer objects and pods + labels: {} + + # Environment variables to add to PgBouncer container + env: [] + +# Configuration for the redis provisioned by the chart +redis: + enabled: true + terminationGracePeriodSeconds: 600 + + # Annotations for Redis Statefulset + annotations: {} + + # Create Service Account + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + automountServiceAccountToken: true + + # Specifies whether a Service Account should be created + create: true + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + name: ~ + + # Annotations to add to worker Kubernetes Service Account. + annotations: {} + + service: + # Service type + type: "ClusterIP" + + # If using ClusterIP service type, custom IP address can be specified + clusterIP: + + # If using NodePort service type, custom node port can be specified + nodePort: + + persistence: + # Enable persistent volumes + enabled: true + + # Volume size for worker StatefulSet + size: 1Gi + + # If using a custom storageClass, pass name ref to all statefulSets here + storageClassName: + + # Annotations to add to redis volumes + annotations: {} + + # The name of an existing PVC to use + existingClaim: + + persistentVolumeClaimRetentionPolicy: ~ + # persistentVolumeClaimRetentionPolicy: + # whenDeleted: Delete + # whenScaled: Delete + + # Configuration for empty dir volume (if `redis.persistence.enabled` == 'false') + # emptyDirConfig: + # sizeLimit: 1Gi + # medium: Memory + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # If set use as redis secret. Make sure to also set `data.brokerUrlSecretName` value. + passwordSecretName: ~ + + # If `passwordSecretName` is not specified, set `password` field. + # Otherwise a new password will be generated on install + # Note: password can only be set during 'helm install', not 'helm upgrade'. + password: ~ + + # Add custom annotations to the redis password secret + passwordSecretAnnotations: {} + + # This setting tells Kubernetes that its ok to evict + # when it wants to scale a node down. + safeToEvict: true + + # Select certain nodes for redis pods. + nodeSelector: {} + affinity: {} + tolerations: [] + topologySpreadConstraints: [] + priorityClassName: ~ + + # Set to 0 for backwards-compatibility + uid: 0 + + # (deprecated, use `redis.securityContexts` instead) + securityContext: {} + # runAsUser: 999 + # runAsGroup: 0 + + # Detailed default security context for redis for container and pod level + securityContexts: + pod: {} + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + # Labels specific to redis objects and pods + labels: {} + + # Pod annotations for Redis pods (templated) + podAnnotations: {} + +# Auth secret for a private registry (deprecated, use `imagePullSecrets` instead) +# This is used if pulling Airflow images from a private registry +registry: + # Name of the Kubernetes secret containing Base64 encoded credentials to connect to a private registry + # (deprecated, use `imagePullSecrets` instead). + secretName: ~ + + # Credentials to connect to a private registry, these will get Base64 encoded and stored in a secret + # (deprecated, use `imagePullSecrets` instead - requires manual secret creation). + connection: {} + # Example: + # connection: + # user: ~ + # pass: ~ + # host: ~ + # email: ~ + +# Elasticsearch logging configuration +elasticsearch: + # Enable elasticsearch task logging + enabled: false + + # A secret containing the connection + secretName: ~ + + # Object representing the connection, if `secretName` not specified + connection: {} + # Example: + # connection: + # scheme: ~ + # user: ~ + # pass: ~ + # host: ~ + # port: ~ + + # Add custom annotations to the elasticsearch secret + secretAnnotations: {} + +# OpenSearch logging configuration +opensearch: + # Enable opensearch task logging + enabled: false + + # A secret containing the connection + secretName: ~ + + # Object representing the connection, if `secretName` not specified + connection: {} + # Example: + # connection: + # scheme: ~ + # user: ~ + # pass: ~ + # host: ~ + # port: ~ + +# All ports used by chart +ports: + flowerUI: 5555 + airflowUI: 8080 + workerLogs: 8793 + triggererLogs: 8794 + redisDB: 6379 + statsdIngest: 9125 + statsdScrape: 9102 + pgbouncer: 6543 + pgbouncerScrape: 9127 + apiServer: 8080 + +# Define any ResourceQuotas for namespace +quotas: {} + +# Define default/max/min values for pods and containers in namespace +limits: [] + +# This runs as a CronJob to cleanup old pods spawned by the KubernetesExecutor. +# It is required to have KubernetesExecutor enabled. +cleanup: + enabled: false + + # Run every 15 minutes (templated). + schedule: "*/15 * * * *" + # To select a random-ish, deterministic starting minute between 3 and 12 inclusive for each release: + # schedule: '{{- add 3 (regexFind ".$" (adler32sum .Release.Name)) -}}-59/15 * * * *' + # To select the last digit of unix epoch time as the starting minute on each deploy: + # schedule: '{{- now | unixEpoch | trunc -1 -}}-59/* * * * *' + + # Command to use when running the cleanup CronJob (templated). + command: ~ + + # Args to use when running the cleanup CronJob (templated). + args: ["bash", "-c", "exec airflow kubernetes cleanup-pods --namespace={{ .Release.Namespace }}"] + + # `jobAnnotations` are annotations on the cleanup CronJob + jobAnnotations: {} + + # Select certain nodes for Airflow cleanup pods. + nodeSelector: {} + affinity: {} + tolerations: [] + topologySpreadConstraints: [] + priorityClassName: ~ + + # Pod annotations for cleanup pods (templated) + podAnnotations: {} + + # Labels specific to cleanup objects and pods + labels: {} + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Create Service Account + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + automountServiceAccountToken: true + + # Specifies whether a Service Account should be created + create: true + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + name: ~ + + # Annotations to add to cleanup CronJob Kubernetes Service Account. + annotations: {} + + # When not set, the values defined in the global `securityContext` will be used + # (deprecated, use `cleanup.securityContexts` instead) + securityContext: {} + # runAsUser: 50000 + # runAsGroup: 0 + + env: [] + + # Detailed default security context for cleanup for container level + securityContexts: + pod: {} + container: {} + + # container level lifecycle hooks + containerLifecycleHooks: {} + + # Specify history limit + # When set, overwrite the default k8s number of successful and failed CronJob executions that are saved. + failedJobsHistoryLimit: ~ + successfulJobsHistoryLimit: ~ + +# This runs as a CronJob to cleanup database for old entries. +databaseCleanup: + enabled: false + applyCustomEnv: true + + # Run every week on Sunday at midnight (templated). + schedule: "0 0 * * 0" + + # Command to use when running the database cleanup CronJob (templated). + command: ~ + + # Args to use when running the database cleanup CronJob (templated). + args: + - "bash" + - "-c" + - >- + CLEAN_TS=$(date -d "-{{ .Values.databaseCleanup.retentionDays }} days" +"%Y-%m-%dT%H:%M:%S"); + echo "Cleaning up metadata DB entries older than ${CLEAN_TS}"; + exec airflow db clean --clean-before-timestamp "${CLEAN_TS}" --yes + {{- if .Values.databaseCleanup.skipArchive }} --skip-archive{{ end }} + {{- if .Values.databaseCleanup.verbose }} --verbose{{ end }} + {{- with .Values.databaseCleanup.batchSize }} --batch-size {{ . }}{{ end }} + {{- with .Values.databaseCleanup.tables }} --tables {{ . | join "," }}{{ end }} + + # Number of days to retain entries in the metadata database. + retentionDays: 90 + + # Don't preserve purged records in an archive table + skipArchive: false + + # Table names to perform maintenance on. Supported values in: + # https://airflow.apache.org/docs/apache-airflow/stable/cli-and-env-variables-ref.html#clean + tables: [] + + # Maximum number of rows to delete or archive in a single transaction + batchSize: ~ + + # Make logging output more verbose + verbose: true + + # `jobAnnotations` are annotations on the database cleanup CronJob + jobAnnotations: {} + + # Select certain nodes for Airflow database cleanup pods. + nodeSelector: {} + affinity: {} + tolerations: [] + topologySpreadConstraints: [] + priorityClassName: ~ + + # Pod annotations for database cleanup pods (templated) + podAnnotations: {} + + # Labels specific to database cleanup objects and pods + labels: {} + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Create Service Account + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + automountServiceAccountToken: true + + # Specifies whether a Service Account should be created + create: true + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + name: ~ + + # Annotations to add to database cleanup CronJob Kubernetes Service Account. + annotations: {} + + env: [] + + # Detailed default security context for database cleanup for container level + securityContexts: + pod: {} + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + # Specify history limit + # When set, overwrite the default k8s number of successful and failed CronJob executions that are saved. + failedJobsHistoryLimit: 1 + successfulJobsHistoryLimit: 1 + + # Time to live (in seconds) for Jobs created by this CronJob after they finish. + ttlSecondsAfterFinished: ~ + +# Configuration for postgresql subchart +# Uses bitnamilegacy images to avoid Bitnami licensing restrictions +# Not recommended for production - use external database instead +postgresql: + enabled: true + image: + repository: bitnamilegacy/postgresql + tag: "16.1.0-debian-11-r15" + auth: + enablePostgresUser: true + postgresPassword: postgres + username: "" + password: "" + +# Config settings to go into the mounted airflow.cfg +# +# Please note that these values are passed through the `tpl` function, so are +# all subject to being rendered as go templates. If you need to include a +# literal `{{` in a value, it must be expressed like this: +# a: '{{ "{{ not a template }}" }}' +# +# Do not set config containing secrets via plain text values, use Env Var or k8s secret object +# yamllint disable rule:line-length +config: + core: + dags_folder: '{{ include "airflow_dags" . }}' + # This is ignored when used with the official Docker image + load_examples: 'False' + executor: '{{ .Values.executor }}' + auth_manager: "airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager" + logging: + remote_logging: '{{- ternary "True" "False" (or .Values.elasticsearch.enabled .Values.opensearch.enabled) }}' + colored_console_log: 'False' + metrics: + statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}' + statsd_port: 9125 + statsd_prefix: airflow + statsd_host: '{{ printf "%s-statsd" (include "airflow.fullname" .) }}' + fab: + enable_proxy_fix: 'True' + webserver: + # For Airflow 2.X + enable_proxy_fix: 'True' + celery: + flower_url_prefix: '{{ ternary "" .Values.ingress.flower.path (eq .Values.ingress.flower.path "/") }}' + worker_concurrency: 16 + sync_parallelism: '{{ include "cpu_count" (((.Values.scheduler).resources).limits).cpu }}' + scheduler: + standalone_dag_processor: '{{ ternary "True" "False" (or (semverCompare ">=3.0.0" .Values.airflowVersion) (.Values.dagProcessor.enabled | default false)) }}' + dag_processor: + # This value is generated by default from `.Values.dagProcessor.dagBundleConfigList` using the `dag_bundle_config_list` helper function. + # It is recommended to configure this via `dagProcessor.dagBundleConfigList` rather than overriding `config.dag_processor.dag_bundle_config_list` directly. + dag_bundle_config_list: '{{ include "dag_bundle_config_list" . }}' + elasticsearch: + json_format: 'True' + log_id_template: "{dag_id}-{task_id}-{run_id}-{map_index}-{try_number}" + elasticsearch_configs: + max_retries: 3 + timeout: 30 + retry_timeout: 'True' + kerberos: + keytab: '{{ .Values.kerberos.keytabPath }}' + reinit_frequency: '{{ .Values.kerberos.reinitFrequency }}' + principal: '{{ .Values.kerberos.principal }}' + ccache: '{{ .Values.kerberos.ccacheMountPath }}/{{ .Values.kerberos.ccacheFileName }}' + celery_kubernetes_executor: + kubernetes_queue: 'kubernetes' + kubernetes_executor: + namespace: '{{ .Release.Namespace }}' + pod_template_file: '{{ include "airflow_pod_template_file" . }}/pod_template_file.yaml' + worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}' + worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}' + multi_namespace_mode: '{{ ternary "True" "False" .Values.multiNamespaceMode }}' + +# yamllint enable rule:line-length + +# Whether Airflow can launch workers and/or pods in multiple namespaces +# If true, it creates ClusterRole/ClusterRolebinding (with access to entire cluster) +multiNamespaceMode: false + +# `podTemplate` is a templated string which overwrites the content of `pod_template_file.yaml` used by +# KubernetesExecutor. The default `podTemplate` will use `workers` configuration parameters +# (e.g. `workers.resources`). As such, you normally won't need to override this directly, however, +# you can still provide a completely custom `pod_template_file.yaml` if desired. +# If not set, a default one is created using `files/pod-template-file.kubernetes-helm-yaml`. +podTemplate: ~ +# The following example is NOT functional, but meant to be illustrative of how you can provide a custom +# `pod_template_file`. You're better off starting with the default in +# `files/pod-template-file.kubernetes-helm-yaml` and modifying from there. +# We will set `priorityClassName` in this example: +# podTemplate: | +# apiVersion: v1 +# kind: Pod +# metadata: +# name: placeholder-name +# labels: +# tier: airflow +# component: worker +# release: {{ .Release.Name }} +# spec: +# priorityClassName: high-priority +# containers: +# - name: base +# ... + +dags: + # Where dags volume will be mounted. Works for both persistence and gitSync. + # If not specified, dags mount path will be set to $AIRFLOW_HOME/dags + mountPath: ~ + persistence: + # Annotations for dags PVC + annotations: {} + + # Enable persistent volume for storing dags + enabled: false + + # Volume size for dags + size: 1Gi + + # If using a custom storageClass, pass name here + storageClassName: + + # Access mode of the persistent volume + accessMode: ReadWriteOnce + + # The name of an existing PVC to use + existingClaim: + + # Optional subpath for dag volume mount + subPath: ~ + + gitSync: + enabled: false + + # Git repo clone url + repo: https://github.com/apache/airflow.git + # SSH example: git@github.com:apache/airflow.git + # HTTPS example: https://github.com/apache/airflow.git + + branch: v2-2-stable + rev: HEAD + + # The git revision (branch, tag, or hash) to check out, v4 only + ref: v2-2-stable + + depth: 1 + + # The number of consecutive failures allowed before aborting + maxFailures: 0 + + # Subpath within the repo where dags are located. + # Should be "" if dags are at repo root + subPath: "tests/dags" + + # If your repo needs a username/password, you can load them to a k8s secret + # + # credentialsSecret: git-credentials + # + # Secret example: + # apiVersion: v1 + # kind: Secret + # metadata: + # name: git-credentials + # data: + # # For git-sync v3 + # GIT_SYNC_USERNAME: + # GIT_SYNC_PASSWORD: + # # For git-sync v4 + # GITSYNC_USERNAME: + # GITSYNC_PASSWORD: + + # If you are using an ssh clone url, you can load the ssh private key to a k8s secret + # + # sshKeySecret: airflow-ssh-secret + # + # Secret example: + # apiVersion: v1 + # kind: Secret + # metadata: + # name: airflow-ssh-secret + # data: + # gitSshKey: + + # If `sshKeySecret` is not specified, you can set `sshKey` + # sshKey: | + # -----BEGIN {OPENSSH PRIVATE KEY}----- + # ... + # -----END {OPENSSH PRIVATE KEY}----- + + # If you are using an ssh private key, you can additionally + # specify the content of your known_hosts file + # knownHosts: | + # , + # , + + # Interval between git sync attempts in seconds. + # High values are more likely to cause DAGs to become out of sync between different components. + # Low values cause more traffic to the remote git repository. + # Go-style duration string (e.g. "100ms" or "0.1s" = 100ms). + # For backwards compatibility, wait will be used if it is specified. + period: 5s + wait: ~ + + # Add variables from secret into gitSync containers, such proxy-config + envFrom: ~ + # envFrom: | + # - secretRef: + # name: 'proxy-config' + + containerName: git-sync + uid: 65533 + + # When not set, the values defined in the global `securityContext` will be used + # (deprecated, use `dags.gitSync.securityContexts` instead) + securityContext: {} + # runAsUser: 65533 + # runAsGroup: 0 + + securityContexts: + container: {} + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + # Git-Sync liveness service HTTP bind port + httpPort: 1234 + + # Setting this to true, will remove readinessProbe usage and configure livenessProbe to + # use a dedicated Git-Sync liveness service. In future, behaviour with value true will be + # default one and old one will be removed + recommendedProbeSetting: false + + startupProbe: + enabled: true + timeoutSeconds: 1 + initialDelaySeconds: 0 + periodSeconds: 5 + failureThreshold: 10 + + # As Git-Sync is not service-type object, the usage of this section will be removed. + # By setting `dags.gitSync.recommendedProbeSetting` to 'true', you will enable future behaviour. + readinessProbe: {} + + # The behaviour of the LivenessProbe will change with the next release of Helm Chart. + # To enable future behaviour set `dags.gitSync.recommendedProbeSetting` to 'true'. + # New behaviour uses the recommended liveness configuration by using Git-Sync built-in + # liveness service + livenessProbe: {} + # enabled: true + # timeoutSeconds: 1 + # initialDelaySeconds: 0 + # periodSeconds: 5 + # failureThreshold: 10 + + # Mount additional volumes into git-sync. + extraVolumeMounts: [] + # It can be templated like in the following example: + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + + # Supported env vars for gitsync can be found at https://github.com/kubernetes/git-sync + env: [] + # - name: "" + # value: "" + + # Configuration for empty dir volume + # emptyDirConfig: + # sizeLimit: 1Gi + # medium: Memory + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +logs: + # Configuration for empty dir volume (if `logs.persistence.enabled` == 'false') + # emptyDirConfig: + # sizeLimit: 1Gi + # medium: Memory + + persistence: + # Enable persistent volume for storing logs + enabled: false + + # Volume size for logs + size: 100Gi + + # Annotations for the logs PVC + annotations: {} + + # If using a custom storageClass, pass name here + storageClassName: + + # The name of an existing PVC to use + existingClaim: + + # The subpath of the existing PVC to use + subPath: diff --git a/charts/airflow/values_schema.schema.json b/charts/airflow/values_schema.schema.json new file mode 100644 index 0000000..64179d5 --- /dev/null +++ b/charts/airflow/values_schema.schema.json @@ -0,0 +1,104 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "description": "This schema is used to validate `values.schema.json` to ensure each parameter has `default` and `description` set, and that top level properties have a `x-docsSection` set.", + "definitions": { + "leafs": { + "additionalProperties": { + "if": { + "not": { + "properties": { + "description": { + "pattern": "^Labels for the configmap$|^Labels for the secret$|^Annotations for the configmap$|^Annotations for the secret$" + } + } + } + }, + "then": { + "additionalProperties": { + "$ref": "#/definitions/leafs" + } + } + }, + "if": { + "oneOf": [ + { + "properties": { + "type": { + "const": "integer" + } + } + }, + { + "properties": { + "type": { + "const": "number" + } + } + }, + { + "properties": { + "type": { + "const": "string" + } + } + }, + { + "properties": { + "type": { + "const": "boolean" + } + } + }, + { + "properties": { + "type": { + "const": "object" + }, + "properties": false + } + }, + { + "properties": { + "type": { + "const": "array" + }, + "items": false + } + } + ] + }, + "then": { + "required": [ + "description", + "default" + ] + } + } + }, + "required": [ + "x-docsSectionOrder" + ], + "properties": { + "section_order": { + "type": "array", + "items": { + "type": "string" + } + }, + "properties": { + "additionalProperties": { + "allOf": [ + { + "$ref": "#/definitions/leafs" + }, + { + "required": [ + "x-docsSection" + ] + } + ] + } + } + } +} diff --git a/manifests/air-flow/values.yaml b/manifests/air-flow/values.yaml new file mode 100644 index 0000000..6888260 --- /dev/null +++ b/manifests/air-flow/values.yaml @@ -0,0 +1,71 @@ +nodeSelector: + workload: general + +# Web UI Ingress Configuration +ingress: + web: + enabled: true + ingressClassName: traefik + annotations: + cert-manager.io/cluster-issuer: letsencrypt + traefik.ingress.kubernetes.io/router.entrypoints: web + hosts: + - name: airflow.dvirlabs.com + tls: + enabled: false + +# Flower (task monitoring) Ingress Configuration + flower: + enabled: true + ingressClassName: traefik + annotations: + cert-manager.io/cluster-issuer: letsencrypt + traefik.ingress.kubernetes.io/router.entrypoints: web + hosts: + - name: airflow-flower.dvirlabs.com + tls: + enabled: false + +# PostgreSQL Configuration +postgresql: + enabled: true + auth: + username: airflow + password: airflow123 + database: airflow + primary: + nodeSelector: + workload: general + persistence: + enabled: true + storageClass: nfs-client + size: 10Gi + +# Airflow Executor Configuration +executor: KubernetesExecutor + +# Redis Configuration for task queue +redis: + enabled: true + auth: + enabled: false + +# Airflow Home directory +airflowHome: /opt/airflow + +# Default resources for Airflow pods +resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + +# Statsd exporter for metrics +statsd: + enabled: true + +# Labels +labels: + app: airflow