Jinja2 Support: Environment variables, logic and loops

Since version 0.4.1 dbx supports Jinja2 rendering for JSON and YAML based configurations. This allows you to use environment variables in the deployment, add variable-based conditions, Jinja filters and for loops to make your deployment more flexible for CI pipelines.

To add Jinja2 support to your deployment file, please add postfix .j2 to the name of your deployment file, for example deployment.yml.j2. Deployment files stored at conf/deployment.(json|yml|yaml).j2. will be auto-discovered.

Please find examples on how to use Jinja2 templates below:

deployment.json.j2deployment.yml.j2

{
  "default": {
    "jobs": [
      {
        "name": "your-job-name",
        "timeout_seconds": "{{ TIMEOUT }}",
          {% if (ENVIRONMENT == "production") %}
            "email_notifications": {
              "on_failure": [
                "presetEmail@test.com",
                "test@test.com"
              ]
            },
          {% endif %}
        "new_cluster": {
          "spark_version": "7.3.x-cpu-ml-scala2.12",
          "node_type_id": "some-node-type",
          "aws_attributes": {
            "first_on_demand": 0,
            "availability": "{{ AVAILABILITY | default('SPOT') }}"
          },
          "num_workers": 2
        },
        "libraries": [],
          {% if (ENVIRONMENT == "production") %}
            "max_retries": {{ MAX_RETRY | default(-1) }},
          {% else %}
            "max_retries": {{ MAX_RETRY | default(3) }},
          {% endif %}
        "spark_python_task": {
          "python_file": "tests/deployment-configs/placeholder_1.py"
        }
      }
    ]
  }
}