From 84750821635603adfe0c3d3f2ffac5706af97f2c Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Sat, 24 Aug 2024 21:12:05 +0200 Subject: [PATCH 1/4] Use periodic triggers in all templates --- .../resources/{{.project_name}}_job.yml.tmpl | 8 ++++---- .../resources/{{.project_name}}_job.yml.tmpl | 8 ++++---- .../resources/{{.project_name}}_sql_job.yml.tmpl | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index bad12c755c7..43fd2e5c82a 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -3,10 +3,10 @@ resources: {{.project_name}}_job: name: {{.project_name}}_job - schedule: - # Run every day at 9:27 AM - quartz_cron_expression: 21 27 9 * * ? - timezone_id: UTC + trigger: + periodic: + interval: 1 + unit: DAYS email_notifications: on_failure: diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index dc79e3a1719..0350d42cfb3 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -10,10 +10,10 @@ resources: {{.project_name}}_job: name: {{.project_name}}_job - schedule: - # Run every day at 8:37 AM - quartz_cron_expression: '44 37 8 * * ?' - timezone_id: Europe/Amsterdam + trigger: + periodic: + interval: 1 + unit: DAYS {{- if not is_service_principal}} diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl index 31d2d21a9f8..2e6004186c9 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl @@ -4,10 +4,10 @@ resources: {{.project_name}}_sql_job: name: {{.project_name}}_sql_job - schedule: - # Run every day at 7:17 AM - quartz_cron_expression: '44 17 7 * * ?' - timezone_id: Europe/Amsterdam + trigger: + periodic: + interval: 1 + unit: DAYS {{- if not is_service_principal}} From 1c34dc92a43924287cd6a6f94fd1b1ab0c5c3ec0 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Wed, 4 Sep 2024 16:07:21 +0200 Subject: [PATCH 2/4] Add comments --- .../{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl | 1 + .../{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl | 1 + .../resources/{{.project_name}}_sql_job.yml.tmpl | 1 + 3 files changed, 3 insertions(+) diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index 43fd2e5c82a..edeafd9b1d9 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -4,6 +4,7 @@ resources: name: {{.project_name}}_job trigger: + # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create periodic: interval: 1 unit: DAYS diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index 0350d42cfb3..29a93258249 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -11,6 +11,7 @@ resources: name: {{.project_name}}_job trigger: + # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create periodic: interval: 1 unit: DAYS diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl index 2e6004186c9..31bdf72d54d 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl @@ -5,6 +5,7 @@ resources: name: {{.project_name}}_sql_job trigger: + # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create periodic: interval: 1 unit: DAYS From baaa234b43104d5ce893737c0e63fcef74eec2b5 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Mon, 9 Sep 2024 13:49:57 +0200 Subject: [PATCH 3/4] Extend URL with an anchor --- .../{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl | 2 +- .../{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl | 2 +- .../resources/{{.project_name}}_sql_job.yml.tmpl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index edeafd9b1d9..c4520f769d3 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -4,7 +4,7 @@ resources: name: {{.project_name}}_job trigger: - # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create + # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create#trigger periodic: interval: 1 unit: DAYS diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index 29a93258249..416e9efcb7d 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -11,7 +11,7 @@ resources: name: {{.project_name}}_job trigger: - # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create + # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create#trigger periodic: interval: 1 unit: DAYS diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl index 31bdf72d54d..54c995ea516 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl @@ -5,7 +5,7 @@ resources: name: {{.project_name}}_sql_job trigger: - # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create + # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create#trigger periodic: interval: 1 unit: DAYS From a3c2c5b73591ffaf08228c91e656a356831e30bc Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Tue, 10 Sep 2024 14:19:47 +0200 Subject: [PATCH 4/4] Extend comment --- .../{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl | 2 +- .../{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl | 2 +- .../resources/{{.project_name}}_sql_job.yml.tmpl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index c4520f769d3..e23c8dbced7 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -4,7 +4,7 @@ resources: name: {{.project_name}}_job trigger: - # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create#trigger + # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger periodic: interval: 1 unit: DAYS diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index 416e9efcb7d..d2100e9084a 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -11,7 +11,7 @@ resources: name: {{.project_name}}_job trigger: - # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create#trigger + # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger periodic: interval: 1 unit: DAYS diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl index 54c995ea516..4e6803da97a 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl @@ -5,7 +5,7 @@ resources: name: {{.project_name}}_sql_job trigger: - # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create#trigger + # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger periodic: interval: 1 unit: DAYS