Support serverless mode in default-python template

acceptance/bundle/templates/default-python/README.md

@@ -0,0 +1,6 @@
+The 'serverless' and 'classic' directories contain full tests: they
+have full output of materialized template, perform "bundle validate"
+and in the future will perform deploy/summary/run.
+
+Other directories (serverless-auto-\*) contain short tests: they only do
+"bundle init" and then check that the output matches 'serverless' or 'classic' exactly.

acceptance/bundle/templates/default-python/input.json → acceptance/bundle/templates/default-python/classic/input.json

@@ -2,5 +2,6 @@
     "project_name": "my_default_python",
     "include_notebook": "yes",
     "include_dlt": "yes",
-    "include_python": "yes"
+    "include_python": "yes",
+    "serverless": "no"
 }

acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff

@@ -0,0 +1,48 @@
+--- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/resources/my_default_python.job.yml
++++ output/my_default_python/resources/my_default_python.job.yml
+@@ -17,4 +17,5 @@
+       tasks:
+         - task_key: notebook_task
++          job_cluster_key: job_cluster
+           notebook_task:
+             notebook_path: ../src/notebook.ipynb
+@@ -29,17 +30,20 @@
+           depends_on:
+             - task_key: refresh_pipeline
+-          environment_key: default
++          job_cluster_key: job_cluster
+           python_wheel_task:
+             package_name: my_default_python
+             entry_point: main
++          libraries:
++            # By default we just include the .whl file generated for the my_default_python package.
++            # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
++            # for more information on how to add other libraries.
++            - whl: ../dist/*.whl
+
+-      # A list of task execution environment specifications that can be referenced by tasks of this job.
+-      environments:
+-        - environment_key: default
+-
+-          # Full documentation of this spec can be found at:
+-          # https://docs.databricks.com/api/workspace/jobs/create#environments-spec
+-          spec:
+-            client: "1"
+-            dependencies:
+-              - ../dist/*.whl
++      job_clusters:
++        - job_cluster_key: job_cluster
++          new_cluster:
++            spark_version: 15.4.x-scala2.12
++            node_type_id: i3.xlarge
++            autoscale:
++                min_workers: 1
++                max_workers: 4
+--- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/resources/my_default_python.pipeline.yml
++++ output/my_default_python/resources/my_default_python.pipeline.yml
+@@ -6,5 +6,4 @@
+       catalog: main
+       target: my_default_python_${bundle.target}
+-      serverless: true
+       libraries:
+         - notebook:

acceptance/bundle/templates/default-python/classic/script

@@ -0,0 +1,13 @@
+trace $CLI bundle init default-python --config-file ./input.json --output-dir output
+
+cd output/my_default_python
+trace $CLI bundle validate -t dev
+trace $CLI bundle validate -t prod
+
+# Do not affect this repository's git behaviour #2318
+mv .gitignore out.gitignore
+
+cd ../../
+
+# Calculate the difference from the serverless template
+diff.py $TESTDIR/../serverless/output output/ > out.compare-vs-serverless.diff

acceptance/bundle/templates/default-python/serverless-auto-disabled/input.json

@@ -0,0 +1,7 @@
+{
+    "project_name": "my_default_python",
+    "include_notebook": "yes",
+    "include_dlt": "yes",
+    "include_python": "yes",
+    "serverless": "auto"
+}

acceptance/bundle/templates/default-python/serverless-auto-disabled/output.txt

@@ -0,0 +1,11 @@
+
+>>> [CLI] bundle init default-python --config-file ./input.json --output-dir output
+
+Welcome to the default Python template for Databricks Asset Bundles!
+Warn: Auto-detected that serverless is not enabled for your workspace 900800700600
+Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): [DATABRICKS_URL]
+
+✨ Your new project has been created in the 'my_default_python' directory!
+
+Please refer to the README.md file for "getting started" instructions.
+See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.

acceptance/bundle/templates/default-python/serverless-auto-disabled/script

@@ -0,0 +1,7 @@
+trace $CLI bundle init default-python --config-file ./input.json --output-dir output
+mv output/my_default_python/.gitignore output/my_default_python/out.gitignore
+
+# serverless is disabled, same output as for classic
+diff.py $TESTDIR/../classic/output output/
+
+rm -fr output

acceptance/bundle/templates/default-python/serverless-auto-disabled/test.toml

@@ -0,0 +1,3 @@
+[[Server]]
+Pattern = "GET /api/2.0/settings-api/workspace/{workspaceId}/serverless_job_nb"
+Response.Body = '{}'

acceptance/bundle/templates/default-python/serverless-auto-enabled/input.json

@@ -0,0 +1,7 @@
+{
+    "project_name": "my_default_python",
+    "include_notebook": "yes",
+    "include_dlt": "yes",
+    "include_python": "yes",
+    "serverless": "auto"
+}

acceptance/bundle/templates/default-python/serverless-auto-enabled/output.txt

@@ -0,0 +1,10 @@
+
+>>> [CLI] bundle init default-python --config-file ./input.json --output-dir output
+
+Welcome to the default Python template for Databricks Asset Bundles!
+Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): [DATABRICKS_URL]
+
+✨ Your new project has been created in the 'my_default_python' directory!
+
+Please refer to the README.md file for "getting started" instructions.
+See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.

acceptance/bundle/templates/default-python/serverless-auto-enabled/script

@@ -0,0 +1,7 @@
+trace $CLI bundle init default-python --config-file ./input.json --output-dir output
+mv output/my_default_python/.gitignore output/my_default_python/out.gitignore
+
+# serverless is enabled, same output as serverless enabled explicitly:
+diff.py $TESTDIR/../serverless/output output/
+
+rm -fr output

acceptance/bundle/templates/default-python/serverless-auto-enabled/test.toml

@@ -0,0 +1,3 @@
+[[Server]]
+Pattern = "GET /api/2.0/settings-api/workspace/{workspaceId}/serverless_job_nb"
+Response.Body = '''{"setting": {"value": {"preview_enablement_val": {"enabled": true}}}}'''

acceptance/bundle/templates/default-python/serverless-auto-error/input.json

@@ -0,0 +1,7 @@
+{
+    "project_name": "my_default_python",
+    "include_notebook": "yes",
+    "include_dlt": "yes",
+    "include_python": "yes",
+    "serverless": "auto"
+}

acceptance/bundle/templates/default-python/serverless-auto-error/output.txt

@@ -0,0 +1,11 @@
+
+>>> [CLI] bundle init default-python --config-file ./input.json --output-dir output
+
+Welcome to the default Python template for Databricks Asset Bundles!
+Warn: Failed to detect if serverless is supported: /api/2.0/settings-api/workspace/900800700600/serverless_job_nb failed: Internal Server Error
+Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): [DATABRICKS_URL]
+
+✨ Your new project has been created in the 'my_default_python' directory!
+
+Please refer to the README.md file for "getting started" instructions.
+See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.

acceptance/bundle/templates/default-python/serverless-auto-error/script

@@ -0,0 +1,6 @@
+trace $CLI bundle init default-python --config-file ./input.json --output-dir output
+mv output/my_default_python/.gitignore output/my_default_python/out.gitignore
+
+# If serverless is "auto" and there was an error reaching settings-api, we should get the same output as 'classic'
+diff.py $TESTDIR/../classic/output output/
+rm -fr output

acceptance/bundle/templates/default-python/serverless-auto-error/test.toml

@@ -0,0 +1,3 @@
+[[Server]]
+Pattern = "GET /api/2.0/settings-api/workspace/{workspaceId}/serverless_job_nb"
+Response.StatusCode = 500

acceptance/bundle/templates/default-python/serverless/input.json

@@ -0,0 +1,7 @@
+{
+    "project_name": "my_default_python",
+    "include_notebook": "yes",
+    "include_dlt": "yes",
+    "include_python": "yes",
+    "serverless": "yes"
+}

acceptance/bundle/templates/default-python/serverless/output.txt

@@ -0,0 +1,30 @@
+
+>>> [CLI] bundle init default-python --config-file ./input.json --output-dir output
+
+Welcome to the default Python template for Databricks Asset Bundles!
+Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): [DATABRICKS_URL]
+
+✨ Your new project has been created in the 'my_default_python' directory!
+
+Please refer to the README.md file for "getting started" instructions.
+See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
+
+>>> [CLI] bundle validate -t dev
+Name: my_default_python
+Target: dev
+Workspace:
+  Host: [DATABRICKS_URL]
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev
+
+Validation OK!
+
+>>> [CLI] bundle validate -t prod
+Name: my_default_python
+Target: prod
+Workspace:
+  Host: [DATABRICKS_URL]
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod
+
+Validation OK!

acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/__builtins__.pyi

@@ -0,0 +1,3 @@
+# Typings for Pylance in Visual Studio Code
+# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md
+from databricks.sdk.runtime import *

acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "databricks.databricks",
+        "ms-python.vscode-pylance",
+        "redhat.vscode-yaml"
+    ]
+}

acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/settings.json

@@ -0,0 +1,16 @@
+{
+    "python.analysis.stubPath": ".vscode",
+    "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\<codecell\\>|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])",
+    "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------",
+    "python.testing.pytestArgs": [
+        "."
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true,
+    "python.analysis.extraPaths": ["src"],
+    "files.exclude": {
+        "**/*.egg-info": true,
+        "**/__pycache__": true,
+        ".pytest_cache": true,
+    },
+}

acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md

@@ -0,0 +1,49 @@
+# my_default_python
+
+The 'my_default_python' project was generated by using the default-python template.
+
+## Getting started
+
+1. Install the Databricks CLI from https://docs.databricks.com/dev-tools/cli/databricks-cli.html
+
+2. Authenticate to your Databricks workspace, if you have not done so already:
+    ```
+    $ databricks configure
+    ```
+
+3. To deploy a development copy of this project, type:
+    ```
+    $ databricks bundle deploy --target dev
+    ```
+    (Note that "dev" is the default target, so the `--target` parameter
+    is optional here.)
+
+    This deploys everything that's defined for this project.
+    For example, the default template would deploy a job called
+    `[dev yourname] my_default_python_job` to your workspace.
+    You can find that job by opening your workpace and clicking on **Workflows**.
+
+4. Similarly, to deploy a production copy, type:
+   ```
+   $ databricks bundle deploy --target prod
+   ```
+
+   Note that the default job from the template has a schedule that runs every day
+   (defined in resources/my_default_python.job.yml). The schedule
+   is paused when deploying in development mode (see
+   https://docs.databricks.com/dev-tools/bundles/deployment-modes.html).
+
+5. To run a job or pipeline, use the "run" command:
+   ```
+   $ databricks bundle run
+   ```
+6. Optionally, install the Databricks extension for Visual Studio code for local development from
+   https://docs.databricks.com/dev-tools/vscode-ext.html. It can configure your
+   virtual environment and setup Databricks Connect for running unit tests locally.
+   When not using these tools, consult your development environment's documentation
+   and/or the documentation for Databricks Connect for manually setting up your environment
+   (https://docs.databricks.com/en/dev-tools/databricks-connect/python/index.html).
+
+7. For documentation on the Databricks asset bundles format used
+   for this project, and for CI/CD configuration, see
+   https://docs.databricks.com/dev-tools/bundles/index.html.

acceptance/bundle/templates/default-python/serverless/output/my_default_python/databricks.yml

@@ -0,0 +1,29 @@
+# This is a Databricks asset bundle definition for my_default_python.
+# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
+bundle:
+  name: my_default_python
+  uuid: [UUID]
+
+include:
+  - resources/*.yml
+
+targets:
+  dev:
+    # The default target uses 'mode: development' to create a development copy.
+    # - Deployed resources get prefixed with '[dev my_user_name]'
+    # - Any job schedules and triggers are paused by default.
+    # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.
+    mode: development
+    default: true
+    workspace:
+      host: [DATABRICKS_URL]
+
+  prod:
+    mode: production
+    workspace:
+      host: [DATABRICKS_URL]
+      # We explicitly deploy to /Workspace/Users/[USERNAME] to make sure we only have a single copy.
+      root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target}
+    permissions:
+      - user_name: [USERNAME]
+        level: CAN_MANAGE

acceptance/bundle/templates/default-python/serverless/output/my_default_python/fixtures/.gitkeep

@@ -0,0 +1,22 @@
+# Fixtures
+
+This folder is reserved for fixtures, such as CSV files.
+
+Below is an example of how to load fixtures as a data frame:
+
+```
+import pandas as pd
+import os
+
+def get_absolute_path(*relative_parts):
+    if 'dbutils' in globals():
+        base_dir = os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) # type: ignore
+        path = os.path.normpath(os.path.join(base_dir, *relative_parts))
+        return path if path.startswith("/Workspace") else "/Workspace" + path
+    else:
+        return os.path.join(*relative_parts)
+
+csv_file = get_absolute_path("..", "fixtures", "mycsv.csv")
+df = pd.read_csv(csv_file)
+display(df)
+```

acceptance/bundle/templates/default-python/serverless/output/my_default_python/out.gitignore

@@ -0,0 +1,8 @@
+.databricks/
+build/
+dist/
+__pycache__/
+*.egg-info
+.venv/
+scratch/**
+!scratch/README.md

acceptance/bundle/templates/default-python/serverless/output/my_default_python/pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+testpaths = tests
+pythonpath = src

acceptance/bundle/templates/default-python/serverless/output/my_default_python/requirements-dev.txt

@@ -0,0 +1,29 @@
+## requirements-dev.txt: dependencies for local development.
+##
+## For defining dependencies used by jobs in Databricks Workflows, see
+## https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
+
+## Add code completion support for DLT
+databricks-dlt
+
+## pytest is the default package used for testing
+pytest
+
+## Dependencies for building wheel files
+setuptools
+wheel
+
+## databricks-connect can be used to run parts of this project locally.
+## See https://docs.databricks.com/dev-tools/databricks-connect.html.
+##
+## databricks-connect is automatically installed if you're using Databricks
+## extension for Visual Studio Code
+## (https://docs.databricks.com/dev-tools/vscode-ext/dev-tasks/databricks-connect.html).
+##
+## To manually install databricks-connect, either follow the instructions
+## at https://docs.databricks.com/dev-tools/databricks-connect.html
+## to install the package system-wide. Or uncomment the line below to install a
+## version of db-connect that corresponds to the Databricks Runtime version used
+## for this project.
+#
+# databricks-connect>=15.4,<15.5