Fix alias of DuckDBPyRelation.query

.github/actions/ccache-action/action.yml

@@ -0,0 +1,23 @@
+name: 'Setup Ccache'
+inputs:
+  key:
+    description: 'Cache key (defaults to github.job)'
+    required: false
+    default: ''
+runs:
+  using: "composite"
+  steps:
+     - name: Setup Ccache
+       uses: hendrikmuhs/ccache-action@main
+       with:
+         key: ${{ inputs.key || github.job }}
+         save: ${{ github.repository != 'duckdb/duckdb-python' || contains('["refs/heads/main", "refs/heads/v1.4-andium", "refs/heads/v1.5-variegata"]', github.ref) }}
+         # Dump verbose ccache statistics report at end of CI job.
+         verbose: 1
+         # Increase per-directory limit: 5*1024 MB / 16 = 320 MB.
+         # Note: `layout=subdirs` computes the size limit divided by 16 dirs.
+         # See also: https://ccache.dev/manual/4.9.html#_cache_size_management
+         max-size: 1500MB
+         # Evicts all cache files that were not touched during the job run.
+         # Removing cache files from previous runs avoids creating huge caches.
+         evict-old-files: 'job'

.github/workflows/packaging_wheels.yml

@@ -25,15 +25,112 @@ on:
         type: string
 
 jobs:
+  seed_wheels:
+    name: 'Seed: cp314-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}'
+    strategy:
+      fail-fast: false
+      matrix:
+        python: [ cp314 ]
+        platform:
+          - { os: windows-2022,     arch: amd64,      cibw_system: win }
+          - { os: windows-11-arm,   arch: ARM64,      cibw_system: win }
+          - { os: ubuntu-24.04,     arch: x86_64,     cibw_system: manylinux }
+          - { os: ubuntu-24.04-arm, arch: aarch64,    cibw_system: manylinux }
+          - { os: macos-15,         arch: arm64,      cibw_system: macosx }
+          - { os: macos-15,         arch: universal2, cibw_system: macosx }
+          - { os: macos-15-intel,   arch: x86_64,     cibw_system: macosx }
+        minimal:
+          - ${{ inputs.minimal }}
+        exclude:
+          - { minimal: true, platform: { arch: universal2 } }
+    runs-on: ${{ matrix.platform.os }}
+    env:
+      CCACHE_DIR: ${{ github.workspace }}/.ccache
+      ### cibuildwheel configuration
+      #
+      # This is somewhat brittle, so be careful with changes. Some notes for our future selves (and others):
+      # - cibw will change its cwd to a temp dir and create a separate venv for testing. It then installs the wheel it
+      #   built into that venv, and run the CIBW_TEST_COMMAND. We have to install all dependencies ourselves, and make
+      #   sure that the pytest config in pyproject.toml is available.
+      # - CIBW_BEFORE_TEST installs the test dependencies by exporting them into a pylock.toml. At the time of writing,
+      #   `uv sync --no-install-project` had problems correctly resolving dependencies using resolution environments
+      #   across all platforms we build for. This might be solved in newer uv versions.
+      # - CIBW_TEST_COMMAND specifies pytest conf from pyproject.toml. --confcutdir is needed to prevent pytest from
+      #   traversing the full filesystem, which produces an error on Windows.
+      # - CIBW_TEST_SKIP we always skip tests for *-macosx_universal2 builds, because we run tests for arm64 and x86_64.
+      CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }}
+      CIBW_TEST_SOURCES: tests
+      CIBW_BEFORE_TEST: >
+        uv export --only-group test --no-emit-project  --quiet --output-file pylock.toml --directory {project} &&
+        uv pip install -r pylock.toml
+      CIBW_TEST_COMMAND: >
+        uv run -v pytest --confcutdir=. --rootdir . -c {project}/pyproject.toml ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }}
+
+    steps:
+      - name: Checkout DuckDB Python
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.duckdb-python-sha }}
+          fetch-depth: 0
+          submodules: true
+
+      - name: Checkout DuckDB
+        shell: bash
+        if: ${{ inputs.duckdb-sha }}
+        run: |
+          cd external/duckdb
+          git fetch origin
+          git checkout ${{ inputs.duckdb-sha }}
+
+      - name: Set CIBW_ENVIRONMENT
+        shell: bash
+        run: |
+          cibw_env=""
+          if [[ "${{ matrix.platform.cibw_system }}" == "manylinux" ]]; then
+            cibw_env="CCACHE_DIR=/host${{ github.workspace }}/.ccache"
+          fi
+          if [[ -n "${{ inputs.set-version }}" ]]; then
+            cibw_env="${cibw_env:+$cibw_env }OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}"
+          fi
+          if [[ -n "$cibw_env" ]]; then
+            echo "CIBW_ENVIRONMENT=${cibw_env}" >> $GITHUB_ENV
+          fi
+
+      - name: Setup Ccache
+        uses: ./.github/actions/ccache-action
+        with:
+          key: ${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
+
+      # Install Astral UV, which will be used as build-frontend for cibuildwheel
+      - uses: astral-sh/setup-uv@v7
+        with:
+          version: "0.9.0"
+          enable-cache: false
+          cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
+
+      - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels
+        uses: pypa/cibuildwheel@v3.2
+        env:
+          CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }}
+          CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
+
+      - name: Upload wheel
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
+          path: wheelhouse/*.whl
+          compression-level: 0
+
   build_wheels:
     name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}'
+    needs: seed_wheels
     strategy:
       fail-fast: false
       matrix:
-        python: [ cp310, cp311, cp312, cp313, cp314 ]
+        python: [ cp310, cp311, cp312, cp313 ]
         platform:
-          - { os: windows-2022,     arch: amd64,      cibw_system: win }
-          - { os: windows-11-arm,   arch: ARM64,      cibw_system: win } # cibw requires ARM64 to be uppercase
+          - { os: windows-2025,     arch: amd64,      cibw_system: win }
+          - { os: windows-11-arm,   arch: ARM64,      cibw_system: win }
           - { os: ubuntu-24.04,     arch: x86_64,     cibw_system: manylinux }
           - { os: ubuntu-24.04-arm, arch: aarch64,    cibw_system: manylinux }
           - { os: macos-15,         arch: arm64,      cibw_system: macosx }
@@ -46,9 +143,10 @@ jobs:
           - { minimal: true, python: cp312 }
           - { minimal: true, python: cp313 }
           - { minimal: true, platform: { arch: universal2 } }
-          - { python: cp310,  platform: { os: windows-11-arm, arch: ARM64 } }  # too many dependency problems for win arm64
+          - { python: cp310,  platform: { os: windows-11-arm, arch: ARM64 } }
     runs-on: ${{ matrix.platform.os }}
     env:
+      CCACHE_DIR: ${{ github.workspace }}/.ccache
       ### cibuildwheel configuration
       #
       # This is somewhat brittle, so be careful with changes. Some notes for our future selves (and others):
@@ -85,11 +183,24 @@ jobs:
           git fetch origin
           git checkout ${{ inputs.duckdb-sha }}
 
-      # Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds
-      - name: Set OVERRIDE_GIT_DESCRIBE
+      - name: Set CIBW_ENVIRONMENT
         shell: bash
-        if: ${{ inputs.set-version != '' }}
-        run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV
+        run: |
+          cibw_env=""
+          if [[ "${{ matrix.platform.cibw_system }}" == "manylinux" ]]; then
+            cibw_env="CCACHE_DIR=/host${{ github.workspace }}/.ccache"
+          fi
+          if [[ -n "${{ inputs.set-version }}" ]]; then
+            cibw_env="${cibw_env:+$cibw_env }OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}"
+          fi
+          if [[ -n "$cibw_env" ]]; then
+            echo "CIBW_ENVIRONMENT=${cibw_env}" >> $GITHUB_ENV
+          fi
+
+      - name: Setup Ccache
+        uses: ./.github/actions/ccache-action
+        with:
+          key: ${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
 
       # Install Astral UV, which will be used as build-frontend for cibuildwheel
       - uses: astral-sh/setup-uv@v7

.github/workflows/release.yml

@@ -39,7 +39,6 @@ defaults:
 jobs:
   build_sdist:
     name: Build an sdist and determine versions
-    if: ${{ github.ref != 'refs/heads/main' }}
     uses: ./.github/workflows/packaging_sdist.yml
     with:
       testsuite: all

CMakeLists.txt

@@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.29)
 
 project(duckdb_py LANGUAGES CXX)
 
-# Always use C++11
-set(CMAKE_CXX_STANDARD 11)
+# Always use C++17
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")

_duckdb-stubs/__init__.pyi

@@ -537,7 +537,9 @@ class DuckDBPyRelation:
     def distinct(self) -> DuckDBPyRelation: ...
     def except_(self, other_rel: Self) -> DuckDBPyRelation: ...
     def execute(self) -> DuckDBPyRelation: ...
-    def explain(self, type: ExplainType | ExplainTypeLiteral = ExplainType.STANDARD) -> str: ...
+    def explain(
+        self, type: ExplainType | ExplainTypeLiteral = ExplainType.STANDARD, format: str | None = None
+    ) -> str: ...
     def favg(
         self, expression: str, groups: str = "", window_spec: str = "", projected_columns: str = ""
     ) -> DuckDBPyRelation: ...

duckdb/experimental/spark/sql/type_utils.py

@@ -19,6 +19,7 @@
     IntegerType,
     LongType,
     MapType,
+    NullType,
     ShortType,
     StringType,
     StructField,
@@ -27,6 +28,7 @@
     TimeNTZType,
     TimestampMillisecondNTZType,
     TimestampNanosecondNTZType,
+    TimestampNanosecondType,
     TimestampNTZType,
     TimestampSecondNTZType,
     TimestampType,
@@ -41,6 +43,7 @@
 )
 
 _sqltype_to_spark_class = {
+    "null": NullType,
     "boolean": BooleanType,
     "utinyint": UnsignedByteType,
     "tinyint": ByteType,
@@ -62,9 +65,10 @@
     "time with time zone": TimeType,
     "timestamp": TimestampNTZType,
     "timestamp with time zone": TimestampType,
-    "timestamp_ms": TimestampNanosecondNTZType,
-    "timestamp_ns": TimestampMillisecondNTZType,
+    "timestamp_ms": TimestampMillisecondNTZType,
+    "timestamp_ns": TimestampNanosecondNTZType,
     "timestamp_s": TimestampSecondNTZType,
+    "timestamptz_ns": TimestampNanosecondType,
     "interval": DayTimeIntervalType,
     "list": ArrayType,
     "struct": StructType,

duckdb/experimental/spark/sql/types.py

@@ -49,6 +49,7 @@
     "TimestampMillisecondNTZType",
     "TimestampNTZType",
     "TimestampNanosecondNTZType",
+    "TimestampNanosecondType",
     "TimestampSecondNTZType",
     "TimestampType",
     "UUIDType",
@@ -239,6 +240,26 @@ def fromInternal(self, ts: int) -> datetime.datetime:  # noqa: D102
             return datetime.datetime.fromtimestamp(ts // 1000000).replace(microsecond=ts % 1000000)
 
 
+class TimestampNanosecondType(AtomicType, metaclass=DataTypeSingleton):
+    """Timestamp (datetime.datetime) data type with timezone information with nanosecond precision."""
+
+    def __init__(self) -> None:  # noqa: D107
+        super().__init__(DuckDBPyType("TIMESTAMPTZ_NS"))
+
+    def needConversion(self) -> bool:  # noqa: D102
+        return True
+
+    @classmethod
+    def typeName(cls) -> str:  # noqa: D102
+        return "timestamptz_ns"
+
+    def toInternal(self, dt: datetime.datetime) -> int:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def fromInternal(self, ts: int) -> datetime.datetime:  # noqa: D102
+        raise ContributionsAcceptedError
+
+
 class TimestampNTZType(AtomicType, metaclass=DataTypeSingleton):
     """Timestamp (datetime.datetime) data type without timezone information with microsecond precision."""
 

duckdb_packaging/setuptools_scm_version.py

@@ -12,7 +12,7 @@
 from ._versioning import format_version, parse_version
 
 # MAIN_BRANCH_VERSIONING should be 'True' on main branch only
-MAIN_BRANCH_VERSIONING = False
+MAIN_BRANCH_VERSIONING = True
 
 SCM_PRETEND_ENV_VAR = "SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DUCKDB"
 SCM_GLOBAL_PRETEND_ENV_VAR = "SETUPTOOLS_SCM_PRETEND_VERSION"

pyproject.toml

@@ -88,6 +88,12 @@ version_scheme = "duckdb_packaging.setuptools_scm_version:version_scheme"
 local_scheme = "no-local-version"
 fallback_version = "0.0.1.dev1"
 
+# main only: count dev distance from the last *minor* tag (v*.*.0), so a patch
+# tag (e.g. v1.5.4) merged in from a release branch can't reset .devN.
+# Release branches must NOT have this, they correctly count from v*.*.* (the default).
+[tool.setuptools_scm.scm.git]
+describe_command = "git describe --dirty --tags --long --abbrev=40 --match v*.*.0"
+
 # Override: if COVERAGE is set then:
 # - we create a RelWithDebInfo build
 # - we make sure we use a persistent build dir so we get access to the .gcda files
@@ -122,7 +128,7 @@ cmake.build-type = "Debug"
 [[tool.scikit-build.overrides]]
 if.state = "editable"
 if.env.COVERAGE = false
-if.platform-system = "Darwin"
+if.platform-system = "(?i)darwin"
 inherit.cmake.define = "append"
 cmake.define.DISABLE_UNITY = "1"
 
@@ -333,7 +339,7 @@ packages = ["duckdb", "_duckdb"]
 strict = true
 warn_unreachable = true
 pretty = true
-python_version = "3.10"
+python_version = "3.12"
 exclude = [
     "duckdb/experimental/", # not checking the pyspark API
     "duckdb/query_graph/",  # old and unmaintained (should probably remove)

scripts/cache_data.json

@@ -532,7 +532,9 @@
             "polars.DataFrame",
             "polars.LazyFrame",
             "polars.col",
-            "polars.lit"
+            "polars.lit",
+            "polars.Series",
+            "polars.Decimal"
         ],
         "required": false
     },
@@ -822,5 +824,17 @@
         "full_path": "polars.lit",
         "name": "lit",
         "children": []
+    },
+    "polars.Series": {
+        "type": "attribute",
+        "full_path": "polars.Series",
+        "name": "Series",
+        "children": []
+    },
+    "polars.Decimal": {
+        "type": "attribute",
+        "full_path": "polars.Decimal",
+        "name": "Decimal",
+        "children": []
     }
 }

scripts/imports.py

@@ -111,6 +111,8 @@
 polars.LazyFrame
 polars.col
 polars.lit
+polars.Series
+polars.Decimal
 
 import duckdb
 import duckdb.filesystem

src/duckdb_py/arrow/CMakeLists.txt

@@ -1,6 +1,7 @@
 # this is used for clang-tidy checks
 add_library(
-  python_arrow OBJECT arrow_array_stream.cpp arrow_export_utils.cpp
-                      polars_filter_pushdown.cpp pyarrow_filter_pushdown.cpp)
+  python_arrow OBJECT
+  arrow_array_stream.cpp arrow_export_utils.cpp filter_pushdown_visitor.cpp
+  polars_filter_pushdown.cpp pyarrow_filter_pushdown.cpp)
 
 target_link_libraries(python_arrow PRIVATE _duckdb_dependencies)