From 99c66c965f402781791515494860be8dc143c57d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Tue, 28 Jul 2026 12:29:08 +0100 Subject: [PATCH 1/4] MINOR: Build test artifacts once in CI --- .github/workflows/test.yml | 228 +++++++++++++++++++++++++++++-------- ci/scripts/build.sh | 6 +- ci/scripts/test.sh | 68 +++++++++-- compose.yaml | 85 ++++++++++++++ 4 files changed, 330 insertions(+), 57 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 653b16fa32..276209b987 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,25 +37,11 @@ env: DOCKER_VOLUME_PREFIX: ".docker/" jobs: - ubuntu: - name: AMD64 ${{ matrix.name }} JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} + build-java: + name: Build Java artifacts runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - jdk: [17, 21, 23] - maven: [3.9.9] - image: [ubuntu, conda-jni-cdata] - include: - - image: ubuntu - name: "Ubuntu" - - image: conda-jni-cdata - name: "Conda JNI" - env: - JDK: ${{ matrix.jdk }} - MAVEN: ${{ matrix.maven }} steps: - name: Checkout Arrow uses: actions/checkout@v7 @@ -65,34 +51,75 @@ jobs: - name: Cache Docker Volumes uses: actions/cache@v6 with: - path: .docker - key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} - restore-keys: maven-${{ matrix.jdk }}-${{ matrix.maven }}- - - name: Execute Docker Build + path: .docker/maven-cache + key: java-build-17-3.9.9-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: java-build-17-3.9.9- + - name: Build without tests + shell: bash env: - # Enables build caching, but not strictly required DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + JDK: 17 + MAVEN: 3.9.9 run: | docker compose run \ + --rm \ -e CI=true \ -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ - ${{ matrix.image }} + ubuntu-artifacts + if [ -d .docker ]; then + sudo chown -R "$(id -u):$(id -g)" .docker + fi + - name: Pack reusable artifacts + shell: bash + run: | + tar -czf java-build.tgz \ + .docker/java-build \ + .docker/jni-dist \ + .docker/maven-cache/repository/org/apache/arrow + - name: Upload reusable artifacts + uses: actions/upload-artifact@v5 + with: + name: java-build + path: java-build.tgz + retention-days: 1 - macos: - name: ${{ matrix.arch }} macOS ${{ matrix.macos }} Java JDK ${{ matrix.jdk }} - runs-on: macos-${{ matrix.macos }} + test-java: + name: ${{ matrix.name || format('AMD64 Ubuntu JDK {0} Maven {1}', matrix.jdk, matrix.maven) }} + needs: build-java + runs-on: ${{ matrix.os }} if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 30 strategy: fail-fast: false matrix: + target: [ubuntu] + jdk: [17, 21, 23] + maven: [3.9.9] include: - - arch: AMD64 + - target: ubuntu + os: ubuntu-latest + compose_service: ubuntu-test-prebuilt + - target: macos-intel + name: AMD64 macOS 15-intel Java JDK 17 + os: macos-15-intel jdk: 17 - macos: 15-intel - - arch: AArch64 + maven: 3.9.9 + compose_service: '' + - target: macos-arm + name: AArch64 macOS latest Java JDK 17 + os: macos-latest jdk: 17 - macos: latest + maven: 3.9.9 + compose_service: '' + - target: windows + name: AMD64 Windows Server 2022 Java JDK 17 + os: windows-latest + jdk: 17 + maven: 3.9.9 + compose_service: '' + env: + JDK: ${{ matrix.jdk }} + MAVEN: ${{ matrix.maven }} steps: - name: Checkout Arrow uses: actions/checkout@v7 @@ -100,50 +127,153 @@ jobs: fetch-depth: 0 submodules: recursive - name: Set up Java + if: ${{ matrix.compose_service == '' }} uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: ${{ matrix.jdk }} cache: 'maven' - - name: Build + - name: Cache Docker Volumes + if: ${{ matrix.compose_service != '' }} + uses: actions/cache@v6 + with: + path: .docker/maven-cache + key: java-build-17-3.9.9-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: java-build-17-3.9.9- + - name: Download reusable artifacts + uses: actions/download-artifact@v6 + with: + name: java-build + - name: Restore reusable artifacts + shell: bash + run: | + mkdir -p .docker + rm -rf \ + .docker/java-build \ + .docker/jni-dist \ + .docker/maven-cache/repository/org/apache/arrow + tar -xzf java-build.tgz + - name: Restore reusable artifacts for hosted runner + if: ${{ matrix.compose_service == '' }} + shell: bash + run: | + rm -rf build jni + cp -a .docker/java-build/build build + cp -a .docker/jni-dist jni + mkdir -p "${HOME}/.m2/repository/org/apache" + cp -a .docker/maven-cache/repository/org/apache/arrow "${HOME}/.m2/repository/org/apache/" + - name: Test prebuilt artifacts on macOS/Windows + if: ${{ matrix.compose_service == '' }} shell: bash env: + ARROW_JAVA_TEST_PREBUILT: "ON" DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: ci/scripts/build.sh . build jni - - name: Test + run: ci/scripts/test.sh . build jni + - name: Test prebuilt artifacts on Ubuntu Docker + if: ${{ matrix.compose_service != '' }} shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: ci/scripts/test.sh . build jni + run: | + docker compose run \ + --rm \ + -e CI=true \ + -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ + ${{ matrix.compose_service }} - windows: - name: AMD64 Windows Server 2022 Java JDK ${{ matrix.jdk }} - runs-on: windows-latest + build-cdata: + name: Build C Data artifacts + runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - jdk: [17] + env: + JDK: 17 + MAVEN: 3.9.9 steps: - name: Checkout Arrow uses: actions/checkout@v7 with: fetch-depth: 0 submodules: recursive - - name: Set up Java - uses: actions/setup-java@v5 + - name: Cache Docker Volumes + uses: actions/cache@v6 with: - java-version: ${{ matrix.jdk }} - distribution: 'temurin' - cache: 'maven' - - name: Build + path: .docker/maven-cache + key: cdata-${{ env.JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: cdata-${{ env.JDK }}-${{ env.MAVEN }}- + - name: Build C Data without tests shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: ci/scripts/build.sh . build jni - - name: Test + run: | + docker compose run \ + --rm \ + -e CI=true \ + -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ + cdata-artifacts + if [ -d .docker ]; then + sudo chown -R "$(id -u):$(id -g)" .docker + fi + - name: Pack reusable C Data artifacts + shell: bash + run: | + tar -czf cdata-build.tgz \ + .docker/cdata-build \ + .docker/cdata-jni-dist \ + .docker/maven-cache/repository/org/apache/arrow + - name: Upload reusable C Data artifacts + uses: actions/upload-artifact@v5 + with: + name: cdata-build + path: cdata-build.tgz + retention-days: 1 + + test-cdata: + name: AMD64 Conda JNI JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} + needs: build-cdata + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + jdk: [17, 21, 23] + maven: [3.9.9] + env: + JDK: ${{ matrix.jdk }} + MAVEN: ${{ matrix.maven }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: recursive + - name: Cache Docker Volumes + uses: actions/cache@v5 + with: + path: .docker/maven-cache + key: cdata-17-3.9.9-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: cdata-17-3.9.9- + - name: Download reusable C Data artifacts + uses: actions/download-artifact@v6 + with: + name: cdata-build + - name: Restore reusable C Data artifacts + shell: bash + run: | + mkdir -p .docker + rm -rf \ + .docker/cdata-build \ + .docker/cdata-jni-dist \ + .docker/maven-cache/repository/org/apache/arrow + tar -xzf cdata-build.tgz + - name: Test C Data without compiler lifecycle shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: ci/scripts/test.sh . build jni + run: | + docker compose run \ + --rm \ + -e CI=true \ + -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ + cdata-test-prebuilt diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 146a40cf7a..3fcb10e242 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -46,7 +46,11 @@ cp -r "${source_dir}/dev" "${build_dir}" # crawl back up to the top. GNU realpath has --relative-to but this does not # work on macOS -poms=$(find "${source_dir}" -not \( -path "${source_dir}"/build -prune \) -type f -name pom.xml) +poms=$( + find "${source_dir}" \ + \( -path "${source_dir}"/build -o -path "${source_dir}"/.docker \) -prune \ + -o -type f -name pom.xml -print +) if [[ "$OSTYPE" == "darwin"* ]]; then poms=$(echo "$poms" | xargs -n1 python -c "import sys; import os.path; print(os.path.relpath(sys.argv[1], '${source_dir}'))") else diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index 8061ee455d..55b2808f76 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -31,14 +31,61 @@ if [ -d "${java_jni_dist_dir}" ]; then java_jni_dist_dir="$(cd "${java_jni_dist_dir}" && pwd)" fi -mvn="mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" -# Use `2 * ncores` threads -mvn="${mvn} -T 2C" -mvn="${mvn} -Denforcer.skip=true" +mvn=( + mvn + -B + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -T + 2C + -Denforcer.skip=true +) + +run_tests() { + local log_name=$1 + shift + + if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then + run_prebuilt_tests "${log_name}" "${@}" -DfailIfNoTests=false surefire:test + else + "${@}" test + fi +} + +run_prebuilt_tests() { + local log_name=$1 + shift + + set -o pipefail + "${@}" | tee "${source_dir}/${log_name}" + + if grep -E "Compiling [0-9]+ source files?" "${source_dir}/${log_name}"; then + echo "Unexpected compilation occurred while running prebuilt tests." + exit 1 + fi + + if ! grep -q "Tests run:" "${source_dir}/${log_name}"; then + echo "No surefire test summary found; tests may have been skipped." + exit 1 + fi +} pushd "${build_dir}" -${mvn} -Darrow.test.dataRoot="${source_dir}/testing/data" test +if [[ "${ARROW_JAVA_TEST_BASE:-ON}" = "ON" ]]; then + run_tests \ + surefire.log \ + "${mvn[@]}" \ + -Darrow.test.dataRoot="${source_dir}/testing/data" + + if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then + run_prebuilt_tests \ + opens-surefire.log \ + "${mvn[@]}" \ + -DfailIfNoTests=false \ + -pl memory/memory-core \ + org.apache.maven.plugins:maven-surefire-plugin:test@opens-tests + fi +fi projects=() if [ "${ARROW_JAVA_JNI}" = "ON" ]; then @@ -47,7 +94,9 @@ if [ "${ARROW_JAVA_JNI}" = "ON" ]; then projects+=(gandiva) fi if [ "${#projects[@]}" -gt 0 ]; then - ${mvn} test \ + run_tests \ + jni-surefire.log \ + "${mvn[@]}" \ -Parrow-jni \ -pl "$( IFS=, @@ -57,7 +106,12 @@ if [ "${#projects[@]}" -gt 0 ]; then fi if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then - ${mvn} test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir="${java_jni_dist_dir}" + run_tests \ + cdata-surefire.log \ + "${mvn[@]}" \ + -Parrow-c-data \ + -pl c \ + -Darrow.c.jni.dist.dir="${java_jni_dist_dir}" fi popd diff --git a/compose.yaml b/compose.yaml index 4fd825e5a5..43435fd1af 100644 --- a/compose.yaml +++ b/compose.yaml @@ -30,6 +30,14 @@ volumes: name: ccache-cache maven-cache: name: maven-cache + java-build: + name: java-build + jni-dist: + name: jni-dist + cdata-build: + name: cdata-build + cdata-jni-dist: + name: cdata-jni-dist services: ubuntu: @@ -50,6 +58,33 @@ services: /arrow-java/ci/scripts/build.sh /arrow-java /build /jni && /arrow-java/ci/scripts/test.sh /arrow-java /build /jni" + ubuntu-artifacts: + # Builds reusable pure Java artifacts for CI test jobs. + image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK} + volumes: + - .:/arrow-java:delegated + - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated + - ${DOCKER_VOLUME_PREFIX}java-build:/build-output:delegated + - ${DOCKER_VOLUME_PREFIX}jni-dist:/jni:delegated + command: + /bin/bash -c " + rm -rf /build-output/build && + find /jni -mindepth 1 -maxdepth 1 -exec rm -rf {} + && + /arrow-java/ci/scripts/build.sh /arrow-java /build-output/build /jni" + + ubuntu-test-prebuilt: + # Runs pure Java tests against prebuilt CI artifacts on Ubuntu. + image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK} + volumes: + - .:/arrow-java:delegated + - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated + - ${DOCKER_VOLUME_PREFIX}java-build:/build-output:delegated + - ${DOCKER_VOLUME_PREFIX}jni-dist:/jni:delegated + environment: + ARROW_JAVA_TEST_PREBUILT: "ON" + command: + /bin/bash -c "/arrow-java/ci/scripts/test.sh /arrow-java /build-output/build /jni" + conda-jni-cdata: # Builds and tests just the C Data Interface JNI library and JARs. # (No dependencies on arrow-cpp.) @@ -84,6 +119,56 @@ services: /arrow-java/ci/scripts/build.sh /arrow-java /build /jni && /arrow-java/ci/scripts/test.sh /arrow-java /build /jni" + cdata-artifacts: + # Builds reusable C Data Interface JNI artifacts for CI test jobs. + image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + build: + context: . + dockerfile: ci/docker/conda-jni.dockerfile + cache_from: + - ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + args: + jdk: ${JDK} + maven: ${MAVEN} + user: root + volumes: + - .:/arrow-java:delegated + - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated + - ${DOCKER_VOLUME_PREFIX}cdata-build:/build-output:delegated + - ${DOCKER_VOLUME_PREFIX}cdata-jni-dist:/jni:delegated + environment: + ARROW_JAVA_CDATA: "ON" + command: + /bin/bash -c " + rm -rf /build-output/build && + find /jni -mindepth 1 -maxdepth 1 -exec rm -rf {} + && + /arrow-java/ci/scripts/jni_build.sh /arrow-java /tmp/cdata-jni /tmp/cdata-native /jni && + /arrow-java/ci/scripts/build.sh /arrow-java /build-output/build /jni" + + cdata-test-prebuilt: + # Runs C Data Interface tests against prebuilt CI artifacts. + image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + build: + context: . + dockerfile: ci/docker/conda-jni.dockerfile + cache_from: + - ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + args: + jdk: ${JDK} + maven: ${MAVEN} + user: root + volumes: + - .:/arrow-java:delegated + - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated + - ${DOCKER_VOLUME_PREFIX}cdata-build:/build-output:delegated + - ${DOCKER_VOLUME_PREFIX}cdata-jni-dist:/jni:delegated + environment: + ARROW_JAVA_CDATA: "ON" + ARROW_JAVA_TEST_BASE: "OFF" + ARROW_JAVA_TEST_PREBUILT: "ON" + command: + /bin/bash -c "/arrow-java/ci/scripts/test.sh /arrow-java /build-output/build /jni" + vcpkg-jni: # Builds all the JNI libraries, but not the JARs. # (Requires arrow-cpp.) From ee3d78fabd93a243160650255300da2950ded680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Wed, 29 Jul 2026 15:23:51 +0100 Subject: [PATCH 2/4] Simplify build-once test workflow --- .github/workflows/test.yml | 73 +++++++++++--------------------------- ci/scripts/build.sh | 6 +--- ci/scripts/test.sh | 24 +++++-------- compose.yaml | 58 +++++++++++------------------- 4 files changed, 52 insertions(+), 109 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 276209b987..ad51e1295e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,6 +35,7 @@ permissions: env: DOCKER_VOLUME_PREFIX: ".docker/" + MAVEN: 3.9.9 jobs: build-java: @@ -52,29 +53,23 @@ jobs: uses: actions/cache@v6 with: path: .docker/maven-cache - key: java-build-17-3.9.9-${{ hashFiles('compose.yaml', '**/pom.xml') }} - restore-keys: java-build-17-3.9.9- + key: java-build-17-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: java-build-17-${{ env.MAVEN }}- - name: Build without tests - shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} JDK: 17 - MAVEN: 3.9.9 run: | docker compose run \ --rm \ -e CI=true \ -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ ubuntu-artifacts - if [ -d .docker ]; then - sudo chown -R "$(id -u):$(id -g)" .docker - fi + sudo chown -R "$(id -u):$(id -g)" .docker - name: Pack reusable artifacts - shell: bash run: | tar -czf java-build.tgz \ .docker/java-build \ - .docker/jni-dist \ .docker/maven-cache/repository/org/apache/arrow - name: Upload reusable artifacts uses: actions/upload-artifact@v5 @@ -84,7 +79,7 @@ jobs: retention-days: 1 test-java: - name: ${{ matrix.name || format('AMD64 Ubuntu JDK {0} Maven {1}', matrix.jdk, matrix.maven) }} + name: ${{ matrix.name || format('AMD64 Ubuntu JDK {0} Maven 3.9.9', matrix.jdk) }} needs: build-java runs-on: ${{ matrix.os }} if: ${{ !contains(github.event.pull_request.title, 'WIP') }} @@ -92,34 +87,22 @@ jobs: strategy: fail-fast: false matrix: - target: [ubuntu] jdk: [17, 21, 23] - maven: [3.9.9] + os: [ubuntu-latest] include: - - target: ubuntu - os: ubuntu-latest + - os: ubuntu-latest compose_service: ubuntu-test-prebuilt - - target: macos-intel - name: AMD64 macOS 15-intel Java JDK 17 + - name: AMD64 macOS 15-intel Java JDK 17 os: macos-15-intel jdk: 17 - maven: 3.9.9 - compose_service: '' - - target: macos-arm - name: AArch64 macOS latest Java JDK 17 + - name: AArch64 macOS latest Java JDK 17 os: macos-latest jdk: 17 - maven: 3.9.9 - compose_service: '' - - target: windows - name: AMD64 Windows Server 2022 Java JDK 17 + - name: AMD64 Windows Server 2022 Java JDK 17 os: windows-latest jdk: 17 - maven: 3.9.9 - compose_service: '' env: JDK: ${{ matrix.jdk }} - MAVEN: ${{ matrix.maven }} steps: - name: Checkout Arrow uses: actions/checkout@v7 @@ -127,19 +110,19 @@ jobs: fetch-depth: 0 submodules: recursive - name: Set up Java - if: ${{ matrix.compose_service == '' }} + if: ${{ !matrix.compose_service }} uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: ${{ matrix.jdk }} cache: 'maven' - name: Cache Docker Volumes - if: ${{ matrix.compose_service != '' }} + if: ${{ matrix.compose_service }} uses: actions/cache@v6 with: path: .docker/maven-cache - key: java-build-17-3.9.9-${{ hashFiles('compose.yaml', '**/pom.xml') }} - restore-keys: java-build-17-3.9.9- + key: java-build-17-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: java-build-17-${{ env.MAVEN }}- - name: Download reusable artifacts uses: actions/download-artifact@v6 with: @@ -147,30 +130,26 @@ jobs: - name: Restore reusable artifacts shell: bash run: | - mkdir -p .docker rm -rf \ .docker/java-build \ - .docker/jni-dist \ .docker/maven-cache/repository/org/apache/arrow tar -xzf java-build.tgz - name: Restore reusable artifacts for hosted runner - if: ${{ matrix.compose_service == '' }} + if: ${{ !matrix.compose_service }} shell: bash run: | - rm -rf build jni cp -a .docker/java-build/build build - cp -a .docker/jni-dist jni mkdir -p "${HOME}/.m2/repository/org/apache" cp -a .docker/maven-cache/repository/org/apache/arrow "${HOME}/.m2/repository/org/apache/" - name: Test prebuilt artifacts on macOS/Windows - if: ${{ matrix.compose_service == '' }} + if: ${{ !matrix.compose_service }} shell: bash env: ARROW_JAVA_TEST_PREBUILT: "ON" DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} run: ci/scripts/test.sh . build jni - name: Test prebuilt artifacts on Ubuntu Docker - if: ${{ matrix.compose_service != '' }} + if: ${{ matrix.compose_service }} shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} @@ -188,7 +167,6 @@ jobs: timeout-minutes: 30 env: JDK: 17 - MAVEN: 3.9.9 steps: - name: Checkout Arrow uses: actions/checkout@v7 @@ -202,7 +180,6 @@ jobs: key: cdata-${{ env.JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} restore-keys: cdata-${{ env.JDK }}-${{ env.MAVEN }}- - name: Build C Data without tests - shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} run: | @@ -211,11 +188,8 @@ jobs: -e CI=true \ -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ cdata-artifacts - if [ -d .docker ]; then - sudo chown -R "$(id -u):$(id -g)" .docker - fi + sudo chown -R "$(id -u):$(id -g)" .docker - name: Pack reusable C Data artifacts - shell: bash run: | tar -czf cdata-build.tgz \ .docker/cdata-build \ @@ -229,7 +203,7 @@ jobs: retention-days: 1 test-cdata: - name: AMD64 Conda JNI JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} + name: AMD64 Conda JNI JDK ${{ matrix.jdk }} Maven 3.9.9 needs: build-cdata runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} @@ -238,10 +212,8 @@ jobs: fail-fast: false matrix: jdk: [17, 21, 23] - maven: [3.9.9] env: JDK: ${{ matrix.jdk }} - MAVEN: ${{ matrix.maven }} steps: - name: Checkout Arrow uses: actions/checkout@v6 @@ -252,23 +224,20 @@ jobs: uses: actions/cache@v5 with: path: .docker/maven-cache - key: cdata-17-3.9.9-${{ hashFiles('compose.yaml', '**/pom.xml') }} - restore-keys: cdata-17-3.9.9- + key: cdata-17-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: cdata-17-${{ env.MAVEN }}- - name: Download reusable C Data artifacts uses: actions/download-artifact@v6 with: name: cdata-build - name: Restore reusable C Data artifacts - shell: bash run: | - mkdir -p .docker rm -rf \ .docker/cdata-build \ .docker/cdata-jni-dist \ .docker/maven-cache/repository/org/apache/arrow tar -xzf cdata-build.tgz - name: Test C Data without compiler lifecycle - shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} run: | diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 3fcb10e242..4e3f2ad517 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -46,11 +46,7 @@ cp -r "${source_dir}/dev" "${build_dir}" # crawl back up to the top. GNU realpath has --relative-to but this does not # work on macOS -poms=$( - find "${source_dir}" \ - \( -path "${source_dir}"/build -o -path "${source_dir}"/.docker \) -prune \ - -o -type f -name pom.xml -print -) +poms=$(find "${source_dir}" \( -path "${source_dir}"/build -o -path "${source_dir}"/.docker \) -prune -o -type f -name pom.xml -print) if [[ "$OSTYPE" == "darwin"* ]]; then poms=$(echo "$poms" | xargs -n1 python -c "import sys; import os.path; print(os.path.relpath(sys.argv[1], '${source_dir}'))") else diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index 55b2808f76..f6c1a68d8e 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -set -ex +set -exo pipefail if [[ "${ARROW_JAVA_TEST:-ON}" != "ON" ]]; then exit @@ -41,45 +41,41 @@ mvn=( ) run_tests() { - local log_name=$1 - shift - if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then - run_prebuilt_tests "${log_name}" "${@}" -DfailIfNoTests=false surefire:test + run_prebuilt_tests "${@}" -DfailIfNoTests=false surefire:test else "${@}" test fi } run_prebuilt_tests() { - local log_name=$1 - shift + local log + log=$(mktemp) - set -o pipefail - "${@}" | tee "${source_dir}/${log_name}" + "${@}" | tee "${log}" - if grep -E "Compiling [0-9]+ source files?" "${source_dir}/${log_name}"; then + if grep -E "Compiling [0-9]+ source files?" "${log}"; then echo "Unexpected compilation occurred while running prebuilt tests." exit 1 fi - if ! grep -q "Tests run:" "${source_dir}/${log_name}"; then + if ! grep -q "Tests run:" "${log}"; then echo "No surefire test summary found; tests may have been skipped." exit 1 fi + + rm -f "${log}" } pushd "${build_dir}" if [[ "${ARROW_JAVA_TEST_BASE:-ON}" = "ON" ]]; then run_tests \ - surefire.log \ "${mvn[@]}" \ -Darrow.test.dataRoot="${source_dir}/testing/data" if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then run_prebuilt_tests \ - opens-surefire.log \ "${mvn[@]}" \ -DfailIfNoTests=false \ -pl memory/memory-core \ @@ -95,7 +91,6 @@ if [ "${ARROW_JAVA_JNI}" = "ON" ]; then fi if [ "${#projects[@]}" -gt 0 ]; then run_tests \ - jni-surefire.log \ "${mvn[@]}" \ -Parrow-jni \ -pl "$( @@ -107,7 +102,6 @@ fi if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then run_tests \ - cdata-surefire.log \ "${mvn[@]}" \ -Parrow-c-data \ -pl c \ diff --git a/compose.yaml b/compose.yaml index 43435fd1af..cd682b99c2 100644 --- a/compose.yaml +++ b/compose.yaml @@ -32,13 +32,28 @@ volumes: name: maven-cache java-build: name: java-build - jni-dist: - name: jni-dist cdata-build: name: cdata-build cdata-jni-dist: name: cdata-jni-dist +x-cdata-prebuilt: &cdata-prebuilt + image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + build: + context: . + dockerfile: ci/docker/conda-jni.dockerfile + cache_from: + - ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + args: + jdk: ${JDK} + maven: ${MAVEN} + user: root + volumes: + - .:/arrow-java:delegated + - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated + - ${DOCKER_VOLUME_PREFIX}cdata-build:/build-output:delegated + - ${DOCKER_VOLUME_PREFIX}cdata-jni-dist:/jni:delegated + services: ubuntu: # Build and test arrow-java on Ubuntu. @@ -65,12 +80,10 @@ services: - .:/arrow-java:delegated - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - ${DOCKER_VOLUME_PREFIX}java-build:/build-output:delegated - - ${DOCKER_VOLUME_PREFIX}jni-dist:/jni:delegated command: /bin/bash -c " rm -rf /build-output/build && - find /jni -mindepth 1 -maxdepth 1 -exec rm -rf {} + && - /arrow-java/ci/scripts/build.sh /arrow-java /build-output/build /jni" + /arrow-java/ci/scripts/build.sh /arrow-java /build-output/build /tmp/jni" ubuntu-test-prebuilt: # Runs pure Java tests against prebuilt CI artifacts on Ubuntu. @@ -79,11 +92,10 @@ services: - .:/arrow-java:delegated - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - ${DOCKER_VOLUME_PREFIX}java-build:/build-output:delegated - - ${DOCKER_VOLUME_PREFIX}jni-dist:/jni:delegated environment: ARROW_JAVA_TEST_PREBUILT: "ON" command: - /bin/bash -c "/arrow-java/ci/scripts/test.sh /arrow-java /build-output/build /jni" + /bin/bash -c "/arrow-java/ci/scripts/test.sh /arrow-java /build-output/build /tmp/jni" conda-jni-cdata: # Builds and tests just the C Data Interface JNI library and JARs. @@ -121,21 +133,7 @@ services: cdata-artifacts: # Builds reusable C Data Interface JNI artifacts for CI test jobs. - image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration - build: - context: . - dockerfile: ci/docker/conda-jni.dockerfile - cache_from: - - ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration - args: - jdk: ${JDK} - maven: ${MAVEN} - user: root - volumes: - - .:/arrow-java:delegated - - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - - ${DOCKER_VOLUME_PREFIX}cdata-build:/build-output:delegated - - ${DOCKER_VOLUME_PREFIX}cdata-jni-dist:/jni:delegated + <<: *cdata-prebuilt environment: ARROW_JAVA_CDATA: "ON" command: @@ -147,21 +145,7 @@ services: cdata-test-prebuilt: # Runs C Data Interface tests against prebuilt CI artifacts. - image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration - build: - context: . - dockerfile: ci/docker/conda-jni.dockerfile - cache_from: - - ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration - args: - jdk: ${JDK} - maven: ${MAVEN} - user: root - volumes: - - .:/arrow-java:delegated - - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - - ${DOCKER_VOLUME_PREFIX}cdata-build:/build-output:delegated - - ${DOCKER_VOLUME_PREFIX}cdata-jni-dist:/jni:delegated + <<: *cdata-prebuilt environment: ARROW_JAVA_CDATA: "ON" ARROW_JAVA_TEST_BASE: "OFF" From d53578072a449f5f710320198d3ff621c354cbdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Fri, 31 Jul 2026 16:21:25 +0100 Subject: [PATCH 3/4] Restore custom Surefire test coverage --- .github/workflows/test.yml | 4 ++-- ci/scripts/test.sh | 22 ++++++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ad51e1295e..f04827ce29 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -216,12 +216,12 @@ jobs: JDK: ${{ matrix.jdk }} steps: - name: Checkout Arrow - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: .docker/maven-cache key: cdata-17-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index f6c1a68d8e..93abe6f0e7 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -70,16 +70,30 @@ run_prebuilt_tests() { pushd "${build_dir}" if [[ "${ARROW_JAVA_TEST_BASE:-ON}" = "ON" ]]; then - run_tests \ - "${mvn[@]}" \ - -Darrow.test.dataRoot="${source_dir}/testing/data" - if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then + run_prebuilt_tests \ + "${mvn[@]}" \ + -Darrow.test.dataRoot="${source_dir}/testing/data" \ + -DfailIfNoTests=false \ + -pl "!vector" \ + surefire:test + for execution in default-test run-unsafe; do + run_prebuilt_tests \ + "${mvn[@]}" \ + -Darrow.test.dataRoot="${source_dir}/testing/data" \ + -DfailIfNoTests=false \ + -pl vector \ + "org.apache.maven.plugins:maven-surefire-plugin:test@${execution}" + done run_prebuilt_tests \ "${mvn[@]}" \ -DfailIfNoTests=false \ -pl memory/memory-core \ org.apache.maven.plugins:maven-surefire-plugin:test@opens-tests + else + run_tests \ + "${mvn[@]}" \ + -Darrow.test.dataRoot="${source_dir}/testing/data" fi fi From b43254321e3b6b319294b1b1a919126d8182626f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Fri, 31 Jul 2026 18:18:57 +0100 Subject: [PATCH 4/4] Make Vector test passes explicit --- ci/scripts/test.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index 93abe6f0e7..491f3cf861 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -77,14 +77,19 @@ if [[ "${ARROW_JAVA_TEST_BASE:-ON}" = "ON" ]]; then -DfailIfNoTests=false \ -pl "!vector" \ surefire:test - for execution in default-test run-unsafe; do - run_prebuilt_tests \ - "${mvn[@]}" \ - -Darrow.test.dataRoot="${source_dir}/testing/data" \ - -DfailIfNoTests=false \ - -pl vector \ - "org.apache.maven.plugins:maven-surefire-plugin:test@${execution}" - done + # Direct Surefire skips Vector's lifecycle-bound allocator test passes. + run_prebuilt_tests \ + "${mvn[@]}" \ + -Darrow.test.dataRoot="${source_dir}/testing/data" \ + -DfailIfNoTests=false \ + -pl vector \ + org.apache.maven.plugins:maven-surefire-plugin:test@default-test + run_prebuilt_tests \ + "${mvn[@]}" \ + -Darrow.test.dataRoot="${source_dir}/testing/data" \ + -DfailIfNoTests=false \ + -pl vector \ + org.apache.maven.plugins:maven-surefire-plugin:test@run-unsafe run_prebuilt_tests \ "${mvn[@]}" \ -DfailIfNoTests=false \