From f176df3965bb06cb2599444e69aeb346834f28ea Mon Sep 17 00:00:00 2001 From: Almas Abdrazak Date: Thu, 21 May 2026 08:42:22 -0700 Subject: [PATCH 1/4] Upgrade libcrypt version --- mongodb-crypt/build.gradle.kts | 177 ++++++++++++++++++++++++++------- 1 file changed, 143 insertions(+), 34 deletions(-) diff --git a/mongodb-crypt/build.gradle.kts b/mongodb-crypt/build.gradle.kts index a59ccefc02..336e45e5ee 100644 --- a/mongodb-crypt/build.gradle.kts +++ b/mongodb-crypt/build.gradle.kts @@ -16,6 +16,10 @@ import ProjectExtensions.configureJarManifest import ProjectExtensions.configureMavenPublication import de.undercouch.gradle.tasks.download.Download +import org.gradle.api.GradleException +import java.io.ByteArrayOutputStream +import javax.inject.Inject +import org.gradle.process.ExecOperations plugins { id("project.java") @@ -53,59 +57,164 @@ val jnaLibPlatform: String = val jnaLibsPath: String = System.getProperty("jnaLibsPath", "${jnaResourcesDir}${jnaLibPlatform}") val jnaResources: String = System.getProperty("jna.library.path", jnaLibsPath) -// Download jnaLibs that match the git tag or revision to jnaResourcesBuildDir -val downloadRevision = "1.17.3" -val binariesArchiveName = "libmongocrypt-java.tar.gz" +// Download jnaLibs that match the libmongocrypt release version to jnaResourcesBuildDir. +val downloadRevision = "1.18.1" +val downloadUrlBase = "https://github.com/mongodb/libmongocrypt/releases/download/$downloadRevision" /** - * The name of the archive includes downloadRevision to ensure that: - * - the archive is downloaded if the revision changes. - * - the archive is not downloaded if the revision is the same and archive had already been saved in build output. + * Maps a JNA platform key (the directory consumed by `jna.library.path`) to the libmongocrypt GitHub release tarball + * that ships its native library, plus the path of that library inside the tarball. The tarball name and its internal + * layout differ per platform, so both must be tracked explicitly. + * + * libmongocrypt's signature assets replace the `.tar.gz` suffix with `.asc` (e.g. + * `libmongocrypt-linux-x86_64-glibc_2_7-nocrypto-1.18.1.asc`). */ -val localBinariesArchiveName = "libmongocrypt-java-$downloadRevision.tar.gz" - -val downloadUrl: String = - "https://mciuploads.s3.amazonaws.com/libmongocrypt/java/$downloadRevision/$binariesArchiveName" +data class CryptBinary(val jnaPlatform: String, val tarball: String, val libPathInTarball: String) { + val signature: String = tarball.removeSuffix(".tar.gz") + ".asc" +} -val jnaMapping: Map = - mapOf( - "rhel-62-64-bit" to "linux-x86-64", - "rhel72-zseries-test" to "linux-s390x", - "rhel-71-ppc64el" to "linux-ppc64le", - "ubuntu1604-arm64" to "linux-aarch64", - "windows-test" to "win32-x86-64", - "macos" to "darwin") +val cryptBinaries: List = + listOf( + CryptBinary( + "linux-x86-64", + "libmongocrypt-linux-x86_64-glibc_2_7-nocrypto-$downloadRevision.tar.gz", + "lib64/libmongocrypt.so"), + CryptBinary( + "linux-s390x", + "libmongocrypt-linux-s390x-glibc_2_7-nocrypto-$downloadRevision.tar.gz", + "lib64/libmongocrypt.so"), + CryptBinary( + "linux-ppc64le", + "libmongocrypt-linux-ppc64le-glibc_2_17-nocrypto-$downloadRevision.tar.gz", + "lib64/libmongocrypt.so"), + CryptBinary( + "linux-aarch64", + "libmongocrypt-linux-arm64-glibc_2_17-nocrypto-$downloadRevision.tar.gz", + "lib64/libmongocrypt.so"), + CryptBinary("win32-x86-64", "libmongocrypt-windows-x86_64-$downloadRevision.tar.gz", "bin/mongocrypt.dll"), + CryptBinary("darwin", "libmongocrypt-macos-universal-$downloadRevision.tar.gz", "lib/libmongocrypt.dylib")) sourceSets { main { java { resources { srcDirs(jnaResourcesDir) } } } } +/** + * Public key used to sign libmongocrypt release tarballs. See: + * https://www.mongodb.com/docs/manual/tutorial/verify-mongodb-packages/#std-label-verify-pkgs + */ +val libmongocryptPublicKeyUrl = "https://pgp.mongodb.com/libmongocrypt.pub" +val libmongocryptPublicKeyFile = "libmongocrypt.pub" + tasks.register("downloadJava") { - src(downloadUrl) - dest("${jnaDownloadsDir}/$localBinariesArchiveName") + src( + cryptBinaries.flatMap { listOf("$downloadUrlBase/${it.tarball}", "$downloadUrlBase/${it.signature}") } + + libmongocryptPublicKeyUrl) + dest(jnaDownloadsDir) overwrite(true) - /* To make sure we don't download archive with binaries if it hasn't been changed in S3 bucket since last download.*/ + /* Skip URLs whose remote artifact hasn't changed since the last download. */ onlyIfModified(true) } +/* + * Verify the signature of every downloaded libmongocrypt tarball before extracting it. + * Per DRIVERS-3441, drivers that bundle libmongocrypt must verify GPG signatures of + * release tarballs against the official MongoDB libmongocrypt signing key. + * + * The keyring is kept under `build/` so this task does not touch the developer's + * system GPG keyring and so `./gradlew clean` resets the trust state. + */ +val skipCryptVerify = providers.gradleProperty("skipCryptVerify").map { it.toBoolean() }.orElse(false) + +abstract class VerifyLibmongocryptTask : DefaultTask() { + @get:Inject abstract val execOps: ExecOperations + + @get:InputFiles abstract val tarballs: ConfigurableFileCollection + @get:InputFiles abstract val signatures: ConfigurableFileCollection + @get:InputFile abstract val publicKey: RegularFileProperty + @get:Input abstract val skipVerify: Property + @get:OutputDirectory abstract val gnupgHome: DirectoryProperty + + @TaskAction + fun verify() { + if (skipVerify.get()) { + logger.warn( + "SKIPPING libmongocrypt signature verification because -PskipCryptVerify=true was set. " + + "Do not use this for release builds.") + return + } + + try { + execOps.exec { + commandLine("gpg", "--version") + standardOutput = ByteArrayOutputStream() + } + } catch (e: Exception) { + throw GradleException( + "gpg is required to verify libmongocrypt tarballs since 1.18.0 but was not found on PATH. " + + "Install gpg (e.g. `apt-get install gnupg`, `brew install gnupg`, Gpg4win on Windows), " + + "or pass -PskipCryptVerify=true for offline development builds.", + e + ) + } + + val home = + gnupgHome.get().asFile.apply { + deleteRecursively() + mkdirs() + // GPG refuses to use a homedir with permissions broader than the owner. + setReadable(false, false) + setReadable(true, true) + setWritable(false, false) + setWritable(true, true) + setExecutable(false, false) + setExecutable(true, true) + } + + execOps.exec { commandLine("gpg", "--homedir", home.path, "--batch", "--import", publicKey.get().asFile.path) } + + val tarballList = tarballs.files.toList() + val signatureList = signatures.files.toList() + check(tarballList.size == signatureList.size) { + "Expected each tarball to have a matching signature: ${tarballList.size} tarballs vs ${signatureList.size} signatures." + } + tarballList.zip(signatureList).forEach { (tarball, signature) -> + execOps.exec { + commandLine("gpg", "--homedir", home.path, "--batch", "--verify", signature.path, tarball.path) + } + } + } +} + +tasks.register("verifyJava") { + dependsOn("downloadJava") + tarballs.from(cryptBinaries.map { "$jnaDownloadsDir/${it.tarball}" }) + signatures.from(cryptBinaries.map { "$jnaDownloadsDir/${it.signature}" }) + publicKey.set(file("$jnaDownloadsDir/$libmongocryptPublicKeyFile")) + skipVerify.set(skipCryptVerify) + gnupgHome.set(layout.buildDirectory.dir("jnaLibs/gnupg")) +} + tasks.register("unzipJava") { /* Clean up the directory first if the task is not UP-TO-DATE. - This can happen if the download revision has been changed and the archive is downloaded again. + This can happen if the download revision has been changed and the archives are downloaded again. */ doFirst { - println("Cleaning up $jnaResourcesDir") + println("Clearing $jnaResourcesDir before extraction") delete(jnaResourcesDir) } - from(tarTree(resources.gzip("${jnaDownloadsDir}/$localBinariesArchiveName"))) - include( - jnaMapping.keys.flatMap { - listOf( - "${it}/nocrypto/**/libmongocrypt.so", "${it}/lib/**/libmongocrypt.dylib", "${it}/bin/**/mongocrypt.dll") - }) - eachFile { path = "${jnaMapping[path.substringBefore("/")]}/${name}" } + cryptBinaries.forEach { spec -> + from(tarTree(resources.gzip("$jnaDownloadsDir/${spec.tarball}"))) { + include(spec.libPathInTarball) + eachFile { path = "${spec.jnaPlatform}/${name}" } + includeEmptyDirs = false + } + } into(jnaResourcesDir) - dependsOn("downloadJava") + dependsOn("downloadJava", "verifyJava") - doLast { println("jna.library.path contents: \n ${fileTree(jnaResourcesDir).files.joinToString(",\n ")}") } + doLast { + println("Extracted libmongocrypt $downloadRevision binaries to $jnaResourcesDir:") + fileTree(jnaResourcesDir).files.sortedBy { it.path }.forEach { println(" $it") } + } } // The `processResources` task (defined by the `java-library` plug-in) consumes files in the main @@ -134,8 +243,8 @@ tasks.withType { | System properties: | ================= | - | jnaLibsPath : Custom local JNA library path for inclusion into the build (rather than downloading from s3) - | gitRevision : Optional Git Revision to download the built resources for from s3. + | jnaLibsPath : Custom local JNA library path for inclusion into the build (rather than downloading from the libmongocrypt GitHub release) + | gitRevision : Optional Git Revision to download the built resources for from the libmongocrypt GitHub release. """.trimMargin() } From 62f7f9a775bd31fa33cd2d33369a23f7a1b29ea8 Mon Sep 17 00:00:00 2001 From: Almas Abdrazak Date: Thu, 21 May 2026 11:45:56 -0700 Subject: [PATCH 2/4] address failed build --- mongodb-crypt/build.gradle.kts | 51 ++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/mongodb-crypt/build.gradle.kts b/mongodb-crypt/build.gradle.kts index 336e45e5ee..1d2f7c37c8 100644 --- a/mongodb-crypt/build.gradle.kts +++ b/mongodb-crypt/build.gradle.kts @@ -130,7 +130,14 @@ abstract class VerifyLibmongocryptTask : DefaultTask() { @get:InputFiles abstract val signatures: ConfigurableFileCollection @get:InputFile abstract val publicKey: RegularFileProperty @get:Input abstract val skipVerify: Property - @get:OutputDirectory abstract val gnupgHome: DirectoryProperty + + /* + * Scratch keyring directory. Marked @Internal (not @OutputDirectory) because GnuPG leaves a + * `S.gpg-agent` Unix domain socket inside it, which Gradle's output snapshotter cannot fingerprint + * (`IOException: not a regular file`). The directory is genuinely ephemeral - nothing downstream + * consumes it, and re-running gpg from scratch every time is cheap. + */ + @get:Internal abstract val gnupgHome: DirectoryProperty @TaskAction fun verify() { @@ -170,14 +177,31 @@ abstract class VerifyLibmongocryptTask : DefaultTask() { execOps.exec { commandLine("gpg", "--homedir", home.path, "--batch", "--import", publicKey.get().asFile.path) } - val tarballList = tarballs.files.toList() - val signatureList = signatures.files.toList() - check(tarballList.size == signatureList.size) { - "Expected each tarball to have a matching signature: ${tarballList.size} tarballs vs ${signatureList.size} signatures." - } - tarballList.zip(signatureList).forEach { (tarball, signature) -> - execOps.exec { - commandLine("gpg", "--homedir", home.path, "--batch", "--verify", signature.path, tarball.path) + try { + // Pair each tarball with its signature explicitly by basename. ConfigurableFileCollection + // exposes files as a Set with no guaranteed iteration order, so zipping the two collections + // would risk verifying mismatched pairs. + val signaturesByName = signatures.files.associateBy { it.name } + tarballs.files.forEach { tarball -> + val signatureName = tarball.name.removeSuffix(".tar.gz") + ".asc" + val signature = + signaturesByName[signatureName] + ?: throw GradleException( + "Missing signature $signatureName for ${tarball.name}; expected it next to the tarball.") + execOps.exec { + commandLine("gpg", "--homedir", home.path, "--batch", "--verify", signature.path, tarball.path) + } + } + } finally { + // Shut down gpg-agent so its leftover Unix domain socket does not accumulate or confuse + // a subsequent run that reuses the homedir. + try { + execOps.exec { + commandLine("gpgconf", "--homedir", home.path, "--kill", "gpg-agent") + isIgnoreExitValue = true + } + } catch (_: Exception) { + // Best-effort cleanup; not a build failure. } } } @@ -243,8 +267,13 @@ tasks.withType { | System properties: | ================= | - | jnaLibsPath : Custom local JNA library path for inclusion into the build (rather than downloading from the libmongocrypt GitHub release) - | gitRevision : Optional Git Revision to download the built resources for from the libmongocrypt GitHub release. + | jnaLibsPath : Custom local JNA library path for inclusion into the build (rather than downloading the libmongocrypt GitHub release). + | + | Project properties: + | =================== + | + | skipCryptVerify : Pass -PskipCryptVerify=true to skip GPG verification of downloaded libmongocrypt tarballs. + | Intended for offline development; do not use for release builds. """.trimMargin() } From e56e5b35d06c36c949e33ffd82ba0f905493c2d3 Mon Sep 17 00:00:00 2001 From: Almas Abdrazak Date: Thu, 21 May 2026 14:00:09 -0700 Subject: [PATCH 3/4] Kotlin codestyle --- mongodb-crypt/build.gradle.kts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/mongodb-crypt/build.gradle.kts b/mongodb-crypt/build.gradle.kts index 1d2f7c37c8..3dba428cd0 100644 --- a/mongodb-crypt/build.gradle.kts +++ b/mongodb-crypt/build.gradle.kts @@ -16,9 +16,9 @@ import ProjectExtensions.configureJarManifest import ProjectExtensions.configureMavenPublication import de.undercouch.gradle.tasks.download.Download -import org.gradle.api.GradleException import java.io.ByteArrayOutputStream import javax.inject.Inject +import org.gradle.api.GradleException import org.gradle.process.ExecOperations plugins { @@ -156,10 +156,9 @@ abstract class VerifyLibmongocryptTask : DefaultTask() { } catch (e: Exception) { throw GradleException( "gpg is required to verify libmongocrypt tarballs since 1.18.0 but was not found on PATH. " + - "Install gpg (e.g. `apt-get install gnupg`, `brew install gnupg`, Gpg4win on Windows), " + - "or pass -PskipCryptVerify=true for offline development builds.", - e - ) + "Install gpg (e.g. `apt-get install gnupg`, `brew install gnupg`, Gpg4win on Windows), " + + "or pass -PskipCryptVerify=true for offline development builds.", + e) } val home = @@ -178,8 +177,10 @@ abstract class VerifyLibmongocryptTask : DefaultTask() { execOps.exec { commandLine("gpg", "--homedir", home.path, "--batch", "--import", publicKey.get().asFile.path) } try { - // Pair each tarball with its signature explicitly by basename. ConfigurableFileCollection - // exposes files as a Set with no guaranteed iteration order, so zipping the two collections + // Pair each tarball with its signature explicitly by basename. + // ConfigurableFileCollection + // exposes files as a Set with no guaranteed iteration order, so zipping the two + // collections // would risk verifying mismatched pairs. val signaturesByName = signatures.files.associateBy { it.name } tarballs.files.forEach { tarball -> From 9dd5e333a00dfa438c59330f0e60a493d0ee46c2 Mon Sep 17 00:00:00 2001 From: Almas Abdrazak Date: Thu, 21 May 2026 14:22:28 -0700 Subject: [PATCH 4/4] address PR comments --- mongodb-crypt/build.gradle.kts | 66 +++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/mongodb-crypt/build.gradle.kts b/mongodb-crypt/build.gradle.kts index 3dba428cd0..94a1abdd36 100644 --- a/mongodb-crypt/build.gradle.kts +++ b/mongodb-crypt/build.gradle.kts @@ -54,10 +54,14 @@ val jnaDownloadsDir = rootProject.file("build/jnaLibs/downloads/").path val jnaResourcesDir = rootProject.file("build/jnaLibs/resources/").path val jnaLibPlatform: String = if (com.sun.jna.Platform.RESOURCE_PREFIX.startsWith("darwin")) "darwin" else com.sun.jna.Platform.RESOURCE_PREFIX -val jnaLibsPath: String = System.getProperty("jnaLibsPath", "${jnaResourcesDir}${jnaLibPlatform}") +// When -DjnaLibsPath is set, the user wants to use a pre-existing local copy of the libmongocrypt +// binaries instead of fetching them from the libmongocrypt GitHub release, so we skip the whole +// download / verify / extract chain. +val userSuppliedJnaLibsPath: String? = System.getProperty("jnaLibsPath") +val jnaLibsPath: String = userSuppliedJnaLibsPath ?: "${jnaResourcesDir}${jnaLibPlatform}" val jnaResources: String = System.getProperty("jna.library.path", jnaLibsPath) -// Download jnaLibs that match the libmongocrypt release version to jnaResourcesBuildDir. +// Download the libmongocrypt per-platform tarballs (and their signatures) to jnaDownloadsDir. val downloadRevision = "1.18.1" val downloadUrlBase = "https://github.com/mongodb/libmongocrypt/releases/download/$downloadRevision" @@ -103,14 +107,17 @@ sourceSets { main { java { resources { srcDirs(jnaResourcesDir) } } } } val libmongocryptPublicKeyUrl = "https://pgp.mongodb.com/libmongocrypt.pub" val libmongocryptPublicKeyFile = "libmongocrypt.pub" -tasks.register("downloadJava") { +tasks.register("downloadCryptLibs") { src( cryptBinaries.flatMap { listOf("$downloadUrlBase/${it.tarball}", "$downloadUrlBase/${it.signature}") } + libmongocryptPublicKeyUrl) dest(jnaDownloadsDir) - overwrite(true) - /* Skip URLs whose remote artifact hasn't changed since the last download. */ + /* Reuse already-downloaded files. Useful for offline builds and reduces network churn. */ + overwrite(false) onlyIfModified(true) + + /* Bypass entirely when the caller has supplied a local libmongocrypt directory. */ + onlyIf { userSuppliedJnaLibsPath == null } } /* @@ -174,14 +181,13 @@ abstract class VerifyLibmongocryptTask : DefaultTask() { setExecutable(true, true) } - execOps.exec { commandLine("gpg", "--homedir", home.path, "--batch", "--import", publicKey.get().asFile.path) } + execOps.exec { + commandLine("gpg", "--homedir", home.path, "--batch", "--quiet", "--import", publicKey.get().asFile.path) + } try { - // Pair each tarball with its signature explicitly by basename. - // ConfigurableFileCollection - // exposes files as a Set with no guaranteed iteration order, so zipping the two - // collections - // would risk verifying mismatched pairs. + // Pair tarballs with signatures by basename; ConfigurableFileCollection.files is an + // unordered Set, so zipping the two collections could mismatch pairs. val signaturesByName = signatures.files.associateBy { it.name } tarballs.files.forEach { tarball -> val signatureName = tarball.name.removeSuffix(".tar.gz") + ".asc" @@ -190,7 +196,17 @@ abstract class VerifyLibmongocryptTask : DefaultTask() { ?: throw GradleException( "Missing signature $signatureName for ${tarball.name}; expected it next to the tarball.") execOps.exec { - commandLine("gpg", "--homedir", home.path, "--batch", "--verify", signature.path, tarball.path) + commandLine( + "gpg", + "--homedir", + home.path, + "--batch", + "--quiet", + "--trust-model", + "always", + "--verify", + signature.path, + tarball.path) } } } finally { @@ -208,16 +224,22 @@ abstract class VerifyLibmongocryptTask : DefaultTask() { } } -tasks.register("verifyJava") { - dependsOn("downloadJava") +tasks.register("verifyCryptLibs") { + dependsOn("downloadCryptLibs") tarballs.from(cryptBinaries.map { "$jnaDownloadsDir/${it.tarball}" }) signatures.from(cryptBinaries.map { "$jnaDownloadsDir/${it.signature}" }) publicKey.set(file("$jnaDownloadsDir/$libmongocryptPublicKeyFile")) skipVerify.set(skipCryptVerify) gnupgHome.set(layout.buildDirectory.dir("jnaLibs/gnupg")) + + /* Bypass entirely when the caller has supplied a local libmongocrypt directory. */ + onlyIf { userSuppliedJnaLibsPath == null } + + /* Always re-verify: gpg is cheap and never trusting a cached "signature was good" decision is safer. */ + outputs.upToDateWhen { false } } -tasks.register("unzipJava") { +tasks.register("extractCryptLibs") { /* Clean up the directory first if the task is not UP-TO-DATE. This can happen if the download revision has been changed and the archives are downloaded again. @@ -234,7 +256,10 @@ tasks.register("unzipJava") { } } into(jnaResourcesDir) - dependsOn("downloadJava", "verifyJava") + dependsOn("downloadCryptLibs", "verifyCryptLibs") + + /* Bypass entirely when the caller has supplied a local libmongocrypt directory. */ + onlyIf { userSuppliedJnaLibsPath == null } doLast { println("Extracted libmongocrypt $downloadRevision binaries to $jnaResourcesDir:") @@ -244,10 +269,11 @@ tasks.register("unzipJava") { // The `processResources` task (defined by the `java-library` plug-in) consumes files in the main // source set. -// Add a dependency on `unzipJava`. `unzipJava` adds libmongocrypt libraries to the main source set. -tasks.processResources { mustRunAfter(tasks.named("unzipJava")) } +// Add a dependency on `extractCryptLibs`, which adds libmongocrypt libraries to the main source +// set. +tasks.processResources { mustRunAfter(tasks.named("extractCryptLibs")) } -tasks.register("downloadJnaLibs") { dependsOn("downloadJava", "unzipJava") } +tasks.register("downloadJnaLibs") { dependsOn("downloadCryptLibs", "verifyCryptLibs", "extractCryptLibs") } tasks.test { systemProperty("jna.debug_load", "true") @@ -259,7 +285,7 @@ tasks.test { println("jna.library.path contents:") println(fileTree(jnaResources) { this.setIncludes(listOf("*.*")) }.files.joinToString(",\n ", " ")) } - dependsOn("downloadJnaLibs", "downloadJava", "unzipJava") + dependsOn("downloadJnaLibs") } tasks.withType {