From e07007ebf6759c27f70746f0cf239d7ad1ef021e Mon Sep 17 00:00:00 2001 From: Caleigh Runge-Hottman Date: Wed, 24 Jun 2026 12:39:14 -0400 Subject: [PATCH] Add Pulp 3 RPM resolution to KonfluxSource [RHELDST-38332] KonfluxSource now queries Pulp 3 by SHA256 to resolve each RPM's pulp_href, which is set as RpmPushItem.src for Pulp-to-Pulp syncing. Changes: - Add Pulp3Client integration: look up RPMs by SHA256 checksum and set src=pulp_href on RpmPushItem - Accept Pulp connection params (pulp_url, pulp_cert, pulp_key, pulp_domain) as constructor args via Source.get() query params - Auto-discover advisory subdirectories instead of requiring an explicit advisories parameter - Add pubtools-pulplib>=2.44.0 as a required dependency Co-Authored-By: Claude Opus 4.6 --- docs/sources/konflux.rst | 9 +- requirements.in | 1 + requirements.txt | 216 ++++++++---- .../backend/konflux_source/konflux_source.py | 251 +++++++++++--- .../RHBA-2020:1234/advisory_cdn_filelist.json | 18 + .../RHBA-2020:1234/advisory_cdn_metadata.json | 45 +++ tests/konflux/test_konflux_source.py | 322 ++++++++++++++++-- 7 files changed, 713 insertions(+), 149 deletions(-) create mode 100644 tests/konflux/data/RHBA-2020:1234/advisory_cdn_filelist.json create mode 100644 tests/konflux/data/RHBA-2020:1234/advisory_cdn_metadata.json diff --git a/docs/sources/konflux.rst b/docs/sources/konflux.rst index 40cde036..1db565e9 100644 --- a/docs/sources/konflux.rst +++ b/docs/sources/konflux.rst @@ -3,7 +3,7 @@ Source: konflux The ``konflux`` push source allows the loading of content from local JSON files organized by advisory. This source is designed for use with Konflux-generated -advisory metadata and does not require network access or external API calls. +advisory metadata and queries a Pulp 3 server to resolve RPM content hrefs. Supported content types: @@ -28,6 +28,11 @@ Multiple advisories can be specified with a comma-separated list: ``konflux:/path/to/konflux/data?advisories=RHSA-2020:0509,RHSA-2020:0510`` +Pulp 3 connection parameters (``pulp_url``, ``pulp_cert``, ``pulp_key``, +``pulp_domain``) are required and can be provided as URL query parameters, +keyword arguments to :meth:`~pushsource.Source.get`, or environment variables +(see :class:`~pushsource.KonfluxSource` for details). + The base directory should contain subdirectories named after each advisory ID. Each advisory subdirectory must contain: @@ -87,9 +92,9 @@ Unlike the `ErrataSource`, the `KonfluxSource`: * Reads from local JSON files rather than querying the Errata API * Does not require Koji integration +* Queries Pulp 3 by SHA256 to resolve RPM ``pulp_href`` for Pulp-to-Pulp syncing * Does not currently support filtering by architecture (this use case may be supported in the future) * Currently produces RPMs and advisories (additional content types such as modules and container images can be supported in the future) -* RPM push items have ``src=None`` (no local RPM files, only metadata) Python API reference -------------------- diff --git a/requirements.in b/requirements.in index a6abb559..df79debb 100644 --- a/requirements.in +++ b/requirements.in @@ -5,6 +5,7 @@ frozenlist2 kobo koji>=1.18 more-executors>=2.7.0 +pubtools-pulplib>=2.44.0 pushcollector pyasn1 python-dateutil diff --git a/requirements.txt b/requirements.txt index 918e4c7a..d54fa414 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,17 +4,29 @@ # # pip-compile --generate-hashes requirements.in # +anyio[trio]==4.12.1 \ + --hash=sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703 \ + --hash=sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c + # via + # httpx + # pubtools-pulplib attrs==26.1.0 \ --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 # via # -r requirements.in # jsonschema + # outcome + # pubtools-pulplib # referencing -certifi==2026.4.22 \ - --hash=sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a \ - --hash=sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580 - # via requests + # trio +certifi==2026.6.17 \ + --hash=sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432 \ + --hash=sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db + # via + # httpcore + # httpx + # requests cffi==2.0.0 \ --hash=sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb \ --hash=sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b \ @@ -232,65 +244,70 @@ charset-normalizer==3.4.7 \ --hash=sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79 \ --hash=sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464 # via requests -cryptography==48.0.0 \ - --hash=sha256:0890f502ddf7d9c6426129c3f49f5c0a39278ed7cd6322c8755ffca6ee675a13 \ - --hash=sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6 \ - --hash=sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8 \ - --hash=sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25 \ - --hash=sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c \ - --hash=sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832 \ - --hash=sha256:27241b1dc9962e056062a8eef1991d02c3a24569c95975bd2322a8a52c6e5e12 \ - --hash=sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c \ - --hash=sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7 \ - --hash=sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c \ - --hash=sha256:3cb07a3ed6431663cd321ea8a000a1314c74211f823e4177fefa2255e057d1ec \ - --hash=sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5 \ - --hash=sha256:4defde8685ae324a9eb9d818717e93b4638ef67070ac9bc15b8ca85f63048355 \ - --hash=sha256:55b7718303bf06a5753dcdccf2f3945cf18ad7bffde41b61226e4db31ab89a9c \ - --hash=sha256:561215ea3879cb1cbbf272867e2efda62476f240fb58c64de6b393ae19246741 \ - --hash=sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86 \ - --hash=sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321 \ - --hash=sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a \ - --hash=sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7 \ - --hash=sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920 \ - --hash=sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e \ - --hash=sha256:76341972e1eff8b4bea859f09c0d3e64b96ce931b084f9b9b7db8ef364c30eff \ - --hash=sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd \ - --hash=sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3 \ - --hash=sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f \ - --hash=sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602 \ - --hash=sha256:84cf79f0dc8b36ac5da873481716e87aef31fcfa0444f9e1d8b4b2cece142855 \ - --hash=sha256:8c7378637d7d88016fa6791c159f698b3d3eed28ebf844ac36b9dc04a14dae18 \ - --hash=sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a \ - --hash=sha256:906cbf0670286c6e0044156bc7d4af9cbb0ef6db9f73e52c3ec56ba6bdde5336 \ - --hash=sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239 \ - --hash=sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74 \ - --hash=sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a \ - --hash=sha256:9be5aafa5736574f8f15f262adc81b2a9869e2cfe9014d52a44633905b40d52c \ - --hash=sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4 \ - --hash=sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c \ - --hash=sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f \ - --hash=sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4 \ - --hash=sha256:a64697c641c7b1b2178e573cbc31c7c6684cd56883a478d75143dbb7118036db \ - --hash=sha256:ad64688338ed4bc1a6618076ba75fd7194a5f1797ac60b47afe926285adb3166 \ - --hash=sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5 \ - --hash=sha256:c17dfe85494deaeddc5ce251aebd1d60bbe6afc8b62071bb0b469431a000124f \ - --hash=sha256:c18684a7f0cc9a3cb60328f496b8e3372def7c5d2df39ac267878b05565aaaae \ - --hash=sha256:cc90c0b39b2e3c65ef52c804b72e3c58f8a04ab2a1871272798e5f9572c17d20 \ - --hash=sha256:db63bf618e5dea46c07de12e900fe1cdd2541e6dc9dbae772a70b7d4d4765f6a \ - --hash=sha256:ea8990436d914540a40ab24b6a77c0969695ed52f4a4874c5137ccf7045a7057 \ - --hash=sha256:ecde28a596bead48b0cfd2a1b4416c3d43074c2d785e3a398d7ec1fc4d0f7fbb \ - --hash=sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c \ - --hash=sha256:fdfef35d751d510fcef5252703621574364fec16418c4a1e5e1055248401054b +cryptography==49.0.0 \ + --hash=sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001 \ + --hash=sha256:07cab27cc7b7e0fd28e5e26bb9eeedde5c135c868b46de4a27845abe94af6122 \ + --hash=sha256:084ef1af862eb07ec46d25f68689f2102a9fc0e05ce7b80f14f5fe51e4eef0f6 \ + --hash=sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c \ + --hash=sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325 \ + --hash=sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69 \ + --hash=sha256:196ecd6a36e4e9aa10270393bb98d8df88fccee0bf1e5128b91ae4eb4375896d \ + --hash=sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36 \ + --hash=sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc \ + --hash=sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6 \ + --hash=sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b \ + --hash=sha256:32703d93296f5c1f4b53349ad3a250c2cae0fdecd3a3dd5d47e616d8d616af27 \ + --hash=sha256:33cd0565932807baddb67b96dbee92f2c374b5c89dee09fd74079aeb8c8dba61 \ + --hash=sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18 \ + --hash=sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db \ + --hash=sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b \ + --hash=sha256:4ae387c9cb68ea569ca17e490d66d8142b81c3cc814bf179974b7d146e490bbb \ + --hash=sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2 \ + --hash=sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459 \ + --hash=sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e \ + --hash=sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21 \ + --hash=sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8 \ + --hash=sha256:73a205dce83953d131a4aa1e0fd917a2fd1c5b1eef251e9d7152efefcbf5caf7 \ + --hash=sha256:7abcee80084cda3f7691f3eb1ce480d8df49cec637b429aa35986c1de71738aa \ + --hash=sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9 \ + --hash=sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db \ + --hash=sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64 \ + --hash=sha256:b20133d204d2bb56ba047642199603876c872026ca53e79c35b83772ab2cc505 \ + --hash=sha256:b39efa323140595abd3ecca8529d321ae50f55f3aa3ba9cc81ea56a6011953d5 \ + --hash=sha256:b47db11c2c3525083296069b98ac5221907455e989ae0c2e3008bde851921615 \ + --hash=sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f \ + --hash=sha256:b970c6da94d5bb18629db453d14f2a1300f6bf59b61e9b82377931ef95504866 \ + --hash=sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6 \ + --hash=sha256:c2bc30226390d60ea19d9f82b19db005fe0452154a23c1c410c12ea801e43561 \ + --hash=sha256:c83782480a4a9da4d0feb51950131ba32e12e70813848b3343f6e18c28a66838 \ + --hash=sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9 \ + --hash=sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7 \ + --hash=sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68 \ + --hash=sha256:d8ecde755e2e91bf773fc94e8c9d730cd7f2007004cb492263a794ec3899a1c8 \ + --hash=sha256:e3fb64c420688e5319ae25113a354015abbd8dffbfbc41781a1ea66fc7622ac3 \ + --hash=sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e \ + --hash=sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a \ + --hash=sha256:f37d847238971164fdbc68ade6f6574aecc9c0af714190e2083429ff68f4ce9d \ + --hash=sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4 \ + --hash=sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493 \ + --hash=sha256:fc1e275c2f1d97b1a6450b8b0ea3ebfa6e087a611c2b26cb2404d48588abab7b # via -r requirements.in -decorator==5.2.1 \ - --hash=sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360 \ - --hash=sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a +decorator==5.3.1 \ + --hash=sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82 \ + --hash=sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c # via gssapi defusedxml==0.7.1 \ --hash=sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69 \ --hash=sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61 - # via koji + # via + # koji + # pubtools-pulplib +exceptiongroup==1.3.1 \ + --hash=sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219 \ + --hash=sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598 + # via + # anyio + # trio frozendict==2.4.7 ; python_version >= "3.6" \ --hash=sha256:05dd27415f913cd11649009f53d97eb565ce7b76787d7869c4733738c10e8d27 \ --hash=sha256:0664092614d2b9d0aa404731f33ad5459a54fe8dab9d1fd45aa714fa6de4d0ef \ @@ -383,11 +400,15 @@ frozendict==2.4.7 ; python_version >= "3.6" \ --hash=sha256:fd7ba56cf6340c732ecb78787c4e9600c4bd01372af7313ded21037126d33ec6 \ --hash=sha256:ffd1a9f9babec9119712e76a39397d8aa0d72ef8c4ccad917c6175d7e7f81b74 \ --hash=sha256:fff8584e3bbdc5c1713cd016fbf4b88babfffd4e5e89b39020f2a208dd24c900 - # via -r requirements.in + # via + # -r requirements.in + # pubtools-pulplib frozenlist2==1.0.0 \ --hash=sha256:33f6c6bb2c7d38524ec3c2d6f2d8a3ee2625a9e13096d8bc64db012b516a95e0 \ --hash=sha256:caffe66813e42de320b10d08b8c0604c7eb3f142a8482ad3130243e084f37a2f - # via -r requirements.in + # via + # -r requirements.in + # pubtools-pulplib gssapi==1.11.1 \ --hash=sha256:0e8b4d76801f2a8f8e6d85746cb9048d47341c6706800b357c61ba09e4741c03 \ --hash=sha256:126352502e15dc42f786a4635e5fb4dc8ae4bbc89354e85ab094c478a9e49beb \ @@ -407,25 +428,45 @@ gssapi==1.11.1 \ --hash=sha256:da43c0e0ae84bb9f04c4e016eac6d3826c6357f827183042ba990ccedeeab052 \ --hash=sha256:e5d01ac02df8fe67c32cd1684c0954e935d50158ebb956fdffbf9aad7695a3b3 # via requests-gssapi -idna==3.14 \ - --hash=sha256:466d810d7a2cc1022bea9b037c39728d51ae7dad40d480fc9b7d7ecf98ba8ee3 \ - --hash=sha256:e677eaf072e290f7b725f9acf0b3a2bd55f9fd6f7c70abe5f0e34823d0accf69 - # via requests +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via httpcore +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via pubtools-pulplib +idna==3.18 \ + --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ + --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 + # via + # anyio + # httpx + # requests + # trio importlib-metadata==8.7.1 ; python_version < "3.10" \ --hash=sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb \ --hash=sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151 - # via -r requirements.in + # via + # -r requirements.in + # pubtools jsonschema==4.25.1 \ --hash=sha256:3fba0169e345c7175110351d456342c364814cfcf3b964ba4587f22915230a63 \ --hash=sha256:e4a9655ce0da0c0b67a085847e00a3a51449e1157f4f75e9fb5aa545e122eb85 - # via pushcollector + # via + # pubtools-pulplib + # pushcollector jsonschema-specifications==2025.9.1 \ --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe \ --hash=sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d # via jsonschema -kobo==0.41.0 \ - --hash=sha256:f4261eab274e0ed4753e98a3397c4706d1065fecb9da3939a26c2e6cc7313713 \ - --hash=sha256:f64b123a0c8037ab46915996bc61364fd6160f0ed03f89cffcb7c3a5fc7b07d2 +kobo==0.42.0 \ + --hash=sha256:8d4bcf13c00083929d6cd7e55cc2602cf4cc86e9ddf62e84547b75007af51045 \ + --hash=sha256:92e384ecc968ac1d0863b5bce0a8913169227b7fe0cce9b40cccf999138633b8 # via -r requirements.in koji==1.36.0 \ --hash=sha256:26226f4be87c5eac094b3ec12d7bc2fe9f5c3898a38a11c2f995596cf83f9596 \ @@ -437,7 +478,24 @@ more-executors==2.11.4 \ --hash=sha256:f1b21d72c4c15069e891d9b96bca05f9abde149e3c11ca54630c5a1a5ee8f4b5 # via # -r requirements.in + # pubtools-pulplib # pushcollector +outcome==1.3.0.post0 \ + --hash=sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8 \ + --hash=sha256:e771c5ce06d1415e356078d3bdd68523f284b4ce5419828922b6871e65eda82b + # via trio +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + # via pubtools +pubtools==1.4.5 \ + --hash=sha256:04544718ad40fc689814e93e5af4cd343ab4688f2883e5e2cae29c8e2d7ae2cd \ + --hash=sha256:c7270bcb8e3a7677f838023603f93e687d9efc2b63248e600bf625580b3bf3bf + # via pubtools-pulplib +pubtools-pulplib==2.44.0 \ + --hash=sha256:04c343db7884ca81b5f9f5985f28ee3bfb0f34aeab2cf5a154a9011a2c89e2ac \ + --hash=sha256:151f1f4d1529a1652de4b74ea31bc7bde455146cae049f28b5eeb3d73791a3ac + # via -r requirements.in pushcollector==1.3.0 \ --hash=sha256:18281fcd4247ca8b668c25348bbc2f9cc032912cecc7037824a74ad8adcbb1c3 \ --hash=sha256:3b0754dd86e9d66ec1b19f13b66be6582e5d1b5087a1f1db3778485948fc0f28 @@ -532,6 +590,7 @@ pyyaml==6.0.3 \ --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 # via # -r requirements.in + # pubtools-pulplib # pushcollector referencing==0.36.2 \ --hash=sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa \ @@ -545,6 +604,7 @@ requests==2.32.5 \ # via # -r requirements.in # koji + # pubtools-pulplib # requests-gssapi requests-gssapi==1.4.0 \ --hash=sha256:146a1d1e74d7ca0b9d21af0f8d2d1ca259e9228ff57fbbf23f92e25217da4520 \ @@ -717,11 +777,29 @@ six==1.17.0 \ # koji # more-executors # python-dateutil +sniffio==1.3.1 \ + --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ + --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc + # via trio +sortedcontainers==2.4.0 \ + --hash=sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88 \ + --hash=sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 + # via trio +tenacity==9.1.2 \ + --hash=sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb \ + --hash=sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138 + # via pubtools-pulplib +trio==0.31.0 \ + --hash=sha256:b5d14cd6293d79298b49c3485ffd9c07e3ce03a6da8c7dfbe0cb3dd7dc9a4774 \ + --hash=sha256:f71d551ccaa79d0cb73017a33ef3264fde8335728eb4c6391451fe5d253a9d5b + # via anyio typing-extensions==4.15.0 \ --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 # via + # anyio # cryptography + # exceptiongroup # referencing urllib3==2.6.3 \ --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ diff --git a/src/pushsource/_impl/backend/konflux_source/konflux_source.py b/src/pushsource/_impl/backend/konflux_source/konflux_source.py index ed52f456..15f4a6af 100644 --- a/src/pushsource/_impl/backend/konflux_source/konflux_source.py +++ b/src/pushsource/_impl/backend/konflux_source/konflux_source.py @@ -1,5 +1,7 @@ import logging +import os +import anyio from more_executors import Executors from ...source import Source @@ -9,6 +11,8 @@ from .konflux_loader import KonfluxLoader +from pubtools.pulplib import Pulp3Client + LOG = logging.getLogger("pushsource.konflux") @@ -30,6 +34,12 @@ def __init__( self, url, advisories, + pulp_url=None, + pulp_cert=None, + pulp_key=None, + pulp_user=None, + pulp_password=None, + pulp_domain=None, threads=4, timeout=60 * 60, ): @@ -46,6 +56,38 @@ def __init__( Advisory ID(s) to process. Can be a single string or list. Multiple IDs can be comma-separated. + pulp_url (str): + URL of hosted Pulp server + (e.g., https://packages.redhat.com/). + If omitted, uses ``PUSHSOURCE_KONFLUX_PULP_URL`` + environment variable. + + pulp_cert (str): + Path to TLS client certificate for Pulp authentication. + If omitted, uses ``PUSHSOURCE_KONFLUX_PULP_CERT`` + environment variable. + + pulp_key (str): + Path to TLS client key for Pulp authentication. + If omitted, uses ``PUSHSOURCE_KONFLUX_PULP_KEY`` + environment variable. + + pulp_user (str): + Username for Pulp basic authentication. + If omitted, uses ``PUSHSOURCE_KONFLUX_PULP_USER`` + environment variable. Use with ``pulp_password`` + as an alternative to certificate authentication. + + pulp_password (str): + Password for Pulp basic authentication. + If omitted, uses ``PUSHSOURCE_KONFLUX_PULP_PASSWORD`` + environment variable. + + pulp_domain (str): + Pulp domain name (e.g., "konflux-myteam-tenant"). + If omitted, uses ``PUSHSOURCE_KONFLUX_PULP_DOMAIN`` + environment variable. + threads (int): Number of threads for concurrent processing. @@ -53,11 +95,56 @@ def __init__( Timeout in seconds for operations. """ self._base_dir = url - self._advisories = list_argument(advisories) self._threads = threads self._timeout = timeout self._loader = KonfluxLoader(url) + + self._advisories = list_argument(advisories) + + # Resolve Pulp params from env vars if not provided + pulp_url = pulp_url or os.environ.get("PUSHSOURCE_KONFLUX_PULP_URL") + pulp_cert = pulp_cert or os.environ.get("PUSHSOURCE_KONFLUX_PULP_CERT") + pulp_key = pulp_key or os.environ.get("PUSHSOURCE_KONFLUX_PULP_KEY") + pulp_user = pulp_user or os.environ.get("PUSHSOURCE_KONFLUX_PULP_USER") + pulp_password = pulp_password or os.environ.get( + "PUSHSOURCE_KONFLUX_PULP_PASSWORD" + ) + pulp_domain = pulp_domain or os.environ.get("PUSHSOURCE_KONFLUX_PULP_DOMAIN") + + # Validate required params + if not pulp_url: + raise RuntimeError("Required parameter not provided: pulp_url") + if not pulp_domain: + raise RuntimeError("Required parameter not provided: pulp_domain") + + # Validate that at least one auth method is provided + has_cert_auth = pulp_cert and pulp_key + has_basic_auth = pulp_user and pulp_password + if not has_cert_auth and not has_basic_auth: + raise RuntimeError( + "Pulp authentication not configured. Provide either " + "pulp_cert/pulp_key for mTLS or pulp_user/pulp_password " + "for basic authentication." + ) + + LOG.info( + "Initializing Pulp3 client: url=%s, domain=%s, auth=%s", + pulp_url, + pulp_domain, + "cert" if has_cert_auth else "basic", + ) + + # Build Pulp3Client kwargs + self._pulp_client_kwargs = { + "url": pulp_url, + "domain": pulp_domain, + } + if has_cert_auth: + self._pulp_client_kwargs["cert"] = (pulp_cert, pulp_key) + elif has_basic_auth: + self._pulp_client_kwargs["auth"] = (pulp_user, pulp_password) + self._executor = Executors.thread_pool( name="pushsource-konflux", max_workers=threads ).with_cancel_on_shutdown() @@ -152,6 +239,9 @@ def _create_rpm_items(self, data): Since we don't use koji, we construct RPM push items directly from the information in advisory_cdn_filelist.json. + Collects all RPM SHA256 checksums and resolves their pulp_hrefs + in a single batch query, then builds the push items. + Parameters: data (KonfluxAdvisoryData): Advisory data containing metadata and filelist @@ -159,64 +249,129 @@ def _create_rpm_items(self, data): Returns: list[RpmPushItem]: List of RPM push items """ - items = [] + # First pass: collect all RPM info and SHA256 checksums + rpm_entries = [] + all_sha256s = [] for build_nvr, build_data in data.filelist.items(): - if "rpms" in build_data: - checksums = build_data.get("checksums", {}) - sig_key = build_data.get("sig_key") - - for rpm_filename, destinations in build_data["rpms"].items(): - # Construct RPM push item - item = self._create_rpm_item( - filename=rpm_filename, - build_nvr=build_nvr, - destinations=destinations, - checksums=checksums, - signing_key=sig_key, - origin=data.advisory_id, + if "rpms" not in build_data: + continue + checksums = build_data.get("checksums", {}) + sig_key = build_data.get("sig_key") + + for rpm_filename, destinations in build_data["rpms"].items(): + md5sum = checksums.get("md5", {}).get(rpm_filename) + sha256sum = checksums.get("sha256", {}).get(rpm_filename) + + if not sha256sum: + raise RuntimeError( + "No SHA256 checksum found for RPM %s in advisory %s" + % (rpm_filename, data.advisory_id) ) - items.append(item) + + all_sha256s.append(sha256sum) + rpm_entries.append( + { + "filename": rpm_filename, + "build_nvr": build_nvr, + "destinations": destinations, + "md5sum": md5sum, + "sha256sum": sha256sum, + "signing_key": sig_key, + } + ) + + # Batch-resolve all SHA256 checksums to pulp_hrefs + href_map = anyio.run(self._resolve_rpm_hrefs, all_sha256s) + + # Second pass: build push items using resolved hrefs + items = [] + for entry in rpm_entries: + sha256sum = entry["sha256sum"] + pulp_href = href_map.get(sha256sum) + if not pulp_href: + raise RuntimeError( + "RPM %s (sha256=%s) not found in Pulp" + % (entry["filename"], sha256sum) + ) + + items.append( + RpmPushItem( + name=entry["filename"], + state="PENDING", + src=pulp_href, + dest=sorted(entry["destinations"]), + md5sum=entry["md5sum"], + sha256sum=sha256sum, + origin=data.advisory_id, + build=entry["build_nvr"], + signing_key=entry["signing_key"], + ) + ) return items - def _create_rpm_item( - self, filename, build_nvr, destinations, checksums, signing_key, origin - ): - """Create a single RpmPushItem from filelist data. + # Pulp's content search API enforces a filter complexity limit, + # rejecting queries with more than 7 OR clauses with: + # {"q": ["Filter expression exceeds allowed complexity."]} + _PULP_BATCH_SIZE = 7 + + async def _resolve_rpm_hrefs(self, sha256sums, batch_size=_PULP_BATCH_SIZE): + """Resolve SHA256 checksums to pulp_hrefs in batched queries. + + Uses one Pulp3Client instance and queries checksums in batches + to stay within Pulp's filter complexity limits. Parameters: - filename (str): - RPM filename - build_nvr (str): - Build NVR - destinations (list[str]): - List of repository destinations - checksums (dict): - Dict with 'md5' and 'sha256' checksum mappings - signing_key (str): - Signing key ID - origin (str): - Advisory ID + sha256sums (list[str]): + List of SHA256 checksums to resolve + + batch_size (int): + Maximum number of checksums per query (default: 7) Returns: - RpmPushItem: RPM push item + dict: Mapping of sha256 -> pulp_href + + Raises: + RuntimeError: If any RPM is not found or has no pulp_href """ - # Extract checksums for this specific RPM - md5sum = checksums.get("md5", {}).get(filename) - sha256sum = checksums.get("sha256", {}).get(filename) - - return RpmPushItem( - name=filename, - state="PENDING", - src=None, # RPMs are stored in artifact storage - dest=sorted(destinations), - md5sum=md5sum, - sha256sum=sha256sum, - origin=origin, - build=build_nvr, - signing_key=signing_key, - ) + href_map = {} + + async with Pulp3Client(**self._pulp_client_kwargs) as client: + for i in range(0, len(sha256sums), batch_size): + batch = sha256sums[i : i + batch_size] + query = client.build_query_sha256(batch) + + results = await client.search_content( + query=query, + fields=["pulp_href", "name", "sha256"], + limit=len(batch), + ) + + for result in results: + sha256 = result.get("sha256") + pulp_href = result.get("pulp_href") + if not pulp_href: + raise RuntimeError( + "RPM with SHA256 %s found but has no pulp_href: %s" + % (sha256, result) + ) + href_map[sha256] = pulp_href + LOG.debug( + "Found RPM in Pulp: sha256=%s, pulp_href=%s, name=%s", + sha256, + pulp_href, + result.get("name"), + ) + + LOG.info( + "Resolved %d/%d RPMs in Pulp (batch %d)", + len(href_map), + len(sha256sums), + (i // batch_size) + 1, + ) + + return href_map # Register the backend diff --git a/tests/konflux/data/RHBA-2020:1234/advisory_cdn_filelist.json b/tests/konflux/data/RHBA-2020:1234/advisory_cdn_filelist.json new file mode 100644 index 00000000..9d33661f --- /dev/null +++ b/tests/konflux/data/RHBA-2020:1234/advisory_cdn_filelist.json @@ -0,0 +1,18 @@ +{ + "bash-5.0.17-1.el8": { + "checksums": { + "md5": { + "bash-5.0.17-1.el8.x86_64.rpm": "aabbccdd00112233aabbccdd00112233" + }, + "sha256": { + "bash-5.0.17-1.el8.x86_64.rpm": "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" + } + }, + "rpms": { + "bash-5.0.17-1.el8.x86_64.rpm": [ + "rhel-8-for-x86_64-baseos-rpms__8" + ] + }, + "sig_key": "fd431d51" + } +} diff --git a/tests/konflux/data/RHBA-2020:1234/advisory_cdn_metadata.json b/tests/konflux/data/RHBA-2020:1234/advisory_cdn_metadata.json new file mode 100644 index 00000000..26c671a5 --- /dev/null +++ b/tests/konflux/data/RHBA-2020:1234/advisory_cdn_metadata.json @@ -0,0 +1,45 @@ +{ + "description": "An update for bash is now available.", + "from": "release-engineering@redhat.com", + "id": "RHBA-2020:1234", + "issued": "2020-04-01 12:00:00 UTC", + "pkglist": [ + { + "packages": [ + { + "arch": "x86_64", + "epoch": "0", + "filename": "bash-5.0.17-1.el8.x86_64.rpm", + "name": "bash", + "release": "1.el8", + "src": "bash-5.0.17-1.el8.src.rpm", + "sum": [ + "md5", + "aabbccdd00112233aabbccdd00112233", + "sha256", + "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" + ], + "version": "5.0.17" + } + ] + } + ], + "pulp_user_metadata": { + "content_types": [ + "rpm" + ] + }, + "pushcount": "1", + "reboot_suggested": false, + "references": [], + "release": "0", + "rights": "Copyright 2020 Red Hat Inc", + "severity": "None", + "solution": "Install the updated package.", + "status": "final", + "summary": "Bug fix update for bash.", + "title": "bash bug fix update", + "type": "bugfix", + "updated": "2020-04-01 12:00:00 UTC", + "version": "1" +} diff --git a/tests/konflux/test_konflux_source.py b/tests/konflux/test_konflux_source.py index 380ef8ff..10fe5639 100644 --- a/tests/konflux/test_konflux_source.py +++ b/tests/konflux/test_konflux_source.py @@ -2,6 +2,9 @@ import pytest import tempfile import json +from unittest.mock import AsyncMock, MagicMock, patch + +import anyio from pushsource import Source from pushsource._impl.backend.konflux_source import KonfluxSource @@ -15,11 +18,39 @@ DATADIR = os.path.join(os.path.dirname(__file__), "data") +# Default Pulp connection params for tests +PULP_PARAMS = { + "pulp_url": "https://pulp.example.com", + "pulp_cert": "/path/to/cert", + "pulp_key": "/path/to/key", + "pulp_domain": "konflux-myteam-tenant", +} + + +def _mock_resolve_hrefs(func, sha256sums): + """Mock for anyio.run that returns a dict mapping each sha256 to a mock href.""" + return { + sha256: "/pulp/api/v3/content/rpm/packages/mock-uuid/" for sha256 in sha256sums + } + def test_load_single_advisory(): """Test loading a single advisory and generating push items.""" - with Source.get("konflux:%s?advisories=RHSA-2020:0509" % DATADIR) as source: - items = list(source) + url = ( + "konflux:%s?" + "pulp_url=https://pulp.example.com" + "&pulp_cert=/path/to/cert" + "&pulp_key=/path/to/key" + "&pulp_domain=konflux-myteam-tenant" + "&advisories=RHSA-2020:0509" + ) % DATADIR + mock_href = "/pulp/api/v3/content/rpm/packages/mock-uuid/" + with patch( + "pushsource._impl.backend.konflux_source.konflux_source.anyio.run", + side_effect=_mock_resolve_hrefs, + ): + with Source.get(url) as source: + items = list(source) # Should have 1 erratum + 7 RPMs assert len(items) == 8 @@ -151,7 +182,7 @@ def test_load_single_advisory(): RpmPushItem( name="sudo-1.8.25p1-4.el8_0.3.ppc64le.rpm", state="PENDING", - src=None, + src=mock_href, dest=[ "rhel-8-for-ppc64le-baseos-rpms__8", "rhel-8-for-ppc64le-baseos-rpms__8_DOT_0", @@ -165,7 +196,7 @@ def test_load_single_advisory(): RpmPushItem( name="sudo-1.8.25p1-4.el8_0.3.src.rpm", state="PENDING", - src=None, + src=mock_href, dest=[ "rhel-8-for-ppc64le-baseos-source-rpms__8", "rhel-8-for-ppc64le-baseos-source-rpms__8_DOT_0", @@ -182,7 +213,7 @@ def test_load_single_advisory(): RpmPushItem( name="sudo-1.8.25p1-4.el8_0.3.x86_64.rpm", state="PENDING", - src=None, + src=mock_href, dest=[ "rhel-8-for-x86_64-baseos-e4s-rpms__8_DOT_0", "rhel-8-for-x86_64-baseos-rpms__8", @@ -197,7 +228,7 @@ def test_load_single_advisory(): RpmPushItem( name="sudo-debuginfo-1.8.25p1-4.el8_0.3.ppc64le.rpm", state="PENDING", - src=None, + src=mock_href, dest=[ "rhel-8-for-ppc64le-baseos-debug-rpms__8", "rhel-8-for-ppc64le-baseos-debug-rpms__8_DOT_0", @@ -211,7 +242,7 @@ def test_load_single_advisory(): RpmPushItem( name="sudo-debuginfo-1.8.25p1-4.el8_0.3.x86_64.rpm", state="PENDING", - src=None, + src=mock_href, dest=[ "rhel-8-for-x86_64-baseos-debug-rpms__8", "rhel-8-for-x86_64-baseos-debug-rpms__8_DOT_0", @@ -226,7 +257,7 @@ def test_load_single_advisory(): RpmPushItem( name="sudo-debugsource-1.8.25p1-4.el8_0.3.ppc64le.rpm", state="PENDING", - src=None, + src=mock_href, dest=[ "rhel-8-for-ppc64le-baseos-debug-rpms__8", "rhel-8-for-ppc64le-baseos-debug-rpms__8_DOT_0", @@ -240,7 +271,7 @@ def test_load_single_advisory(): RpmPushItem( name="sudo-debugsource-1.8.25p1-4.el8_0.3.x86_64.rpm", state="PENDING", - src=None, + src=mock_href, dest=[ "rhel-8-for-x86_64-baseos-debug-rpms__8", "rhel-8-for-x86_64-baseos-debug-rpms__8_DOT_0", @@ -255,25 +286,33 @@ def test_load_single_advisory(): ] -def test_comma_separated_advisories(): - """Test handling of comma-separated advisory IDs.""" - # Note: We only have one advisory in test data, but we can test the parsing - source = KonfluxSource(url=DATADIR, advisories="RHSA-2020:0509,RHSA-2020:0510") - - # Should parse into list - assert len(source._advisories) == 2 - assert "RHSA-2020:0509" in source._advisories - assert "RHSA-2020:0510" in source._advisories +def test_load_comma_separated_advisories(): + """Test loading advisories specified as a comma-separated string.""" + url = ( + "konflux:%s?" + "pulp_url=https://pulp.example.com" + "&pulp_cert=/path/to/cert" + "&pulp_key=/path/to/key" + "&pulp_domain=konflux-myteam-tenant" + "&advisories=RHSA-2020:0509,RHBA-2020:1234" + ) % DATADIR + with patch( + "pushsource._impl.backend.konflux_source.konflux_source.anyio.run", + side_effect=_mock_resolve_hrefs, + ): + with Source.get(url) as source: + items = list(source) + erratum_items = [i for i in items if isinstance(i, ErratumPushItem)] + rpm_items = [i for i in items if isinstance(i, RpmPushItem)] -def test_missing_advisory_directory(): - """Test error handling for missing advisory directory.""" - with pytest.raises(FileNotFoundError) as exc_info: - source = KonfluxSource(url=DATADIR, advisories="RHSA-9999:9999") - with source: - list(source) + # Should have both advisories + assert len(erratum_items) == 2 + advisory_ids = sorted(e.name for e in erratum_items) + assert advisory_ids == ["RHBA-2020:1234", "RHSA-2020:0509"] - assert "advisory_cdn_metadata.json" in str(exc_info.value) + # Should have 7 RPMs from RHSA-2020:0509 + 1 RPM from RHBA-2020:1234 + assert len(rpm_items) == 8 def test_invalid_json(): @@ -292,7 +331,9 @@ def test_invalid_json(): json.dump({}, f) with pytest.raises(ValueError) as exc_info: - source = KonfluxSource(url=tmpdir, advisories="TEST-2020:0001") + source = KonfluxSource( + url=tmpdir, advisories="TEST-2020:0001", **PULP_PARAMS + ) with source: list(source) @@ -301,14 +342,235 @@ def test_invalid_json(): def test_context_manager(): """Test context manager behavior.""" - source = KonfluxSource(url=DATADIR, advisories="RHSA-2020:0509") + source = KonfluxSource(url=DATADIR, advisories="RHSA-2020:0509", **PULP_PARAMS) # Executor should be running assert source._executor is not None - with source: - items = list(source) - assert len(items) > 0 + with patch( + "pushsource._impl.backend.konflux_source.konflux_source.anyio.run", + side_effect=_mock_resolve_hrefs, + ): + with source: + items = list(source) + assert len(items) > 0 # After exit, executor should be shutdown # (We can't easily test this without accessing private state) + + +def test_resolve_rpm_hrefs_success(): + """Test successful batch RPM resolution in Pulp.""" + source = KonfluxSource(url=DATADIR, advisories="RHSA-2020:0509", **PULP_PARAMS) + + # Mock Pulp3Client + mock_client = AsyncMock() + mock_client.__aenter__.return_value = mock_client + # build_query_sha256 is a sync method, use MagicMock + mock_client.build_query_sha256 = MagicMock( + return_value="(sha256=abc123 OR sha256=def456)" + ) + mock_client.search_content.return_value = [ + { + "pulp_href": "/pulp/api/v3/content/rpm/packages/abc-123/", + "name": "test-rpm-1.0-1.x86_64.rpm", + "sha256": "abc123", + }, + { + "pulp_href": "/pulp/api/v3/content/rpm/packages/def-456/", + "name": "test-rpm-1.0-1.src.rpm", + "sha256": "def456", + }, + ] + + with patch( + "pushsource._impl.backend.konflux_source.konflux_source.Pulp3Client", + return_value=mock_client, + ): + result = anyio.run(source._resolve_rpm_hrefs, ["abc123", "def456"]) + + assert result == { + "abc123": "/pulp/api/v3/content/rpm/packages/abc-123/", + "def456": "/pulp/api/v3/content/rpm/packages/def-456/", + } + mock_client.build_query_sha256.assert_called_once_with(["abc123", "def456"]) + mock_client.search_content.assert_called_once_with( + query="(sha256=abc123 OR sha256=def456)", + fields=["pulp_href", "name", "sha256"], + limit=2, + ) + + +def test_resolve_rpm_hrefs_multiple_batches(): + """Test RPM resolution splits into multiple batches.""" + source = KonfluxSource(url=DATADIR, advisories="RHSA-2020:0509", **PULP_PARAMS) + + mock_client = AsyncMock() + mock_client.__aenter__.return_value = mock_client + mock_client.build_query_sha256 = MagicMock( + side_effect=lambda sums: "(sha256=%s)" % sums[0] + ) + mock_client.search_content.side_effect = [ + [ + { + "pulp_href": "/pulp/api/v3/content/rpm/packages/abc-123/", + "name": "test-rpm-1.0-1.x86_64.rpm", + "sha256": "abc123", + }, + ], + [ + { + "pulp_href": "/pulp/api/v3/content/rpm/packages/def-456/", + "name": "test-rpm-1.0-1.src.rpm", + "sha256": "def456", + }, + ], + ] + + with patch( + "pushsource._impl.backend.konflux_source.konflux_source.Pulp3Client", + return_value=mock_client, + ): + result = anyio.run(source._resolve_rpm_hrefs, ["abc123", "def456"], 1) + + assert result == { + "abc123": "/pulp/api/v3/content/rpm/packages/abc-123/", + "def456": "/pulp/api/v3/content/rpm/packages/def-456/", + } + assert mock_client.build_query_sha256.call_count == 2 + assert mock_client.search_content.call_count == 2 + + +def test_resolve_rpm_hrefs_no_href(): + """Test RPM resolution when result has no pulp_href.""" + source = KonfluxSource(url=DATADIR, advisories="RHSA-2020:0509", **PULP_PARAMS) + + mock_client = AsyncMock() + mock_client.__aenter__.return_value = mock_client + mock_client.build_query_sha256 = MagicMock(return_value="(sha256=abc123)") + mock_client.search_content.return_value = [ + { + "name": "test-rpm-1.0-1.x86_64.rpm", + "sha256": "abc123", + } + ] + + with patch( + "pushsource._impl.backend.konflux_source.konflux_source.Pulp3Client", + return_value=mock_client, + ): + with pytest.raises(RuntimeError) as exc_info: + anyio.run(source._resolve_rpm_hrefs, ["abc123"]) + + assert "has no pulp_href" in str(exc_info.value) + + +def test_rpm_not_found_in_pulp(): + """Test error when RPM SHA256 is not found in Pulp results.""" + source = KonfluxSource(url=DATADIR, advisories="RHBA-2020:1234", **PULP_PARAMS) + + # Mock _resolve_rpm_hrefs to return empty map (RPM not found) + with patch( + "pushsource._impl.backend.konflux_source.konflux_source.anyio.run" + ) as mock_run: + mock_run.return_value = {} # No RPMs resolved + + with pytest.raises(RuntimeError) as exc_info: + source._create_rpm_items( + source._loader.load_advisory_data("RHBA-2020:1234") + ) + + assert "not found in Pulp" in str(exc_info.value) + + +def test_missing_sha256_checksum(): + """Test error when RPM has no SHA256 checksum in filelist.""" + source = KonfluxSource(url=DATADIR, advisories="RHBA-2020:1234", **PULP_PARAMS) + + # Load real data then remove sha256 checksums + data = source._loader.load_advisory_data("RHBA-2020:1234") + for build_data in data.filelist.values(): + build_data["checksums"]["sha256"] = {} + + with pytest.raises(RuntimeError) as exc_info: + source._create_rpm_items(data) + + assert "No SHA256 checksum found" in str(exc_info.value) + + +def test_missing_pulp_url(): + """Test that missing pulp_url raises RuntimeError at construction.""" + with pytest.raises(RuntimeError) as exc_info: + KonfluxSource( + url=DATADIR, + advisories="RHSA-2020:0509", + pulp_cert="/path/to/cert", + pulp_key="/path/to/key", + pulp_domain="konflux-myteam-tenant", + ) + + assert "pulp_url" in str(exc_info.value) + + +def test_missing_pulp_domain(): + """Test that missing pulp_domain raises RuntimeError at construction.""" + with pytest.raises(RuntimeError) as exc_info: + KonfluxSource( + url=DATADIR, + advisories="RHSA-2020:0509", + pulp_url="https://pulp.example.com", + pulp_cert="/path/to/cert", + pulp_key="/path/to/key", + ) + + assert "pulp_domain" in str(exc_info.value) + + +def test_missing_pulp_auth(): + """Test that missing Pulp auth raises RuntimeError at construction.""" + with pytest.raises(RuntimeError) as exc_info: + KonfluxSource( + url=DATADIR, + advisories="RHSA-2020:0509", + pulp_url="https://pulp.example.com", + pulp_domain="konflux-myteam-tenant", + ) + + assert "authentication not configured" in str(exc_info.value) + + +def test_basic_auth(): + """Test construction with username/password authentication.""" + source = KonfluxSource( + url=DATADIR, + advisories="RHSA-2020:0509", + pulp_url="https://pulp.example.com", + pulp_user="myuser", + pulp_password="mypassword", + pulp_domain="konflux-myteam-tenant", + ) + + assert source._pulp_client_kwargs["auth"] == ("myuser", "mypassword") + assert "cert" not in source._pulp_client_kwargs + + +def test_build_without_rpms(): + """Test that builds without 'rpms' key are skipped.""" + source = KonfluxSource(url=DATADIR, advisories="RHBA-2020:1234", **PULP_PARAMS) + + data = source._loader.load_advisory_data("RHBA-2020:1234") + + # Add a build entry without 'rpms' key + data.filelist["some-build-1.0-1.el8"] = { + "checksums": {"md5": {}, "sha256": {}}, + "sig_key": "FD431D51", + } + + with patch( + "pushsource._impl.backend.konflux_source.konflux_source.anyio.run", + side_effect=_mock_resolve_hrefs, + ): + items = source._create_rpm_items(data) + + # Should only have 1 RPM (from RHBA-2020:1234), not the build without rpms + assert len(items) == 1