From b2eb701c77ff760853391d647ca3299a0a0ef386 Mon Sep 17 00:00:00 2001 From: Yasin Date: Tue, 22 Aug 2023 10:28:30 +0200 Subject: [PATCH] fix: consider packages.conda for index update and channel mirroring (#638) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: tests – add: handle repodata now manages .conda files correctly in the database * add: test file – create with cph transmute * elobrate stmt for checking package format and error raising --- .../quetz_content_trust/main.py | 5 +- quetz/tasks/indexing.py | 2 +- quetz/tasks/mirror.py | 12 +++- quetz/tests/data/other-package-0.2-0.conda | Bin 0 -> 3148 bytes quetz/tests/test_mirror.py | 53 ++++++++++++++++-- 5 files changed, 63 insertions(+), 9 deletions(-) create mode 100644 quetz/tests/data/other-package-0.2-0.conda diff --git a/plugins/quetz_content_trust/quetz_content_trust/main.py b/plugins/quetz_content_trust/quetz_content_trust/main.py index 087bcebf..c8678df6 100644 --- a/plugins/quetz_content_trust/quetz_content_trust/main.py +++ b/plugins/quetz_content_trust/quetz_content_trust/main.py @@ -40,7 +40,10 @@ def post_index_creation(raw_repodata: dict, channel_name, subdir): from libmambapy import bindings as libmamba_api - for name, metadata in raw_repodata["packages"].items(): + packages = raw_repodata.get("packages", {}) | raw_repodata.get( + "packages.conda", {} + ) + for name, metadata in packages.items(): sig = libmamba_api.sign( json.dumps(metadata, indent=2, sort_keys=True), query[0].private_key ) diff --git a/quetz/tasks/indexing.py b/quetz/tasks/indexing.py index 45d8c255..a1f32562 100644 --- a/quetz/tasks/indexing.py +++ b/quetz/tasks/indexing.py @@ -222,7 +222,7 @@ def update_indexes(dao, pkgstore, channel_name, subdirs=None): logger.exception("Exception post_index_creation:") files[sdir] = [] - packages[sdir] = raw_repodata["packages"] + packages[sdir] = raw_repodata["packages"] | raw_repodata["packages.conda"] repodata = json.dumps(raw_repodata, indent=2, sort_keys=False) diff --git a/quetz/tasks/mirror.py b/quetz/tasks/mirror.py index 879d0717..cc20f749 100644 --- a/quetz/tasks/mirror.py +++ b/quetz/tasks/mirror.py @@ -327,7 +327,7 @@ def initial_sync_mirror( from quetz.main import handle_package_files - packages = repodata.get("packages", {}) + packages = repodata.get("packages", {}) | repodata.get("packages.conda", {}) version_methods = [ _check_checksum(dao, channel_name, arch, "sha256"), @@ -488,7 +488,15 @@ def create_version_from_metadata( ) dao.create_package(channel_name, package_info, user_id, "owner") - pkg_format = "tarbz2" if package_file_name.endswith(".tar.bz2") else ".conda" + if package_file_name.endswith(".conda"): + pkg_format = "conda" + elif package_file_name.endswith(".tar.bz2"): + pkg_format = "tarbz2" + else: + raise ValueError( + f"Unknown package format for package {package_file_name}" + f"in channel {channel_name}" + ) version = dao.create_version( channel_name, package_name, diff --git a/quetz/tests/data/other-package-0.2-0.conda b/quetz/tests/data/other-package-0.2-0.conda new file mode 100644 index 0000000000000000000000000000000000000000..3585926949b72c77622a14425506751ea04ebc24 GIT binary patch literal 3148 zcmai$c{mjM8pmhMG)9PFY-1byz75Kf7E2K^WT(a$yNqqJC;P6(+L$9D%TQ7nLm?6g z3E3J-M^pANq;#5G-TOTE-gBONpY#5n@AJODKfa&8KWIx37!LR~FM}o>-?s&kBoFRi z;XX)^Hv#K~C1BM9!h`X#(zsx}7Zwu|=!@|Q4hzB(FcIEi;r_vRX#*tcD*E3f06+?0 z)$26PcR=`+U$ldgS}?)SJ4`JEiwng1daG%wqYgsAhN;Jd6BMZ$EDwN!82|tU00BSJ z*^U2Vx&X-X<>gZZ1Ud4AaC$mRcgYS?=rysIAD=WS zbV^807{C2oBQyj&d&Q-CsrR95+|*>uJlyC-AH{ey7yeXPspT?9FgZ=A|C$r|#QbxyfI(w&mf9tIpmM z*FRb1aN^{EMOOo{X)8m^m1{$39WIG;$9;OnEt}8FNS~S^F9^~HXUamLma^G6gioGM zcHj8gvum;WjUjt_6Ag;WiwuF7vQ`;<#$|zbw8CuF&z z2aH{&?O>=;skO+eRR4P0#yQr}|iKDIt1V z>*E8GA}WJaF6S=Gh`X&%TtK^>k47u5P8Z!%2Gh?ywVwuHWwx+Lb z7L_f9;80F$!xyOyk*!PxT_w6D=}I_M`{RY1a_5HvvzqOVmz&^Q3W(Pa2E(1{>a!9q z%e=>4TH}NZe&)d{!ySx?Rnu>G%>u2|`d(zV9C!DfFPo97E0KWimq%UQMf_1`adt1x zkP;1_?ta66;j<&0>SF~m`}i>>gkKTszCBl}e4WVTkf}yi<%>tBdghczdfp+lHADxy z*~T;D?&2WIB7%ceiugq(R#cJ0iI+t8n>VO)yvOux?)#I~I1f#Pom<7t)~|n`9W-&V z(=$c8kJM?$&~R5~qy8Y9@kwE(Oyq7?iuZhSy);|Zs_xJ!Gx+N2p3{k3$cyQHqnlN3 z%y^gDRYHpHLmiDfQL}p2OTMWJ_uCL?x9>#SKZw1wG2|bxA*)=NUI$JhQ^K10lF|AI zi+tH2NRuW6=u%u=(eeqHUs9QZJd8?)%9ta2%ri5dWyv10IGrCp4@Kk>r-)YX@Y15* zx1oXtN1WSk05Ei|q4Kw)U{(ZP{yLrZx>SKi+r~Pr9Z|0=f4WXKxk(%9K7HwdP>=N& zY9zfpFgg52#ybfE)svkvyAws08YqkHR*%}20zDyujypFwAk0f@tu zHAGh47>x18z_gjN>zev2In>B4#Fhr+uGlcB#yOLeB4(`yHKX-JMcjq9%W!we6cq#!0tB*1;rHJuWClMX+}}ys|6FL zCgx;duh3XI8xu)LnE&`$a*^EtAF++5=^Jx$63Qe$)e~lE?NVNH&YY*T+%p&vwrP_Z z0cAVP0*XzjDyNLU#74j$P2XCNi)JxhF20XcK3{#Q=zdO`oxFlYg?6m}f~L;!W4^Or zBLfnSv@57hq<0rL8oXtUYOlS_HR>4ixQDu;CN1UTq}V*H7k4D_yF6(GQ?MUxcx!;| z)rj?(WYP}{eYPbrvu-HtGH)C$EKJJfqMgm`I!$zj^1dnFPb{uK%;Usd({^-(QS&W^ z#JYB1HtdF>lKk1cqdmhP<<2`Ld#q7}t4z5#Avw^lRhLUF@nz>449~S>#LiDGTRp>; zoJVI$!S8$H2EH*Bh-^cUFei;##U42hrLKSkk8C%df@ZzNyDwWe*{?{-Ej8jYQsqvZ zlF>BYY2>LdOqyQg27O}WCw~)^e6wVkN<}AcM9!^nN&5xQS?#;cSG?y_M;a`H<}y-| zhCjqwV6b-4h@4E#t%im@(ob%cit58oN2tbO)O)@mjr2Et^{=XTx-j+?9bX8aP*-tA z>|^dQvvWfHJ@Em3z5&NL%0?kjRj^JBnLHAgWd-rYw08*5Gm$p0wFSHOVI>6Jf?#iA z_TEV1o9HJe-?57>7aFkm(~k?VE{Jh@W#~wn+8isMM5Rem;ikg{V9~=LzNAy18!^B|*HSXqywtuO$M8DrC z*n00<=!ZJ*0L9Wn3p@$e7lEtyNB2N~kR>w0iDV)q@xdbnJzjl=>@IT@B`vyn8$zm~ z{a69K$S49ivbByv3wJ~saN0Lu#T6RHf!e(iJtp3fYY(+8CQg@sK#y`{4fzJhwUv*5E-k-cf-QiggrtObN(=MEnIEN0m+@JeQ!`#v}N z{s8-6FUe)`DdtR|ZH?@>_LxhGFMeu3@U)uXXPvm@d#SI`80FW)7iai!rI;NYx~YS=FuSS@LRXf-9kDmlxs;+4z`{%Tc+bttx?g2ze7TJ0 zZ=vQiagkK7P3E&Gd=SdNrgFb20rY4vOn2lN6Z`W?l%5jbbbm-Y>&{#^xTkRHAvX;> zV4R!P+NkOUqGtj<@TV-2torv8jaq;uBHXT$ns7URb_xsnT9cL2uI7xkftAGNX5I}( zny*8b&ifv#m(*xQ@0@PjXlxa>KAUF-Abl}2E)Ys|ENq0d=$I}Y0hCsV1o1O~dY(SV zRE3I`hQvH9nh;2egW_ec*yPN6h(SM9OU;;>UBH_uj+C(oHO=tjwh(m2gL!9!XNC9u zw*qoNv?Y)Y4*LIwec-bHoCM&n^(Wu{@9O_j>VFjhfCS*VgY$0$`>(LS3iW?rB?sr< W={4H&5ad?_bnvDQwg>00-#-AQr*BCB literal 0 HcmV?d00001 diff --git a/quetz/tests/test_mirror.py b/quetz/tests/test_mirror.py index bb1ab7e1..ee59efb9 100644 --- a/quetz/tests/test_mirror.py +++ b/quetz/tests/test_mirror.py @@ -304,6 +304,7 @@ def owner(user, db): DUMMY_PACKAGE_V2 = Path("./test-package-0.2-0.tar.bz2") OTHER_DUMMY_PACKAGE = Path("./other-package-0.1-0.tar.bz2") OTHER_DUMMY_PACKAGE_V2 = Path("./other-package-0.2-0.tar.bz2") +OTHER_DUMMY_PACKAGE_V2_CONDA = Path("./other-package-0.2-0.conda") @pytest.mark.parametrize( @@ -712,6 +713,10 @@ def test_api_methods_for_mirror_channels(client, mirror_channel): ], ) def test_mirror_initial_sync(client, dummy_repo, owner, expected_paths, job_supervisor): + """ + Validate that, after the sync of the mirrored channel, the correct files + (e.g. repodata.json) are present here. + """ response = client.get("/api/dummylogin/bartosz") assert response.status_code == 200 @@ -726,7 +731,7 @@ def test_mirror_initial_sync(client, dummy_repo, owner, expected_paths, job_supe }, ) assert response.status_code == 201 - job_supervisor.run_once() + job_supervisor.run_once() # the sync job is added implicitly to new mirror channels assert dummy_repo == [os.path.join(host, p) for p in expected_paths] @@ -1175,6 +1180,35 @@ def test_can_not_sync_proxy_and_local_channels( } """ +repodata_json_conda = """ +{ + "info": { + "subdir": "linux-64" + }, + "packages": {}, + "packages.conda": { + "other-package-0.2-0.conda": { + "arch": "x86_64", + "build": "0", + "build_number": 0, + "depends": [], + "license": "BSD", + "license_family": "BSD", + "md5": "f5764fa8299aa117fce80be10d76724c", + "name": "other-package", + "platform": "linux", + "sha256": "3ca41044557c3179cb898152ec9414608a55a713cf6dcc1a1b71aed63f4d611c", + "size": 3148, + "subdir": "linux-64", + "timestamp": 1599839787253, + "version": "0.2", + "time_modified": 1608636247 + } + }, + "repodata_version": 1 +} +""" + def test_create_packages_from_channeldata(dao, user, local_channel, db): channeldata = json.loads(channeldata_json) @@ -1249,8 +1283,12 @@ def test_create_versions_from_repodata(dao, user, local_channel, db): @pytest.fixture -def dummy_package_file(config): - filepath = OTHER_DUMMY_PACKAGE_V2 +def dummy_package_file(config, request): + format = request.param if hasattr(request, "param") else ".tar.bz2" + if format == '.conda': + filepath = OTHER_DUMMY_PACKAGE_V2_CONDA + else: # default case: .tar.bz2 + filepath = OTHER_DUMMY_PACKAGE_V2 fid = open(filepath, 'rb') class DummyRemoteFile: @@ -1271,14 +1309,19 @@ def rules(user, db): @pytest.mark.parametrize("user_role", ["owner"]) +@pytest.mark.parametrize("dummy_package_file", [".tar.bz2", ".conda"], indirect=True) def test_handle_repodata_package( dao, user, local_channel, dummy_package_file, rules, config, db ): pkg = Package(name="other-package", channel=local_channel) db.add(pkg) - repodata = json.loads(repodata_json) - package_name, package_data = list(repodata["packages"].items())[0] + if dummy_package_file.filename.endswith(".tar.bz2"): + repodata = json.loads(repodata_json) + package_name, package_data = list(repodata["packages"].items())[0] + else: + repodata = json.loads(repodata_json_conda) + package_name, package_data = list(repodata["packages.conda"].items())[0] pkgstore = config.get_package_store() files_metadata = [(dummy_package_file, package_name, package_data)]