From ff8a0be19472c2caff839c8b6f35acfacc4a723d Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Fri, 3 Jan 2025 18:24:53 +0100 Subject: [PATCH 01/11] Init s3 upload --- Cargo.lock | 850 ++++++++++++++++++++++++++++++--- Cargo.toml | 66 ++- pixi.lock | 98 ++++ pixi.toml | 2 + src/lib.rs | 11 + src/opt.rs | 33 ++ src/upload/mod.rs | 60 +++ test/end-to-end/test_simple.py | 74 +++ 8 files changed, 1106 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a7b6d8969..db4ee3d72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -408,6 +408,380 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-config" +version = "1.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "649316840239f4e58df0b7f620c428f5fababbbca2d504488c641534050bd141" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 0.2.12", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f6f1124d6e19ab6daf7f2e615644305dc6cb2d706892a8a8c0b98db35de020" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.67.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbc644164269a1e38ce7f2f7373629d3fb3d310c0e3feb5573a29744288b24d3" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http-body 0.4.6", + "lru", + "once_cell", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb25f7129c74d36afe33405af4517524df8f74b635af8c2c8e91c1552b8397b2" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d03a3d5ef14851625eafd89660a751776f938bf32f309308b20dcca41c44b568" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3a9f073ae3a53b54421503063dfb87ff1ea83b876f567d92e8b8d9942ba91b" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.2.0", + "once_cell", + "p256", + "percent-encoding", + "ring", + "sha2", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427cb637d15d63d6f9aae26358e1c9a9c09d5aa490d64b09354c8217cfef0f28" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.61.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a05dd41a70fc74051758ee75b5c4db2c0ca070ed9229c3df50e9475cda1cb985" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "http-body 1.0.1", + "httparse", + "hyper 0.14.32", + "hyper-rustls 0.24.2", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls 0.21.12", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.2.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ddc9bd6c28aeb303477170ddd183760a956a03e083b3902a990238a7e3792d" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.2.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -432,6 +806,12 @@ dependencies = [ "backtrace", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.21.7" @@ -444,6 +824,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.6.0" @@ -554,6 +944,16 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bzip2" version = "0.4.4" @@ -846,6 +1246,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -896,6 +1302,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -962,6 +1377,28 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -1077,6 +1514,16 @@ dependencies = [ "sha2", ] +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.11" @@ -1171,12 +1618,44 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der", + "elliptic-curve", + "rfc6979", + "signature", +] + [[package]] name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der", + "digest", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "elsa" version = "1.10.0" @@ -1311,11 +1790,19 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "file_url" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ad84121770c9766a67e4dd0a96b0467c6a5a2c43e873bf853aeaa6632b687c" dependencies = [ "itertools 0.13.0", "percent-encoding", @@ -1679,6 +2166,36 @@ dependencies = [ "async-trait", ] +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core", + "subtle", +] + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.7.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.7" @@ -1690,7 +2207,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.2.0", "indexmap 2.7.0", "slab", "tokio", @@ -1823,6 +2340,17 @@ dependencies = [ "utf8-width", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.2.0" @@ -1834,6 +2362,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1841,7 +2380,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.2.0", ] [[package]] @@ -1852,8 +2391,8 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http", - "http-body", + "http 1.2.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -1863,7 +2402,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92baf25cf0b8c9246baecf3a444546360a97b569168fdf92563ee6a47829920c" dependencies = [ - "http", + "http 1.2.0", "http-serde", "reqwest", "serde", @@ -1882,7 +2421,7 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f056c8559e3757392c8d091e796416e4649d8e49e88b8d76df6c002f05027fd" dependencies = [ - "http", + "http 1.2.0", "serde", ] @@ -1892,6 +2431,12 @@ version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humansize" version = "2.1.3" @@ -1907,6 +2452,30 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.5.2" @@ -1916,9 +2485,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.7", + "http 1.2.0", + "http-body 1.0.1", "httparse", "itoa", "pin-project-lite", @@ -1927,6 +2496,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.5" @@ -1934,14 +2519,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", - "http", - "hyper", + "http 1.2.0", + "hyper 1.5.2", "hyper-util", - "rustls", - "rustls-native-certs", + "rustls 0.23.20", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.1", "tower-service", "webpki-roots", ] @@ -1954,7 +2539,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper", + "hyper 1.5.2", "hyper-util", "native-tls", "tokio", @@ -1971,9 +2556,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.2.0", + "http-body 1.0.1", + "hyper 1.5.2", "pin-project-lite", "socket2", "tokio", @@ -2971,6 +3556,12 @@ dependencies = [ "syn", ] +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + [[package]] name = "overload" version = "0.1.1" @@ -2983,6 +3574,17 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb37767f6569cd834a413442455e0f066d0d522de8630436e2a1761d9726ba56" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2", +] + [[package]] name = "parking" version = "2.2.1" @@ -3251,6 +3853,16 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.31" @@ -3401,7 +4013,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls", + "rustls 0.23.20", "socket2", "thiserror 2.0.9", "tokio", @@ -3419,7 +4031,7 @@ dependencies = [ "rand", "ring", "rustc-hash", - "rustls", + "rustls 0.23.20", "rustls-pki-types", "slab", "thiserror 2.0.9", @@ -3511,8 +4123,6 @@ dependencies = [ [[package]] name = "rattler" version = "0.28.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c90d5a0dd356a4b2289752dd738d5597348d616a7797ec045209c4cc8b65bb9b" dependencies = [ "anyhow", "clap", @@ -3558,6 +4168,9 @@ dependencies = [ "anyhow", "async-once-cell", "async-recursion", + "aws-config", + "aws-sdk-s3", + "aws-smithy-types", "base64 0.22.1", "bzip2 0.5.0", "chrono", @@ -3646,8 +4259,6 @@ dependencies = [ [[package]] name = "rattler_cache" version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266a244026ae404be929fa7cdb36761848dadc238868448d0fcb6c900522ee72" dependencies = [ "anyhow", "dashmap", @@ -3676,8 +4287,6 @@ dependencies = [ [[package]] name = "rattler_conda_types" version = "0.29.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f01db87a34cb9fc92d7be8a8789c79a567c8738a535f3cb79779a8580c33d6" dependencies = [ "chrono", "dirs", @@ -3730,8 +4339,6 @@ dependencies = [ [[package]] name = "rattler_digest" version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ffd5141cd51f0de194c8c05d97f138d2ea70274636ce911bbdf7dd371942a47" dependencies = [ "blake2", "digest", @@ -3747,8 +4354,6 @@ dependencies = [ [[package]] name = "rattler_index" version = "0.20.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c1880cb859882d8863dc67d2fe2a89baf4f749df5491a217972c3c1041fc8b" dependencies = [ "fs-err 3.0.0", "rattler_conda_types", @@ -3784,7 +4389,7 @@ dependencies = [ "fs_extra", "futures", "html-escape", - "http", + "http 1.2.0", "http-cache-semantics", "include_dir", "indexmap 2.7.0", @@ -3823,8 +4428,6 @@ dependencies = [ [[package]] name = "rattler_macros" version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae24d4a02bfa3805a358d4adcdcb6426f0045c56189f13dc69234b1e4fe9fbe" dependencies = [ "quote", "syn", @@ -3833,18 +4436,19 @@ dependencies = [ [[package]] name = "rattler_networking" version = "0.21.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d10545baa6dc04e294d50984aca155d57ee6e92600296a8d42b6d6d27ded833a" dependencies = [ "anyhow", "async-trait", + "aws-config", + "aws-runtime", + "aws-sdk-s3", "base64 0.22.1", "chrono", "dirs", "fslock", "getrandom", "google-cloud-auth", - "http", + "http 1.2.0", "itertools 0.13.0", "keyring", "netrc-rs", @@ -3861,8 +4465,6 @@ dependencies = [ [[package]] name = "rattler_package_streaming" version = "0.22.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a505f4ed90ca9c4eed477d3184bc38621c5d2c334be28bcdde7f16e7c4ef0c" dependencies = [ "bzip2 0.5.0", "chrono", @@ -3890,8 +4492,6 @@ dependencies = [ [[package]] name = "rattler_redaction" version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd7c1c0dd452161e62e6d112ebd12618e61f641609d5205c3a68eb9907254dc2" dependencies = [ "reqwest", "reqwest-middleware", @@ -3901,8 +4501,6 @@ dependencies = [ [[package]] name = "rattler_repodata_gateway" version = "0.21.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ef851fac32e22323c735e5eda2f966687902f595ab3d6f25cf28e2f89e16ced" dependencies = [ "anyhow", "async-compression", @@ -3918,7 +4516,7 @@ dependencies = [ "fs-err 3.0.0", "futures", "hex", - "http", + "http 1.2.0", "http-cache-semantics", "humansize", "humantime", @@ -3956,8 +4554,6 @@ dependencies = [ [[package]] name = "rattler_shell" version = "0.22.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e4f21b5783b0ce087d816669a4539add1ce5c076050856e350435a9a27c61f" dependencies = [ "enum_dispatch", "fs-err 3.0.0", @@ -3975,8 +4571,6 @@ dependencies = [ [[package]] name = "rattler_solve" version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4c73c74bd7747ebe16607055eaa74c2daad359514a0765994642d6fa446193" dependencies = [ "chrono", "futures", @@ -3994,8 +4588,6 @@ dependencies = [ [[package]] name = "rattler_virtual_packages" version = "1.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0e792417bd869e30b9ff9611dd56664de1352c810fb88c53ea4bb543ef6e8fe" dependencies = [ "archspec", "libloading", @@ -4121,6 +4713,12 @@ dependencies = [ "regex-syntax 0.8.5", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -4151,12 +4749,12 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.7", + "http 1.2.0", + "http-body 1.0.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.5.2", + "hyper-rustls 0.27.5", "hyper-tls", "hyper-util", "ipnet", @@ -4169,9 +4767,9 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", - "rustls-native-certs", - "rustls-pemfile", + "rustls 0.23.20", + "rustls-native-certs 0.8.1", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", @@ -4179,7 +4777,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.26.1", "tokio-util", "tower", "tower-service", @@ -4200,7 +4798,7 @@ checksum = "d1ccd3b55e711f91a9885a2fa6fbbb2e39db1776420b062efc058c6410f7e5e3" dependencies = [ "anyhow", "async-trait", - "http", + "http 1.2.0", "reqwest", "serde", "thiserror 1.0.69", @@ -4217,8 +4815,8 @@ dependencies = [ "async-trait", "futures", "getrandom", - "http", - "hyper", + "http 1.2.0", + "hyper 1.5.2", "parking_lot 0.11.2", "reqwest", "reqwest-middleware", @@ -4271,6 +4869,17 @@ dependencies = [ "rand", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "ring" version = "0.17.8" @@ -4383,6 +4992,18 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.20" @@ -4392,11 +5013,23 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework 2.11.1", +] + [[package]] name = "rustls-native-certs" version = "0.8.1" @@ -4409,6 +5042,15 @@ dependencies = [ "security-framework 3.1.0", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -4427,6 +5069,16 @@ dependencies = [ "web-time", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.102.8" @@ -4494,6 +5146,30 @@ dependencies = [ "syn", ] +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "secret-service" version = "4.0.0" @@ -4772,6 +5448,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "simd-adler32" version = "0.3.7" @@ -4820,8 +5506,6 @@ dependencies = [ [[package]] name = "simple_spawn_blocking" version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b31ed96d1593e129cc76cb7aca364fb5c173558bfda922c15aac4e2f2f5844e" dependencies = [ "tokio", ] @@ -4886,6 +5570,16 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "ssri" version = "9.2.0" @@ -5247,13 +5941,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls", + "rustls 0.23.20", "tokio", ] @@ -5638,6 +6342,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "vt100" version = "0.15.2" @@ -6217,6 +6927,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "xxhash-rust" version = "0.8.15" diff --git a/Cargo.toml b/Cargo.toml index 83997ca4c..3d4d19dc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,12 @@ serde_json = "1.0.134" reqwest = { version = "0.12.12", default-features = false, features = [ "multipart", ] } -tokio = { version = "1.42.0", features = ["rt", "macros", "rt-multi-thread", "process"] } +tokio = { version = "1.42.0", features = [ + "rt", + "macros", + "rt-multi-thread", + "process", +] } itertools = "0.13.0" content_inspector = "0.2.4" serde_with = "3.12.0" @@ -129,20 +134,39 @@ regex = "1.11.1" async-recursion = "1.1.1" # Rattler crates -rattler = { version = "0.28.9", default-features = false, features = ["cli-tools", "indicatif"] } +rattler = { version = "0.28.9", default-features = false, features = [ + "cli-tools", + "indicatif", +] } rattler_cache = { version = "0.3.1", default-features = false } -rattler_conda_types = { version = "0.29.7", default-features = false, features = ["rayon"] } -rattler_digest = { version = "1.0.4", default-features = false, features = ["serde"] } +rattler_conda_types = { version = "0.29.7", default-features = false, features = [ + "rayon", +] } +rattler_digest = { version = "1.0.4", default-features = false, features = [ + "serde", +] } rattler_index = { version = "0.20.4", default-features = false } -rattler_networking = { version = "0.21.9", default-features = false } +rattler_networking = { version = "0.21.9", default-features = false, features = [ + "s3", +] } rattler_redaction = { version = "0.1.5" } -rattler_repodata_gateway = { version = "0.21.29", default-features = false, features = ["gateway"] } -rattler_shell = { version = "0.22.12", default-features = false, features = ["sysinfo"] } -rattler_solve = { version = "1.3.1", default-features = false, features = ["resolvo", "serde"] } +rattler_repodata_gateway = { version = "0.21.29", default-features = false, features = [ + "gateway", +] } +rattler_shell = { version = "0.22.12", default-features = false, features = [ + "sysinfo", +] } +rattler_solve = { version = "1.3.1", default-features = false, features = [ + "resolvo", + "serde", +] } rattler_virtual_packages = { version = "1.1.15", default-features = false } rattler_package_streaming = { version = "0.22.20", default-features = false } lazy_static = "1.5.0" reqwest-retry = "0.7.0" +aws-smithy-types = { version = "1.2.11", features = ["rt-tokio"] } +aws-config = "1.5.12" +aws-sdk-s3 = "1.65.0" [target.'cfg(not(target_os = "windows"))'.dependencies] sha2 = { version = "0.10.8", features = ["asm"] } @@ -173,17 +197,17 @@ pre-build = [ [patch.crates-io] # this fork contains fixes for slow zip reading and large file writing -zip = { git = "https://github.com/wolfv/zip2", branch = "patched"} +zip = { git = "https://github.com/wolfv/zip2", branch = "patched" } -# rattler = { path = "../rattler/crates/rattler" } -# rattler_cache = { path = "../rattler/crates/rattler_cache" } -# rattler_conda_types = { path = "../rattler/crates/rattler_conda_types" } -# rattler_digest = { path = "../rattler/crates/rattler_digest" } -# rattler_index = { path = "../rattler/crates/rattler_index" } -# rattler_networking = { path = "../rattler/crates/rattler_networking" } -# rattler_repodata_gateway = { path = "../rattler/crates/rattler_repodata_gateway" } -# rattler_shell = { path = "../rattler/crates/rattler_shell" } -# rattler_solve = { path = "../rattler/crates/rattler_solve" } -# rattler_redaction = { path = "../rattler/crates/rattler_redaction" } -# rattler_virtual_packages = { path = "../rattler/crates/rattler_virtual_packages" } -# rattler_package_streaming = { path = "../rattler/crates/rattler_package_streaming" } +rattler = { path = "../pavelzw-rattler/crates/rattler" } +rattler_cache = { path = "../pavelzw-rattler/crates/rattler_cache" } +rattler_conda_types = { path = "../pavelzw-rattler/crates/rattler_conda_types" } +rattler_digest = { path = "../pavelzw-rattler/crates/rattler_digest" } +rattler_index = { path = "../pavelzw-rattler/crates/rattler_index" } +rattler_networking = { path = "../pavelzw-rattler/crates/rattler_networking" } +rattler_repodata_gateway = { path = "../pavelzw-rattler/crates/rattler_repodata_gateway" } +rattler_shell = { path = "../pavelzw-rattler/crates/rattler_shell" } +rattler_solve = { path = "../pavelzw-rattler/crates/rattler_solve" } +rattler_redaction = { path = "../pavelzw-rattler/crates/rattler_redaction" } +rattler_virtual_packages = { path = "../pavelzw-rattler/crates/rattler_virtual_packages" } +rattler_package_streaming = { path = "../pavelzw-rattler/crates/rattler_package_streaming" } diff --git a/pixi.lock b/pixi.lock index 70c40d44d..04a1a23f4 100644 --- a/pixi.lock +++ b/pixi.lock @@ -82,6 +82,8 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/make-4.3-hd18ef5c_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/mimalloc-2.1.7-hac33072_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/minio-client-2024.11.21.17.21.54-h199d65c_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/minio-server-2024.03.10.02.53.48-hbcca054_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/mold-2.35.1-hc93959d_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-he02047a_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.9.1-pyhd8ed1ab_1.conda @@ -189,6 +191,8 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/llvm-openmp-19.1.5-ha54dae1_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/llvm-tools-15.0.7-hbedff68_4.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/make-4.3-h22f3db7_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-64/minio-client-2024.11.21.17.21.54-hd81af70_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/minio-server-2024.03.10.02.53.48-h8857fd0_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/mpc-1.3.1-h9d8efa1_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/mpfr-4.2.1-haed47dc_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-hf036a51_1.conda @@ -301,6 +305,8 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/llvm-openmp-19.1.5-hdb05f8b_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/llvm-tools-15.0.7-h2621b3d_4.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/make-4.3-he57ea6c_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/minio-client-2024.11.21.17.21.54-h7b83518_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/minio-server-2024.03.10.02.53.48-hf0a4a13_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/mpc-1.3.1-h8f1351a_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/mpfr-4.2.1-hb693164_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h7bae524_1.conda @@ -391,6 +397,8 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/m2w64-gmp-6.1.0-2.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/win-64/m2w64-libwinpthread-git-5.0.0.4634.697f757-2.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/win-64/make-4.3-h3d2af85_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/win-64/minio-client-2024.11.21.17.21.54-h265662a_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/minio-server-2024.03.10.02.53.48-h56e8100_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/msys2-conda-epoch-20160418-1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.9.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/openmp-5.0.0-vc14_1.tar.bz2 @@ -4086,6 +4094,96 @@ packages: license_family: MIT size: 78385 timestamp: 1716552401566 +- conda: https://conda.anaconda.org/conda-forge/linux-64/minio-client-2024.11.21.17.21.54-h199d65c_0.conda + sha256: 2a89995c555c906336c7307976229ca7b378b536e26719c44ad8f610726ba641 + md5: d6132a4a7474f3d59c40c677b2cfb779 + depends: + - libgcc >=13 + - __glibc >=2.17,<3.0.a0 + arch: x86_64 + platform: linux + license: AGPL-3.0-or-later + license_family: AGPL + size: 18807854 + timestamp: 1735897852601 +- conda: https://conda.anaconda.org/conda-forge/osx-64/minio-client-2024.11.21.17.21.54-hd81af70_0.conda + sha256: b4628c7501ddd9a3288bbf8d8f14e5edaa25d2feaa9ecd8c48b1517834915959 + md5: adef5a4b7b2361701258caed17df8ca2 + depends: + - __osx >=10.13 + constrains: + - __osx >=10.12 + arch: x86_64 + platform: osx + license: AGPL-3.0-or-later + license_family: AGPL + size: 19134629 + timestamp: 1735897901873 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/minio-client-2024.11.21.17.21.54-h7b83518_0.conda + sha256: ae059dc56c2832bee4e68e090746b9e6d93132b6262fa25b51adde5686f43c6d + md5: f89f48621c6cd54c3d8bc06f830e1728 + depends: + - __osx >=11.0 + arch: arm64 + platform: osx + license: AGPL-3.0-or-later + license_family: AGPL + size: 17883741 + timestamp: 1735898727826 +- conda: https://conda.anaconda.org/conda-forge/win-64/minio-client-2024.11.21.17.21.54-h265662a_0.conda + sha256: a37a81b0dce7bb7dc57d59dde0e1d08eda0a16e267f7b2d82cf7e990ea3f84d0 + md5: 0e22939f2109a4a0e113ecf3fab8423f + depends: + - vc >=14.2,<15 + - vc14_runtime >=14.29.30139 + - ucrt >=10.0.20348.0 + - vc >=14.2,<15 + - vc14_runtime >=14.29.30139 + - ucrt >=10.0.20348.0 + arch: x86_64 + platform: win + license: AGPL-3.0-or-later + license_family: AGPL + size: 19174140 + timestamp: 1735897885467 +- conda: https://conda.anaconda.org/conda-forge/linux-64/minio-server-2024.03.10.02.53.48-hbcca054_0.conda + sha256: de0b4886ca7b1a27008f3b7d33500550191cd571765a5adb1d7315d40efd4a9e + md5: 3c3572872ca7690eadfaf532a57bfe1b + arch: x86_64 + platform: linux + license: AGPL-3.0-only + license_family: AGPL + size: 29338301 + timestamp: 1710216649717 +- conda: https://conda.anaconda.org/conda-forge/osx-64/minio-server-2024.03.10.02.53.48-h8857fd0_0.conda + sha256: 1b120375ab7629331212a36f22f842288e266b06ab49a961825474dd031e3981 + md5: e4c421d22e108008340181bbc7b70582 + constrains: + - __osx>=10.12 + arch: x86_64 + platform: osx + license: AGPL-3.0-only + license_family: AGPL + size: 29684080 + timestamp: 1710217094567 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/minio-server-2024.03.10.02.53.48-hf0a4a13_0.conda + sha256: 7c29eb913e1d5e8efcb5af9dd908c1e4ef13b3e87cdc486a9ade6c7a8106a2ef + md5: a69c70cb150ba41a60e81b03fc94a954 + arch: arm64 + platform: osx + license: AGPL-3.0-only + license_family: AGPL + size: 28447785 + timestamp: 1710216865211 +- conda: https://conda.anaconda.org/conda-forge/win-64/minio-server-2024.03.10.02.53.48-h56e8100_0.conda + sha256: 575c723ba0ae2d10c1b340afd229c41efd80baeee4c6d75d7f249b449ad66348 + md5: 6126ff5a23af33c9fa987f11783b62e5 + arch: x86_64 + platform: win + license: AGPL-3.0-only + license_family: AGPL + size: 29699962 + timestamp: 1710217343984 - conda: https://conda.anaconda.org/conda-forge/noarch/mkdocs-1.5.3-pyhd8ed1ab_0.conda sha256: cd57f3805a38bc3b3356a37d4bd9af7e227972286f61eae3c88d356216bc7159 md5: 1e432aebaa009b030cce33a554939d8e diff --git a/pixi.toml b/pixi.toml index 03d372b55..474f9af04 100644 --- a/pixi.toml +++ b/pixi.toml @@ -34,6 +34,8 @@ cmake = "3.27.6.*" make = "4.3.*" perl = "5.32.1.*" pytest = "7.4.2.*" +minio-server = "*" +minio-client = "*" pyyaml = ">=6.0.1,<6.1" conda-package-handling = "2.2.0.*" requests = ">=2.32.2,<2.33" diff --git a/src/lib.rs b/src/lib.rs index 25f8f1aaf..942fc3951 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -781,6 +781,17 @@ pub async fn upload_from_args(args: UploadOpts) -> miette::Result<()> { ) .await } + ServerType::S3(s3_opts) => { + upload::upload_package_to_s3( + &store, + s3_opts.channel, + s3_opts.endpoint_url, + s3_opts.region, + s3_opts.force_path_style, + &args.package_files, + ) + .await + } ServerType::CondaForge(conda_forge_opts) => { upload::conda_forge::upload_packages_to_conda_forge( conda_forge_opts, diff --git a/src/opt.rs b/src/opt.rs index 6b5ce2427..b5cb4df9c 100644 --- a/src/opt.rs +++ b/src/opt.rs @@ -502,6 +502,7 @@ pub enum ServerType { Artifactory(ArtifactoryOpts), Prefix(PrefixOpts), Anaconda(AnacondaOpts), + S3(S3Opts), #[clap(hide = true)] CondaForge(CondaForgeOpts), } @@ -603,6 +604,38 @@ pub struct AnacondaOpts { pub force: bool, } +fn parse_s3_url(value: &str) -> Result { + let url: Url = Url::parse(value).map_err(|_| format!("`{}` isn't a valid URL", value))?; + if url.scheme() == "s3" && url.host_str().is_some() { + Ok(url) + } else { + Err(format!( + "Only S3 URLs of format s3://bucket/... can be used, not `{}`", + value + )) + } +} + +/// Options for uploading to S3 +#[derive(Clone, Debug, PartialEq, Parser)] +pub struct S3Opts { + /// The channel URL in the S3 bucket to upload the package to, e.g., s3://my-bucket/my-channel + #[arg(short, long, env = "S3_CHANNEL", value_parser = parse_s3_url)] + pub channel: Url, + + /// The endpoint URL of the S3 backend + #[arg(short, long, env = "S3_ENDPOINT_URL", requires_all = ["region", "force_path_style"])] + pub endpoint_url: Option, + + /// The region of the S3 backend + #[arg(short, long, env = "S3_REGION", requires_all = ["endpoint_url", "force_path_style"])] + pub region: Option, + + /// Whether to use path-style S3 URLs + #[arg(short, long, env = "S3_FORCE_PATH_STYLE", requires_all = ["endpoint_url", "region"])] + pub force_path_style: Option, +} + /// Options for uploading to conda-forge #[derive(Clone, Debug, PartialEq, Parser)] pub struct CondaForgeOpts { diff --git a/src/upload/mod.rs b/src/upload/mod.rs index 291178deb..11c62fbad 100644 --- a/src/upload/mod.rs +++ b/src/upload/mod.rs @@ -12,6 +12,7 @@ use trusted_publishing::{check_trusted_publishing, TrustedPublishResult}; use crate::url_with_trailing_slash::UrlWithTrailingSlash; use miette::{Context, IntoDiagnostic}; +use rattler_networking::s3_middleware::{create_s3_client, S3Config}; use rattler_networking::{Authentication, AuthenticationStorage}; use rattler_redaction::Redact; use reqwest::Method; @@ -312,6 +313,65 @@ pub async fn upload_package_to_anaconda( Ok(()) } +/// Uploads a package to a channel in an S3 bucket. +pub async fn upload_package_to_s3( + storage: &AuthenticationStorage, + channel: Url, + endpoint_url: Option, + region: Option, + force_path_style: Option, + package_files: &Vec, +) -> miette::Result<()> { + // If either endpoint_url or region is set, ensure that both are set + if endpoint_url.is_some() != region.is_some() { + return Err(miette::miette!( + "Both endpoint_url and region must be specified together" + )); + } + let s3_client = if let (Some(endpoint_url), Some(region), Some(force_path_style)) = + (endpoint_url, region, force_path_style) + { + let s3_config = S3Config { + auth_storage: storage.clone(), + endpoint_url, + region, + force_path_style, + }; + create_s3_client(Some(s3_config), Some(channel.clone())).await + } else { + create_s3_client(None, None).await + }; + + let bucket = channel + .host_str() + .ok_or_else(|| miette::miette!("Failed to get host from channel URL"))?; + let channel = channel.path().trim_start_matches('/'); + + for package_file in package_files { + let package = ExtractedPackage::from_package_file(package_file)?; + let subdir = package + .subdir() + .ok_or_else(|| miette::miette!("Failed to get subdir"))?; + let filename = package + .filename() + .ok_or_else(|| miette::miette!("Failed to get filename"))?; + let key = format!("{}/{}/{}", channel, subdir, filename); + let body = aws_smithy_types::byte_stream::ByteStream::from_path(package_file) + .await + .into_diagnostic()?; + s3_client + .put_object() + .key(key) + .body(body) + .bucket(bucket) + .send() + .await + .into_diagnostic()?; + } + + Ok(()) +} + async fn send_request( prepared_request: reqwest::RequestBuilder, package_file: &Path, diff --git a/test/end-to-end/test_simple.py b/test/end-to-end/test_simple.py index 9e2781940..10eb9df5a 100644 --- a/test/end-to-end/test_simple.py +++ b/test/end-to-end/test_simple.py @@ -2,8 +2,10 @@ import json import os import platform +import subprocess from pathlib import Path from subprocess import DEVNULL, STDOUT, CalledProcessError, check_output +from time import sleep import pytest import requests @@ -208,6 +210,78 @@ def test_anaconda_upload( assert requests.get(URL).status_code == 200 +@pytest.fixture() +def minio_server(tmp_path: Path): + tmp_dir = tmp_path / "minio" + # Start minio server + proc = subprocess.Popen( + ["minio", "server", "--address", "localhost:9000", str(tmp_dir)], + ) + # Create a bucket + subprocess.run( + ["mc", "mb", "local/s3-forge"], + env={ + **os.environ, + "MC_HOST_local": "http://minioadmin:minioadmin@localhost:9000", + }, + check=True, + ) + yield + proc.kill() + + +@pytest.fixture() +def s3_credentials_file(tmp_path: Path) -> Path: + path = tmp_path / "credentials.json" + path.write_text( + """\ +{ + "s3://s3-forge/channel": { + "S3Credentials": { + "access_key_id": "minioadmin", + "secret_access_key": "minioadmin" + } + } +}""" + ) + return path + + +@pytest.mark.usefixtures("minio_server") +def test_s3_minio_upload( + rattler_build: RattlerBuild, + recipes: Path, + tmp_path: Path, + s3_credentials_file: Path, + monkeypatch, +): + monkeypatch.setenv("RATTLER_AUTH_FILE", str(s3_credentials_file)) + rattler_build.build(recipes / "globtest", tmp_path) + rattler_build( + "upload", + "-vvv", + "s3", + "--channel", + "s3://s3-forge/channel", + "--region", + "eu-central-1", + "--endpoint-url", + "http://localhost:9000", + "--force-path-style", + "true", + str(get_package(tmp_path, "globtest")), + ) + + # Make sure the package was uploaded + assert b"globtest" in subprocess.check_output( + ["mc", "ls", "--recursive", "local/s3-forge/channel"], + env={ + **os.environ, + "MC_HOST_local": "http://minioadmin:minioadmin@localhost:9000", + }, + ) + + @pytest.mark.skipif( os.name == "nt", reason="recipe does not support execution on windows" ) From f3b3b103d74fdd9bd22f45e27f3b55afbcd4a78f Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Fri, 3 Jan 2025 18:37:21 +0100 Subject: [PATCH 02/11] Adapt new s3config api --- src/upload/mod.rs | 11 ++++++----- .../prefix_detection/create_files_with_prefix.py | 4 +++- test/end-to-end/test_simple.py | 1 - 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/upload/mod.rs b/src/upload/mod.rs index 11c62fbad..ea5456671 100644 --- a/src/upload/mod.rs +++ b/src/upload/mod.rs @@ -328,20 +328,21 @@ pub async fn upload_package_to_s3( "Both endpoint_url and region must be specified together" )); } - let s3_client = if let (Some(endpoint_url), Some(region), Some(force_path_style)) = + let s3_config = if let (Some(endpoint_url), Some(region), Some(force_path_style)) = (endpoint_url, region, force_path_style) { - let s3_config = S3Config { + S3Config::Custom { auth_storage: storage.clone(), endpoint_url, region, force_path_style, - }; - create_s3_client(Some(s3_config), Some(channel.clone())).await + } } else { - create_s3_client(None, None).await + S3Config::FromAWS }; + let s3_client = create_s3_client(s3_config, Some(channel.clone())).await; + let bucket = channel .host_str() .ok_or_else(|| miette::miette!("Failed to get host from channel URL"))?; diff --git a/test-data/recipes/prefix_detection/create_files_with_prefix.py b/test-data/recipes/prefix_detection/create_files_with_prefix.py index 49521548f..c72789af3 100644 --- a/test-data/recipes/prefix_detection/create_files_with_prefix.py +++ b/test-data/recipes/prefix_detection/create_files_with_prefix.py @@ -44,4 +44,6 @@ (force_text_folder / "file_without_prefix").write_bytes(binary_data) if platform.platform().startswith("Windows"): - (is_text_folder / "file_with_forwardslash_prefix").write_text(text_data_with_prefix.replace("\\", "/")) + (is_text_folder / "file_with_forwardslash_prefix").write_text( + text_data_with_prefix.replace("\\", "/") + ) diff --git a/test/end-to-end/test_simple.py b/test/end-to-end/test_simple.py index 10eb9df5a..0ca29b75d 100644 --- a/test/end-to-end/test_simple.py +++ b/test/end-to-end/test_simple.py @@ -5,7 +5,6 @@ import subprocess from pathlib import Path from subprocess import DEVNULL, STDOUT, CalledProcessError, check_output -from time import sleep import pytest import requests From 031bb563e11b142133b6515fd1bb1b42b5369541 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 5 Jan 2025 23:34:24 +0100 Subject: [PATCH 03/11] Fix test --- src/upload/mod.rs | 13 +++++++++---- test/end-to-end/test_simple.py | 9 +++++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/upload/mod.rs b/src/upload/mod.rs index ea5456671..09da21289 100644 --- a/src/upload/mod.rs +++ b/src/upload/mod.rs @@ -12,7 +12,7 @@ use trusted_publishing::{check_trusted_publishing, TrustedPublishResult}; use crate::url_with_trailing_slash::UrlWithTrailingSlash; use miette::{Context, IntoDiagnostic}; -use rattler_networking::s3_middleware::{create_s3_client, S3Config}; +use rattler_networking::s3_middleware::{S3Config, S3}; use rattler_networking::{Authentication, AuthenticationStorage}; use rattler_redaction::Redact; use reqwest::Method; @@ -332,7 +332,6 @@ pub async fn upload_package_to_s3( (endpoint_url, region, force_path_style) { S3Config::Custom { - auth_storage: storage.clone(), endpoint_url, region, force_path_style, @@ -341,7 +340,11 @@ pub async fn upload_package_to_s3( S3Config::FromAWS }; - let s3_client = create_s3_client(s3_config, Some(channel.clone())).await; + let s3 = S3::new(s3_config, storage.clone()); + let s3_client = s3 + .create_s3_client(Some(channel.clone())) + .await + .map_err(|e| miette::miette!(e.to_string()))?; let bucket = channel .host_str() @@ -362,12 +365,14 @@ pub async fn upload_package_to_s3( .into_diagnostic()?; s3_client .put_object() - .key(key) + .key(key.clone()) + .if_none_match("*") // Do not allow overwriting existing files .body(body) .bucket(bucket) .send() .await .into_diagnostic()?; + tracing::info!("Uploaded package to s3://{bucket}/{key}"); } Ok(()) diff --git a/test/end-to-end/test_simple.py b/test/end-to-end/test_simple.py index 0ca29b75d..a8cdbb0d9 100644 --- a/test/end-to-end/test_simple.py +++ b/test/end-to-end/test_simple.py @@ -256,7 +256,7 @@ def test_s3_minio_upload( ): monkeypatch.setenv("RATTLER_AUTH_FILE", str(s3_credentials_file)) rattler_build.build(recipes / "globtest", tmp_path) - rattler_build( + cmd = [ "upload", "-vvv", "s3", @@ -269,7 +269,8 @@ def test_s3_minio_upload( "--force-path-style", "true", str(get_package(tmp_path, "globtest")), - ) + ] + rattler_build(*cmd) # Make sure the package was uploaded assert b"globtest" in subprocess.check_output( @@ -280,6 +281,10 @@ def test_s3_minio_upload( }, ) + # Raise an error if the same package is uploaded again + with pytest.raises(CalledProcessError): + rattler_build(*cmd) + @pytest.mark.skipif( os.name == "nt", reason="recipe does not support execution on windows" From 99ed57fd31786aa12649f603ec32371ed1ebc9a9 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 12 Jan 2025 21:11:02 +0100 Subject: [PATCH 04/11] Fix integration test with minio --- Cargo.lock | 140 ++++++++++++--------------------- Cargo.toml | 15 +++- pixi.lock | 56 ++++++------- pixi.toml | 4 +- test/end-to-end/test_simple.py | 46 ++++++++--- 5 files changed, 123 insertions(+), 138 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db4ee3d72..d6e8d8e3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -352,9 +352,9 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" dependencies = [ "proc-macro2", "quote", @@ -410,14 +410,12 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.12" +version = "1.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "649316840239f4e58df0b7f620c428f5fababbbca2d504488c641534050bd141" +checksum = "c03a50b30228d3af8865ce83376b4e99e1ffa34728220fe2860e4df0bb5278d6" dependencies = [ "aws-credential-types", "aws-runtime", - "aws-sdk-sso", - "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http", @@ -428,14 +426,11 @@ dependencies = [ "aws-types", "bytes", "fastrand", - "hex", "http 0.2.12", - "ring", "time", "tokio", "tracing", "url", - "zeroize", ] [[package]] @@ -452,9 +447,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44f6f1124d6e19ab6daf7f2e615644305dc6cb2d706892a8a8c0b98db35de020" +checksum = "b16d1aa50accc11a4b4d5c50f7fb81cc0cf60328259c587d0e6b0f11385bde46" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -478,9 +473,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.67.0" +version = "1.68.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbc644164269a1e38ce7f2f7373629d3fb3d310c0e3feb5573a29744288b24d3" +checksum = "bc5ddf1dc70287dc9a2f953766a1fe15e3e74aef02fd1335f2afa475c9b4f4fc" dependencies = [ "aws-credential-types", "aws-runtime", @@ -510,55 +505,11 @@ dependencies = [ "url", ] -[[package]] -name = "aws-sdk-sso" -version = "1.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb25f7129c74d36afe33405af4517524df8f74b635af8c2c8e91c1552b8397b2" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-ssooidc" -version = "1.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03a3d5ef14851625eafd89660a751776f938bf32f309308b20dcca41c44b568" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - [[package]] name = "aws-sdk-sts" -version = "1.53.0" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3a9f073ae3a53b54421503063dfb87ff1ea83b876f567d92e8b8d9942ba91b" +checksum = "861d324ef69247c6f3c6823755f408a68877ffb1a9afaff6dd8b0057c760de60" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1802,9 +1753,9 @@ dependencies = [ [[package]] name = "file_url" -version = "0.2.1" +version = "0.2.2" dependencies = [ - "itertools 0.13.0", + "itertools 0.14.0", "percent-encoding", "thiserror 2.0.9", "typed-path", @@ -2896,6 +2847,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.14" @@ -4122,7 +4082,7 @@ dependencies = [ [[package]] name = "rattler" -version = "0.28.9" +version = "0.28.12" dependencies = [ "anyhow", "clap", @@ -4134,14 +4094,14 @@ dependencies = [ "humantime", "indexmap 2.7.0", "indicatif", - "itertools 0.13.0", + "itertools 0.14.0", "memchr", "memmap2 0.9.5", "once_cell", "parking_lot 0.12.3", "rattler_cache", "rattler_conda_types", - "rattler_digest 1.0.4", + "rattler_digest 1.0.5", "rattler_networking", "rattler_package_streaming", "rattler_shell", @@ -4208,7 +4168,7 @@ dependencies = [ "rattler", "rattler_cache", "rattler_conda_types", - "rattler_digest 1.0.4", + "rattler_digest 1.0.5", "rattler_index", "rattler_installs_packages", "rattler_networking", @@ -4258,7 +4218,7 @@ dependencies = [ [[package]] name = "rattler_cache" -version = "0.3.1" +version = "0.3.4" dependencies = [ "anyhow", "dashmap", @@ -4268,10 +4228,10 @@ dependencies = [ "fs4 0.12.0", "futures", "fxhash", - "itertools 0.13.0", + "itertools 0.14.0", "parking_lot 0.12.3", "rattler_conda_types", - "rattler_digest 1.0.4", + "rattler_digest 1.0.5", "rattler_networking", "rattler_package_streaming", "rayon", @@ -4286,7 +4246,7 @@ dependencies = [ [[package]] name = "rattler_conda_types" -version = "0.29.7" +version = "0.29.10" dependencies = [ "chrono", "dirs", @@ -4296,11 +4256,11 @@ dependencies = [ "glob", "hex", "indexmap 2.7.0", - "itertools 0.13.0", + "itertools 0.14.0", "lazy-regex", "nom", "purl", - "rattler_digest 1.0.4", + "rattler_digest 1.0.5", "rattler_macros", "rattler_redaction", "rayon", @@ -4338,7 +4298,7 @@ dependencies = [ [[package]] name = "rattler_digest" -version = "1.0.4" +version = "1.0.5" dependencies = [ "blake2", "digest", @@ -4353,11 +4313,11 @@ dependencies = [ [[package]] name = "rattler_index" -version = "0.20.4" +version = "0.20.7" dependencies = [ "fs-err 3.0.0", "rattler_conda_types", - "rattler_digest 1.0.4", + "rattler_digest 1.0.5", "rattler_package_streaming", "serde_json", "tracing", @@ -4427,7 +4387,7 @@ dependencies = [ [[package]] name = "rattler_macros" -version = "1.0.4" +version = "1.0.5" dependencies = [ "quote", "syn", @@ -4435,12 +4395,11 @@ dependencies = [ [[package]] name = "rattler_networking" -version = "0.21.9" +version = "0.21.10" dependencies = [ "anyhow", "async-trait", "aws-config", - "aws-runtime", "aws-sdk-s3", "base64 0.22.1", "chrono", @@ -4449,7 +4408,7 @@ dependencies = [ "getrandom", "google-cloud-auth", "http 1.2.0", - "itertools 0.13.0", + "itertools 0.14.0", "keyring", "netrc-rs", "reqwest", @@ -4464,7 +4423,7 @@ dependencies = [ [[package]] name = "rattler_package_streaming" -version = "0.22.20" +version = "0.22.23" dependencies = [ "bzip2 0.5.0", "chrono", @@ -4472,7 +4431,7 @@ dependencies = [ "futures-util", "num_cpus", "rattler_conda_types", - "rattler_digest 1.0.4", + "rattler_digest 1.0.5", "rattler_networking", "rattler_redaction", "reqwest", @@ -4491,7 +4450,7 @@ dependencies = [ [[package]] name = "rattler_redaction" -version = "0.1.5" +version = "0.1.6" dependencies = [ "reqwest", "reqwest-middleware", @@ -4500,7 +4459,7 @@ dependencies = [ [[package]] name = "rattler_repodata_gateway" -version = "0.21.29" +version = "0.21.32" dependencies = [ "anyhow", "async-compression", @@ -4520,7 +4479,7 @@ dependencies = [ "http-cache-semantics", "humansize", "humantime", - "itertools 0.13.0", + "itertools 0.14.0", "json-patch", "libc", "md-5", @@ -4530,11 +4489,12 @@ dependencies = [ "pin-project-lite", "rattler_cache", "rattler_conda_types", - "rattler_digest 1.0.4", + "rattler_digest 1.0.5", "rattler_networking", "rattler_redaction", "reqwest", "reqwest-middleware", + "retry-policies", "rmp-serde", "serde", "serde_json", @@ -4553,12 +4513,12 @@ dependencies = [ [[package]] name = "rattler_shell" -version = "0.22.12" +version = "0.22.15" dependencies = [ "enum_dispatch", "fs-err 3.0.0", "indexmap 2.7.0", - "itertools 0.13.0", + "itertools 0.14.0", "rattler_conda_types", "serde_json", "shlex", @@ -4570,13 +4530,13 @@ dependencies = [ [[package]] name = "rattler_solve" -version = "1.3.1" +version = "1.3.4" dependencies = [ "chrono", "futures", - "itertools 0.13.0", + "itertools 0.14.0", "rattler_conda_types", - "rattler_digest 1.0.4", + "rattler_digest 1.0.5", "resolvo 0.8.5", "serde", "tempfile", @@ -4587,7 +4547,7 @@ dependencies = [ [[package]] name = "rattler_virtual_packages" -version = "1.1.15" +version = "1.2.0" dependencies = [ "archspec", "libloading", diff --git a/Cargo.toml b/Cargo.toml index 3d4d19dc8..4dc9d3737 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -164,9 +164,18 @@ rattler_virtual_packages = { version = "1.1.15", default-features = false } rattler_package_streaming = { version = "0.22.20", default-features = false } lazy_static = "1.5.0" reqwest-retry = "0.7.0" -aws-smithy-types = { version = "1.2.11", features = ["rt-tokio"] } -aws-config = "1.5.12" -aws-sdk-s3 = "1.65.0" +aws-smithy-types = { version = "1.2.11", default-features = false, features = [ + "rt-tokio", +] } +aws-config = { version = "1.5.13", default-features = false, features = [ + "rt-tokio", + "rustls", +] } +aws-sdk-s3 = { version = "1.68.0", default-features = false, features = [ + "rt-tokio", + "rustls", + "sigv4a", +] } [target.'cfg(not(target_os = "windows"))'.dependencies] sha2 = { version = "0.10.8", features = ["asm"] } diff --git a/pixi.lock b/pixi.lock index 04a1a23f4..5bed83e9a 100644 --- a/pixi.lock +++ b/pixi.lock @@ -83,7 +83,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/make-4.3-hd18ef5c_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/mimalloc-2.1.7-hac33072_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/minio-client-2024.11.21.17.21.54-h199d65c_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/minio-server-2024.03.10.02.53.48-hbcca054_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/minio-server-2024.12.18.13.15.44-hdd60dd4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/mold-2.35.1-hc93959d_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-he02047a_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.9.1-pyhd8ed1ab_1.conda @@ -192,7 +192,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/llvm-tools-15.0.7-hbedff68_4.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/make-4.3-h22f3db7_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/osx-64/minio-client-2024.11.21.17.21.54-hd81af70_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-64/minio-server-2024.03.10.02.53.48-h8857fd0_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/minio-server-2024.12.18.13.15.44-h063bf7d_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/mpc-1.3.1-h9d8efa1_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/mpfr-4.2.1-haed47dc_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-hf036a51_1.conda @@ -306,7 +306,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/llvm-tools-15.0.7-h2621b3d_4.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/make-4.3-he57ea6c_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/minio-client-2024.11.21.17.21.54-h7b83518_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/minio-server-2024.03.10.02.53.48-hf0a4a13_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/minio-server-2024.12.18.13.15.44-h2e8662c_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/mpc-1.3.1-h8f1351a_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/mpfr-4.2.1-hb693164_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h7bae524_1.conda @@ -398,7 +398,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/m2w64-libwinpthread-git-5.0.0.4634.697f757-2.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/win-64/make-4.3-h3d2af85_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/win-64/minio-client-2024.11.21.17.21.54-h265662a_0.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/minio-server-2024.03.10.02.53.48-h56e8100_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/minio-server-2024.12.18.13.15.44-h8750aaa_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/msys2-conda-epoch-20160418-1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.9.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/openmp-5.0.0-vc14_1.tar.bz2 @@ -4100,8 +4100,6 @@ packages: depends: - libgcc >=13 - __glibc >=2.17,<3.0.a0 - arch: x86_64 - platform: linux license: AGPL-3.0-or-later license_family: AGPL size: 18807854 @@ -4113,8 +4111,6 @@ packages: - __osx >=10.13 constrains: - __osx >=10.12 - arch: x86_64 - platform: osx license: AGPL-3.0-or-later license_family: AGPL size: 19134629 @@ -4124,8 +4120,6 @@ packages: md5: f89f48621c6cd54c3d8bc06f830e1728 depends: - __osx >=11.0 - arch: arm64 - platform: osx license: AGPL-3.0-or-later license_family: AGPL size: 17883741 @@ -4140,50 +4134,48 @@ packages: - vc >=14.2,<15 - vc14_runtime >=14.29.30139 - ucrt >=10.0.20348.0 - arch: x86_64 - platform: win license: AGPL-3.0-or-later license_family: AGPL size: 19174140 timestamp: 1735897885467 -- conda: https://conda.anaconda.org/conda-forge/linux-64/minio-server-2024.03.10.02.53.48-hbcca054_0.conda - sha256: de0b4886ca7b1a27008f3b7d33500550191cd571765a5adb1d7315d40efd4a9e - md5: 3c3572872ca7690eadfaf532a57bfe1b +- conda: https://conda.anaconda.org/conda-forge/linux-64/minio-server-2024.12.18.13.15.44-hdd60dd4_0.conda + sha256: ea636a4bd674e2f0751a6e89a0618eebe83363924596d0e43c1a802634c0613e + md5: c91c07122dc34a7fafc3b09a49e91ba6 arch: x86_64 platform: linux license: AGPL-3.0-only license_family: AGPL - size: 29338301 - timestamp: 1710216649717 -- conda: https://conda.anaconda.org/conda-forge/osx-64/minio-server-2024.03.10.02.53.48-h8857fd0_0.conda - sha256: 1b120375ab7629331212a36f22f842288e266b06ab49a961825474dd031e3981 - md5: e4c421d22e108008340181bbc7b70582 + size: 32515011 + timestamp: 1736260571294 +- conda: https://conda.anaconda.org/conda-forge/osx-64/minio-server-2024.12.18.13.15.44-h063bf7d_0.conda + sha256: 4d0cf308bf8c145e34943934cdc78c2d40f5fbdc6a1f638202ac7c1ba72dcb28 + md5: a43e62694d1e92af1fc0ca14ce40490e constrains: - __osx>=10.12 arch: x86_64 platform: osx license: AGPL-3.0-only license_family: AGPL - size: 29684080 - timestamp: 1710217094567 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/minio-server-2024.03.10.02.53.48-hf0a4a13_0.conda - sha256: 7c29eb913e1d5e8efcb5af9dd908c1e4ef13b3e87cdc486a9ade6c7a8106a2ef - md5: a69c70cb150ba41a60e81b03fc94a954 + size: 32961085 + timestamp: 1736265258316 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/minio-server-2024.12.18.13.15.44-h2e8662c_0.conda + sha256: 64551d4130c6810cffd1c9627dcfc718c4ffd69a809761fea7a3f8286a5c39e0 + md5: d02298271c89b6d058c92de83e355e24 arch: arm64 platform: osx license: AGPL-3.0-only license_family: AGPL - size: 28447785 - timestamp: 1710216865211 -- conda: https://conda.anaconda.org/conda-forge/win-64/minio-server-2024.03.10.02.53.48-h56e8100_0.conda - sha256: 575c723ba0ae2d10c1b340afd229c41efd80baeee4c6d75d7f249b449ad66348 - md5: 6126ff5a23af33c9fa987f11783b62e5 + size: 31547706 + timestamp: 1736260866414 +- conda: https://conda.anaconda.org/conda-forge/win-64/minio-server-2024.12.18.13.15.44-h8750aaa_0.conda + sha256: af7318c0a4352d70eec5317d8ffe5e995956ee25c6fb40192420f6cb5c8eafc6 + md5: 5bd6b7cc8035c4271ea9f53725404938 arch: x86_64 platform: win license: AGPL-3.0-only license_family: AGPL - size: 29699962 - timestamp: 1710217343984 + size: 33036015 + timestamp: 1736260979638 - conda: https://conda.anaconda.org/conda-forge/noarch/mkdocs-1.5.3-pyhd8ed1ab_0.conda sha256: cd57f3805a38bc3b3356a37d4bd9af7e227972286f61eae3c88d356216bc7159 md5: 1e432aebaa009b030cce33a554939d8e diff --git a/pixi.toml b/pixi.toml index 474f9af04..64ce20c3c 100644 --- a/pixi.toml +++ b/pixi.toml @@ -34,8 +34,8 @@ cmake = "3.27.6.*" make = "4.3.*" perl = "5.32.1.*" pytest = "7.4.2.*" -minio-server = "*" -minio-client = "*" +minio-server = ">=2024.12.18" +minio-client = ">=2024.11.21" pyyaml = ">=6.0.1,<6.1" conda-package-handling = "2.2.0.*" requests = ">=2.32.2,<2.33" diff --git a/test/end-to-end/test_simple.py b/test/end-to-end/test_simple.py index a8cdbb0d9..78de5e786 100644 --- a/test/end-to-end/test_simple.py +++ b/test/end-to-end/test_simple.py @@ -5,6 +5,7 @@ import subprocess from pathlib import Path from subprocess import DEVNULL, STDOUT, CalledProcessError, check_output +from time import sleep import pytest import requests @@ -214,17 +215,35 @@ def minio_server(tmp_path: Path): tmp_dir = tmp_path / "minio" # Start minio server proc = subprocess.Popen( - ["minio", "server", "--address", "localhost:9000", str(tmp_dir)], - ) - # Create a bucket - subprocess.run( - ["mc", "mb", "local/s3-forge"], - env={ - **os.environ, - "MC_HOST_local": "http://minioadmin:minioadmin@localhost:9000", - }, - check=True, + [ + f"minio{'.bat' if os.name == 'nt' else ''}", + "server", + "--address", + "0.0.0.0:9000", + str(tmp_dir), + ], ) + # Create a bucket (with retries, in case we have to wait until minio is up and running) + retries = 0 + max_retries = 3 + while retries < max_retries: + try: + subprocess.run( + [f"mc{'.bat' if os.name == 'nt' else ''}", "mb", "local/s3-forge"], + env={ + **os.environ, + "MC_HOST_local": "http://minioadmin:minioadmin@localhost:9000", + }, + check=True, + ) + break + except CalledProcessError: + retries += 1 + sleep(1) + if retries == max_retries: + raise RuntimeError( + "Failed to create bucket, could not reach minio server" + ) yield proc.kill() @@ -274,7 +293,12 @@ def test_s3_minio_upload( # Make sure the package was uploaded assert b"globtest" in subprocess.check_output( - ["mc", "ls", "--recursive", "local/s3-forge/channel"], + [ + f"mc{'.bat' if os.name == 'nt' else ''}", + "ls", + "--recursive", + "local/s3-forge/channel", + ], env={ **os.environ, "MC_HOST_local": "http://minioadmin:minioadmin@localhost:9000", From ebc79494a4ba39313f0eaac045e855e479a8612d Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 12 Jan 2025 21:29:37 +0100 Subject: [PATCH 05/11] Change location for remote rattler --- Cargo.lock | 15 +++++++++++++++ Cargo.toml | 24 ++++++++++++------------ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eff82dfb0..2c9d0640e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1620,6 +1620,7 @@ dependencies = [ [[package]] name = "file_url" version = "0.2.2" +source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" dependencies = [ "itertools 0.14.0", "percent-encoding", @@ -3745,6 +3746,7 @@ dependencies = [ [[package]] name = "rattler" version = "0.28.12" +source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" dependencies = [ "anyhow", "clap", @@ -3881,6 +3883,7 @@ dependencies = [ [[package]] name = "rattler_cache" version = "0.3.4" +source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" dependencies = [ "anyhow", "dashmap", @@ -3909,6 +3912,7 @@ dependencies = [ [[package]] name = "rattler_conda_types" version = "0.29.10" +source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" dependencies = [ "chrono", "dirs", @@ -3945,6 +3949,7 @@ dependencies = [ [[package]] name = "rattler_digest" version = "1.0.5" +source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" dependencies = [ "blake2", "digest", @@ -3960,6 +3965,7 @@ dependencies = [ [[package]] name = "rattler_index" version = "0.20.7" +source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" dependencies = [ "fs-err", "rattler_conda_types", @@ -3973,6 +3979,7 @@ dependencies = [ [[package]] name = "rattler_macros" version = "1.0.5" +source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" dependencies = [ "quote", "syn", @@ -3981,6 +3988,7 @@ dependencies = [ [[package]] name = "rattler_networking" version = "0.21.10" +source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" dependencies = [ "anyhow", "async-trait", @@ -4009,6 +4017,7 @@ dependencies = [ [[package]] name = "rattler_package_streaming" version = "0.22.23" +source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" dependencies = [ "bzip2 0.5.0", "chrono", @@ -4036,6 +4045,7 @@ dependencies = [ [[package]] name = "rattler_redaction" version = "0.1.6" +source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" dependencies = [ "reqwest", "reqwest-middleware", @@ -4045,6 +4055,7 @@ dependencies = [ [[package]] name = "rattler_repodata_gateway" version = "0.21.32" +source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" dependencies = [ "anyhow", "async-compression", @@ -4111,6 +4122,7 @@ dependencies = [ [[package]] name = "rattler_shell" version = "0.22.15" +source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" dependencies = [ "enum_dispatch", "fs-err", @@ -4128,6 +4140,7 @@ dependencies = [ [[package]] name = "rattler_solve" version = "1.3.4" +source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" dependencies = [ "chrono", "futures", @@ -4145,6 +4158,7 @@ dependencies = [ [[package]] name = "rattler_virtual_packages" version = "1.2.0" +source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" dependencies = [ "archspec", "libloading", @@ -5045,6 +5059,7 @@ dependencies = [ [[package]] name = "simple_spawn_blocking" version = "1.0.0" +source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" dependencies = [ "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index d17ee0987..0aa6dc704 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -206,15 +206,15 @@ pre-build = [ # this fork contains fixes for slow zip reading and large file writing zip = { git = "https://github.com/wolfv/zip2", branch = "patched" } -rattler = { path = "../pavelzw-rattler/crates/rattler" } -rattler_cache = { path = "../pavelzw-rattler/crates/rattler_cache" } -rattler_conda_types = { path = "../pavelzw-rattler/crates/rattler_conda_types" } -rattler_digest = { path = "../pavelzw-rattler/crates/rattler_digest" } -rattler_index = { path = "../pavelzw-rattler/crates/rattler_index" } -rattler_networking = { path = "../pavelzw-rattler/crates/rattler_networking" } -rattler_repodata_gateway = { path = "../pavelzw-rattler/crates/rattler_repodata_gateway" } -rattler_shell = { path = "../pavelzw-rattler/crates/rattler_shell" } -rattler_solve = { path = "../pavelzw-rattler/crates/rattler_solve" } -rattler_redaction = { path = "../pavelzw-rattler/crates/rattler_redaction" } -rattler_virtual_packages = { path = "../pavelzw-rattler/crates/rattler_virtual_packages" } -rattler_package_streaming = { path = "../pavelzw-rattler/crates/rattler_package_streaming" } +rattler = { git = "https://github.com/pavelzw/rattler", branch = "s3" } +rattler_cache = { git = "https://github.com/pavelzw/rattler", branch = "s3" } +rattler_conda_types = { git = "https://github.com/pavelzw/rattler", branch = "s3" } +rattler_digest = { git = "https://github.com/pavelzw/rattler", branch = "s3" } +rattler_index = { git = "https://github.com/pavelzw/rattler", branch = "s3" } +rattler_networking = { git = "https://github.com/pavelzw/rattler", branch = "s3" } +rattler_repodata_gateway = { git = "https://github.com/pavelzw/rattler", branch = "s3" } +rattler_shell = { git = "https://github.com/pavelzw/rattler", branch = "s3" } +rattler_solve = { git = "https://github.com/pavelzw/rattler", branch = "s3" } +rattler_redaction = { git = "https://github.com/pavelzw/rattler", branch = "s3" } +rattler_virtual_packages = { git = "https://github.com/pavelzw/rattler", branch = "s3" } +rattler_package_streaming = { git = "https://github.com/pavelzw/rattler", branch = "s3" } From b6269b8cbb7b64efb827d3cd49ab8492d8aa7678 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Fri, 17 Jan 2025 17:53:32 +0100 Subject: [PATCH 06/11] refactor: Update authentication storage to use new rattler scheme --- Cargo.lock | 293 +++++++++++++++++++++++++------------- src/tool_configuration.rs | 10 +- 2 files changed, 197 insertions(+), 106 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c9d0640e..96b9822e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -131,11 +131,12 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "3.0.6" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", + "once_cell", "windows-sys 0.59.0", ] @@ -374,12 +375,14 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.13" +version = "1.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03a50b30228d3af8865ce83376b4e99e1ffa34728220fe2860e4df0bb5278d6" +checksum = "9f40e82e858e02445402906e454a73e244c7f501fcae198977585946c48e8697" dependencies = [ "aws-credential-types", "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http", @@ -390,11 +393,14 @@ dependencies = [ "aws-types", "bytes", "fastrand", + "hex", "http 0.2.12", + "ring", "time", "tokio", "tracing", "url", + "zeroize", ] [[package]] @@ -411,9 +417,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.3" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16d1aa50accc11a4b4d5c50f7fb81cc0cf60328259c587d0e6b0f11385bde46" +checksum = "bee7643696e7fdd74c10f9eb42848a87fe469d35eae9c3323f80aa98f350baac" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -437,9 +443,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.68.0" +version = "1.69.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5ddf1dc70287dc9a2f953766a1fe15e3e74aef02fd1335f2afa475c9b4f4fc" +checksum = "0a88f1c30e4ffa2464f910297c24736ff68cca9e8d2b7d52596b54efd99b9c1e" dependencies = [ "aws-credential-types", "aws-runtime", @@ -469,11 +475,55 @@ dependencies = [ "url", ] +[[package]] +name = "aws-sdk-sso" +version = "1.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921a13ed6aabe2d1258f65ef7804946255c799224440774c30e1a2c65cdf983a" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "196c952738b05dfc917d82a3e9b5ba850822a6d6a86d677afda2a156cc172ceb" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-sts" -version = "1.54.1" +version = "1.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "861d324ef69247c6f3c6823755f408a68877ffb1a9afaff6dd8b0057c760de60" +checksum = "33ef5b73a927ed80b44096f8c20fb4abae65469af15198367e179ae267256e9d" dependencies = [ "aws-credential-types", "aws-runtime", @@ -494,9 +544,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.6" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" +checksum = "690118821e46967b3c4501d67d7d52dd75106a9c54cf36cefa1985cedbe94e05" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -523,9 +573,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427cb637d15d63d6f9aae26358e1c9a9c09d5aa490d64b09354c8217cfef0f28" +checksum = "fa59d1327d8b5053c54bf2eaae63bf629ba9e904434d0835a28ed3c0ed0a614e" dependencies = [ "futures-util", "pin-project-lite", @@ -534,15 +584,16 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.13" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" +checksum = "f2f45a1c384d7a393026bc5f5c177105aa9fa68e4749653b985707ac27d77295" dependencies = [ "aws-smithy-http", "aws-smithy-types", "bytes", "crc32c", "crc32fast", + "crc64fast-nvme", "hex", "http 0.2.12", "http-body 0.4.6", @@ -555,9 +606,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.5" +version = "0.60.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" +checksum = "8b18559a41e0c909b77625adf2b8c50de480a8041e5e4a3f5f7d177db70abc5a" dependencies = [ "aws-smithy-types", "bytes", @@ -566,9 +617,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.11" +version = "0.60.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -587,9 +638,9 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.61.1" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" +checksum = "623a51127f24c30776c8b374295f2df78d92517386f77ba30773f15a30ce1422" dependencies = [ "aws-smithy-types", ] @@ -606,9 +657,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.6" +version = "1.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a05dd41a70fc74051758ee75b5c4db2c0ca070ed9229c3df50e9475cda1cb985" +checksum = "865f7050bbc7107a6c98a397a9fcd9413690c27fa718446967cf03b2d3ac517e" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -650,9 +701,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.11" +version = "1.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ddc9bd6c28aeb303477170ddd183760a956a03e083b3902a990238a7e3792d" +checksum = "a28f6feb647fb5e0d5b50f0472c19a7db9462b74e2fec01bb0b44eedcc834e97" dependencies = [ "base64-simd", "bytes", @@ -685,9 +736,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.3" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +checksum = "b0df5a18c4f951c645300d365fec53a61418bcf4650f604f85fe2a665bfaa0c2" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -761,7 +812,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "848df95320021558dd6bb4c26de3fe66724cdcbdbbf3fa720150b52b086ae568" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "libc", "log", "rustix", @@ -776,9 +827,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1be3f42a67d6d345ecd59f675f3f012d6974981560836e938c22b424b85ce1be" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "bitvec" @@ -937,11 +988,30 @@ dependencies = [ "cipher", ] +[[package]] +name = "cbindgen" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fce8dd7fcfcbf3a0a87d8f515194b49d6135acab73e18bd380d1d93bb1a15eb" +dependencies = [ + "clap", + "heck 0.4.1", + "indexmap 2.7.0", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", + "tempfile", + "toml", +] + [[package]] name = "cc" -version = "1.2.9" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8293772165d9345bdaaa39b45b2109591e63fe5e6fbc23c6ff930a048aa310b" +checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" dependencies = [ "jobserver", "libc", @@ -1158,6 +1228,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32c" version = "0.6.8" @@ -1176,6 +1261,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5e2ee08013e3f228d6d2394116c4549a6df77708442c62d887d83f68ef2ee37" +dependencies = [ + "cbindgen", + "crc", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -1207,7 +1302,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "crossterm_winapi", "futures-core", "mio", @@ -1529,9 +1624,9 @@ dependencies = [ [[package]] name = "enumflags2" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d232db7f5956f3f14313dc2f87985c58bd2c695ce124c8cdd984e08e15ac133d" +checksum = "ba2f4b465f5318854c6f8dd686ede6c0a9dc67d4b1ac241cf0eb51521a309147" dependencies = [ "enumflags2_derive", "serde", @@ -1539,9 +1634,9 @@ dependencies = [ [[package]] name = "enumflags2_derive" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" +checksum = "fc4caf64a58d7a6d65ab00639b046ff54399a39f5f2554728895ace4b297cd79" dependencies = [ "proc-macro2", "quote", @@ -1620,7 +1715,7 @@ dependencies = [ [[package]] name = "file_url" version = "0.2.2" -source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" +source = "git+https://github.com/pavelzw/rattler?branch=s3#73eed8a7d50de3e5e766d555052bfe2ba1225049" dependencies = [ "itertools 0.14.0", "percent-encoding", @@ -1724,16 +1819,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "fslock" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04412b8935272e3a9bae6f48c7bfff74c2911f60525404edfdd28e49884c3bfb" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "funty" version = "2.0.0" @@ -2670,9 +2755,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ "once_cell", "wasm-bindgen", @@ -2797,7 +2882,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "libc", "redox_syscall 0.5.8", ] @@ -2838,9 +2923,9 @@ checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" [[package]] name = "log" -version = "0.4.22" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] name = "lru" @@ -2981,9 +3066,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" dependencies = [ "adler2", ] @@ -3029,7 +3114,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "cfg-if", "cfg_aliases", "libc", @@ -3192,7 +3277,7 @@ version = "0.10.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "cfg-if", "foreign-types", "libc", @@ -3282,9 +3367,9 @@ dependencies = [ [[package]] name = "outref" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "overload" @@ -3728,7 +3813,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "cassowary", "compact_str", "crossterm", @@ -3746,7 +3831,7 @@ dependencies = [ [[package]] name = "rattler" version = "0.28.12" -source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" +source = "git+https://github.com/pavelzw/rattler?branch=s3#73eed8a7d50de3e5e766d555052bfe2ba1225049" dependencies = [ "anyhow", "clap", @@ -3883,7 +3968,7 @@ dependencies = [ [[package]] name = "rattler_cache" version = "0.3.4" -source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" +source = "git+https://github.com/pavelzw/rattler?branch=s3#73eed8a7d50de3e5e766d555052bfe2ba1225049" dependencies = [ "anyhow", "dashmap", @@ -3912,7 +3997,7 @@ dependencies = [ [[package]] name = "rattler_conda_types" version = "0.29.10" -source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" +source = "git+https://github.com/pavelzw/rattler?branch=s3#73eed8a7d50de3e5e766d555052bfe2ba1225049" dependencies = [ "chrono", "dirs", @@ -3949,7 +4034,7 @@ dependencies = [ [[package]] name = "rattler_digest" version = "1.0.5" -source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" +source = "git+https://github.com/pavelzw/rattler?branch=s3#73eed8a7d50de3e5e766d555052bfe2ba1225049" dependencies = [ "blake2", "digest", @@ -3965,7 +4050,7 @@ dependencies = [ [[package]] name = "rattler_index" version = "0.20.7" -source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" +source = "git+https://github.com/pavelzw/rattler?branch=s3#73eed8a7d50de3e5e766d555052bfe2ba1225049" dependencies = [ "fs-err", "rattler_conda_types", @@ -3979,7 +4064,7 @@ dependencies = [ [[package]] name = "rattler_macros" version = "1.0.5" -source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" +source = "git+https://github.com/pavelzw/rattler?branch=s3#73eed8a7d50de3e5e766d555052bfe2ba1225049" dependencies = [ "quote", "syn", @@ -3988,16 +4073,16 @@ dependencies = [ [[package]] name = "rattler_networking" version = "0.21.10" -source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" +source = "git+https://github.com/pavelzw/rattler?branch=s3#73eed8a7d50de3e5e766d555052bfe2ba1225049" dependencies = [ "anyhow", + "async-fd-lock", "async-trait", "aws-config", "aws-sdk-s3", "base64 0.22.1", "chrono", "dirs", - "fslock", "getrandom", "google-cloud-auth", "http 1.2.0", @@ -4017,7 +4102,7 @@ dependencies = [ [[package]] name = "rattler_package_streaming" version = "0.22.23" -source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" +source = "git+https://github.com/pavelzw/rattler?branch=s3#73eed8a7d50de3e5e766d555052bfe2ba1225049" dependencies = [ "bzip2 0.5.0", "chrono", @@ -4045,7 +4130,7 @@ dependencies = [ [[package]] name = "rattler_redaction" version = "0.1.6" -source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" +source = "git+https://github.com/pavelzw/rattler?branch=s3#73eed8a7d50de3e5e766d555052bfe2ba1225049" dependencies = [ "reqwest", "reqwest-middleware", @@ -4055,7 +4140,7 @@ dependencies = [ [[package]] name = "rattler_repodata_gateway" version = "0.21.32" -source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" +source = "git+https://github.com/pavelzw/rattler?branch=s3#73eed8a7d50de3e5e766d555052bfe2ba1225049" dependencies = [ "anyhow", "async-compression", @@ -4122,7 +4207,7 @@ dependencies = [ [[package]] name = "rattler_shell" version = "0.22.15" -source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" +source = "git+https://github.com/pavelzw/rattler?branch=s3#73eed8a7d50de3e5e766d555052bfe2ba1225049" dependencies = [ "enum_dispatch", "fs-err", @@ -4140,7 +4225,7 @@ dependencies = [ [[package]] name = "rattler_solve" version = "1.3.4" -source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" +source = "git+https://github.com/pavelzw/rattler?branch=s3#73eed8a7d50de3e5e766d555052bfe2ba1225049" dependencies = [ "chrono", "futures", @@ -4158,7 +4243,7 @@ dependencies = [ [[package]] name = "rattler_virtual_packages" version = "1.2.0" -source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" +source = "git+https://github.com/pavelzw/rattler?branch=s3#73eed8a7d50de3e5e766d555052bfe2ba1225049" dependencies = [ "archspec", "libloading", @@ -4207,7 +4292,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", ] [[package]] @@ -4540,7 +4625,7 @@ version = "0.38.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "errno", "libc", "linux-raw-sys", @@ -4759,7 +4844,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -4772,7 +4857,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "core-foundation 0.10.0", "core-foundation-sys", "libc", @@ -5046,20 +5131,20 @@ checksum = "1de1d4f81173b03af4c0cbed3c898f6bff5b870e4a7f5d6f4057d62a7a4b686e" [[package]] name = "simple_asn1" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" +checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", - "thiserror 1.0.69", + "thiserror 2.0.11", "time", ] [[package]] name = "simple_spawn_blocking" version = "1.0.0" -source = "git+https://github.com/pavelzw/rattler?branch=s3#8434a39d06e6cfcadfd163889d66a3c618d47471" +source = "git+https://github.com/pavelzw/rattler?branch=s3#73eed8a7d50de3e5e766d555052bfe2ba1225049" dependencies = [ "tokio", ] @@ -5244,7 +5329,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "byteorder", "enum-as-inner", "libc", @@ -5815,9 +5900,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b913a3b5fe84142e269d63cc62b64319ccaf89b748fc31fe025177f767a756c4" +checksum = "744018581f9a3454a9e15beb8a33b017183f1e7c0cd170232a2d1453b23a51c4" dependencies = [ "getrandom", "rand", @@ -5825,9 +5910,9 @@ dependencies = [ [[package]] name = "valuable" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "value-trait" @@ -5919,20 +6004,21 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", @@ -5944,9 +6030,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.49" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", @@ -5957,9 +6043,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5967,9 +6053,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", @@ -5980,9 +6066,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "wasm-streams" @@ -6014,9 +6103,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/src/tool_configuration.rs b/src/tool_configuration.rs index 93a5223d6..67973bf3a 100644 --- a/src/tool_configuration.rs +++ b/src/tool_configuration.rs @@ -97,13 +97,13 @@ pub fn get_auth_store( ) -> Result { match auth_file { Some(auth_file) => { - let mut store = AuthenticationStorage::new(); + let mut store = AuthenticationStorage::empty(); store.add_backend(Arc::from( - authentication_storage::backends::file::FileStorage::new(auth_file)?, + authentication_storage::backends::file::FileStorage::from_path(auth_file)?, )); Ok(store) } - None => Ok(rattler_networking::AuthenticationStorage::default()), + None => Ok(rattler_networking::AuthenticationStorage::from_env_and_defaults()?), } } @@ -126,7 +126,9 @@ pub fn reqwest_client_from_auth_storage( .with(RetryTransientMiddleware::new_with_policy( ExponentialBackoff::builder().build_with_max_retries(3), )) - .with_arc(Arc::new(AuthenticationMiddleware::new(auth_storage))) + .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage( + auth_storage, + ))) .build()) } From b8e39cc8ce62bd23bd661ed70baef57eb7a76a17 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Fri, 17 Jan 2025 18:15:32 +0100 Subject: [PATCH 07/11] Add todo --- src/upload/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/upload/mod.rs b/src/upload/mod.rs index 78d91b4be..2754cdaf3 100644 --- a/src/upload/mod.rs +++ b/src/upload/mod.rs @@ -342,6 +342,8 @@ pub async fn upload_package_to_s3( "Both endpoint_url and region must be specified together" )); } + // TODO: Add CLI args for SECRET_ACCESS_KEY, ACCESS_KEY_ID, and SESSION_TOKEN + // and insert custom auth storage backend at position zero let s3_config = if let (Some(endpoint_url), Some(region), Some(force_path_style)) = (endpoint_url, region, force_path_style) { From 4caffc85f9a5847b17a2889ab6eb730775378834 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Wed, 22 Jan 2025 07:34:45 +0100 Subject: [PATCH 08/11] Update test/end-to-end/test_simple.py --- test/end-to-end/test_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/end-to-end/test_simple.py b/test/end-to-end/test_simple.py index 18f37f1f4..bbe30e22f 100644 --- a/test/end-to-end/test_simple.py +++ b/test/end-to-end/test_simple.py @@ -216,7 +216,7 @@ def minio_server(tmp_path: Path): # Start minio server proc = subprocess.Popen( [ - f"minio{'.bat' if os.name == 'nt' else ''}", + "minio", "server", "--address", "0.0.0.0:9000", From 5a2b14bc32fef3ee3b6f399c7466186197f8e7b8 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Wed, 22 Jan 2025 07:34:53 +0100 Subject: [PATCH 09/11] Update test/end-to-end/test_simple.py --- test/end-to-end/test_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/end-to-end/test_simple.py b/test/end-to-end/test_simple.py index bbe30e22f..178b01f4d 100644 --- a/test/end-to-end/test_simple.py +++ b/test/end-to-end/test_simple.py @@ -229,7 +229,7 @@ def minio_server(tmp_path: Path): while retries < max_retries: try: subprocess.run( - [f"mc{'.bat' if os.name == 'nt' else ''}", "mb", "local/s3-forge"], + ["mc", "mb", "local/s3-forge"], env={ **os.environ, "MC_HOST_local": "http://minioadmin:minioadmin@localhost:9000", From 674bce6602f28b7d61161f08f12d8fddcf9137d3 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Wed, 22 Jan 2025 07:34:59 +0100 Subject: [PATCH 10/11] Update test/end-to-end/test_simple.py --- test/end-to-end/test_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/end-to-end/test_simple.py b/test/end-to-end/test_simple.py index 178b01f4d..1df3718b2 100644 --- a/test/end-to-end/test_simple.py +++ b/test/end-to-end/test_simple.py @@ -294,7 +294,7 @@ def test_s3_minio_upload( # Make sure the package was uploaded assert b"globtest" in subprocess.check_output( [ - f"mc{'.bat' if os.name == 'nt' else ''}", + "mc", "ls", "--recursive", "local/s3-forge/channel", From 73e50f99fa9953d0fb1e44d082a6a7eb444117fa Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Fri, 24 Jan 2025 08:30:01 +0100 Subject: [PATCH 11/11] Fix integration test on Windows? --- .github/workflows/end-to-end.yml | 9 ++++++--- test/end-to-end/test_simple.py | 18 +++--------------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml index 535cc37b4..8b68948b2 100644 --- a/.github/workflows/end-to-end.yml +++ b/.github/workflows/end-to-end.yml @@ -1,6 +1,6 @@ on: push: - branches: [ main ] + branches: [main] paths: - "src/**" - "Cargo.*" @@ -29,7 +29,7 @@ jobs: run_tests: strategy: matrix: - os: [ "ubuntu-latest", "windows-latest", "macos-latest" ] + os: ["ubuntu-latest", "windows-latest", "macos-latest"] fail-fast: false name: Run tests @@ -45,7 +45,10 @@ jobs: # build in release mode so that it's reasonably fast to run the tests - run: pixi run build-release - run: pixi run test-ci - - run: pixi run test-end-to-end -s -v + - run: | + mkdir -p ${{ runner.temp }}/minio-data + pixi run minio server --address 127.0.0.1:9000 ${{ runner.temp }}/minio-data & + pixi run test-end-to-end -s -v env: PREFIX_DEV_READ_ONLY_TOKEN: ${{ secrets.PREFIX_DEV_READ_ONLY_TOKEN }} ANACONDA_ORG_TEST_TOKEN: ${{ secrets.ANACONDA_ORG_TEST_TOKEN }} diff --git a/test/end-to-end/test_simple.py b/test/end-to-end/test_simple.py index 2b7d1aad4..0a09baaf2 100644 --- a/test/end-to-end/test_simple.py +++ b/test/end-to-end/test_simple.py @@ -211,25 +211,14 @@ def test_anaconda_upload( @pytest.fixture() -def minio_server(tmp_path: Path): - tmp_dir = tmp_path / "minio" - # Start minio server - proc = subprocess.Popen( - [ - "minio", - "server", - "--address", - "0.0.0.0:9000", - str(tmp_dir), - ], - ) +def create_s3_channel(): # Create a bucket (with retries, in case we have to wait until minio is up and running) retries = 0 max_retries = 3 while retries < max_retries: try: subprocess.run( - ["mc", "mb", "local/s3-forge"], + ["mc", "mb", "local/s3-forge", "--ignore-existing"], env={ **os.environ, "MC_HOST_local": "http://minioadmin:minioadmin@localhost:9000", @@ -245,7 +234,6 @@ def minio_server(tmp_path: Path): "Failed to create bucket, could not reach minio server" ) yield - proc.kill() @pytest.fixture() @@ -265,7 +253,7 @@ def s3_credentials_file(tmp_path: Path) -> Path: return path -@pytest.mark.usefixtures("minio_server") +@pytest.mark.usefixtures("create_s3_channel") def test_s3_minio_upload( rattler_build: RattlerBuild, recipes: Path,