global caches & cargo-sweep

rust/Earthfile

@@ -1,30 +1,37 @@
-VERSION 0.7
+VERSION --global-cache 0.7
 
 # CARGO runs the cargo command "cargo $args".
 # This UDC should be thread safe. Parallel builds of targets using it should be free of race conditions.
 # Arguments:
 #   - args: Cargo subcommand and its arguments. Required
 #   - keep_fingerprints (false): Do not remove source packages fingerprints. Use only when source packages have been COPYed with --keep-ts option
+#   - sweep_days (4): Number of days to retain target build artifacts in cache
 #   - output: Regex to match the files within the target folder to be copied from the cache to the caller filesystem (image layers).
 #     Use this argument when you want to SAVE an ARTIFACT from the target folder (mounted cache), always trying to minimize the total size of the copied fileset.
 #     For example --output="release/[^\./]+" would keep all the files in /target/release that don't have any extension
 CARGO:
     COMMAND
     ARG --required args
     ARG keep_fingerprints=false
+    ARG sweep_days=4
     ARG output
     IF [ "$keep_fingerprints" = "false" ]
        DO +REMOVE_SOURCE_FINGERPRINTS
     END
-    RUN --mount=type=cache,mode=0777,sharing=shared,target=$CARGO_HOME/registry \
-        --mount=type=cache,mode=0777,sharing=shared,target=$CARGO_HOME/git \
-        --mount=type=cache,mode=0777,target=target \
-        cargo $args; \
-        if [ -n "$output" ]; then \
-          mkdir /earthly_lib_rust_temp; \
-          cd target; \
-          find . -type f -regextype posix-egrep -regex "./$output" -exec cp --parents \{\} /earthly_lib_rust_temp \; ; \
-        fi
+    DO +INSTALL_CARGO_SWEEP
+    DO +RUN_WITH_CACHE --command="echo \"Running cargo $args\" ;
+        cargo $args;
+        if [ -n \"$output\" ]; then
+          echo \"Copying output files\" ;
+          mkdir -p /earthly_lib_rust_temp;
+          cd target;
+          find . -type f -regextype posix-egrep -regex \"./$output\" -exec cp --parents \{\} /earthly_lib_rust_temp \; ;
+          cd ..;
+        fi;
+        echo \"Running cargo sweep -r -t $sweep_days\" ;
+        cargo sweep -r -t $sweep_days;
+        echo \"Running cargo sweep -r -i\" ;
+        cargo sweep -r -i;"
     IF [ "$output" != "" ]
       RUN mkdir -p target; \
           mv /earthly_lib_rust_temp/* target 2>/dev/null || echo "no files found within ./target matching the provided output regexp" ;
@@ -34,21 +41,59 @@ CARGO:
 # This guarantees Cargo compiles the packages when COPY commands of the source folders have a static timestamp (see --keep-ts)
 REMOVE_SOURCE_FINGERPRINTS:
     COMMAND
-    COPY +get-stoml/stoml /tmp/stoml
-    ARG source_libs=$(find . -name Cargo.toml -exec bash -c '/tmp/stoml {} package.name; printf "\n"' \\;)
-    RUN --mount=type=cache,mode=0777,sharing=shared,target=target \
-      find target -name .fingerprint; \
-      fingerprint_folders=$(find target -name .fingerprint) ; \
-      for fingerprint_folder in $fingerprint_folders; do \
-        cd $fingerprint_folder; \
-        for source_lib in $source_libs; do \
-          find . -maxdepth 1 -regex "\./$source_lib-[^-]+" -exec rm -rf {} \; ;\
-        done \
-      done
+    COPY +get-tomljson/tomljson /tmp/tomljson
+    COPY +get-jq/jq /tmp/jq
+    DO +RUN_WITH_CACHE --command="set -e;
+      source_libs=\$(find . -name Cargo.toml -exec bash -c '/tmp/tomljson {} | jq -r .package.name; printf \"\\n\"' \\;) ;
+      fingerprint_folders=\$(find target -name .fingerprint) ;
+      echo \"deleting fingerprints:\";
+      for fingerprint_folder in \$fingerprint_folders; do
+        cd \$fingerprint_folder;
+        for source_lib in \$source_libs; do
+          find . -maxdepth 1 -regex \"\./\$source_lib-[^-]+\" -exec bash -c 'readlink -f {}; rm -rf {}' \; ;
+        done
+      done"
+
+# get-tomljson gets the portable tomljson binary
+get-tomljson:
+    FROM alpine:3.18.3
+    RUN wget -O tomljson.tar.xz https://github.com/pelletier/go-toml/releases/download/v2.1.0/tomljson_2.1.0_linux_amd64.tar.xz; \
+        tar -xf tomljson.tar.xz; \
+        chmod +x tomljson
+    SAVE ARTIFACT ./tomljson tomljson
 
-# get-stoml gets the portable stoml binary
-get-stoml:
+# get-jq gets the portable jq binary
+get-jq:
     FROM alpine:3.18.3
-    RUN wget -O stoml https://github.com/freshautomations/stoml/releases/download/v0.7.1/stoml_linux_amd64; \
-       chmod +x stoml
-    SAVE ARTIFACT ./stoml stoml
+    RUN apk add jq;
+    ARG jq=$(which jq)
+    SAVE ARTIFACT $jq jq
+
+# INSTALL_CARGO_SWEEP installs cargo-sweep if it doesn't exist already
+INSTALL_CARGO_SWEEP:
+    COMMAND
+    DO +RUN_WITH_CACHE --command="set -e;
+        if [ ! -f \$CARGO_HOME/bin/cargo-sweep ]; then
+          cargo install cargo-sweep --root \$CARGO_HOME;
+        fi;"
+
+# RUN_WITH_CACHE runs the passed command with the CARGO caches mounted.
+# Arguments:
+#   - command (required): Command to run, can be any expression.
+#
+# This implementation is not expected to significantly change. Prefer using the
+# `CARGO` UDC if you can, so you can get future improvements transparently.
+RUN_WITH_CACHE:
+    COMMAND
+    ARG command
+    ARG ORIGINAL_CARGO_HOME=$CARGO_HOME
+    ENV CARGO_INSTALL_ROOT=$CARGO_HOME
+    ENV CARGO_HOME="/earthly/.cargo"
+    RUN --mount=type=cache,mode=0777,id="earthly-cargo-cache",sharing=shared,target=/earthly \
+        --mount=type=cache,mode=0777,target=target \
+    set -e; \
+    mkdir -p $CARGO_HOME; \
+    mkdir -p /earthly/bin ;\
+    printf "Running:\n      $command\n"; \
+    eval $command
+    ENV CARGO_HOME=$ORIGINAL_CARGO_HOME