diff --git a/Cargo.lock b/Cargo.lock
index af84a3d..80af0e8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,6 +2,21 @@
# It is not intended for manual editing.
version = 3
+[[package]]
+name = "addr2line"
+version = "0.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97"
+dependencies = [
+ "gimli",
+]
+
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
[[package]]
name = "ahash"
version = "0.7.6"
@@ -33,9 +48,12 @@ dependencies = [
[[package]]
name = "anyhow"
-version = "1.0.56"
+version = "1.0.70"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27"
+checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4"
+dependencies = [
+ "backtrace",
+]
[[package]]
name = "arrayref"
@@ -66,6 +84,21 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+[[package]]
+name = "backtrace"
+version = "0.3.67"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca"
+dependencies = [
+ "addr2line",
+ "cc",
+ "cfg-if 1.0.0",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+]
+
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -410,6 +443,12 @@ dependencies = [
"wasi",
]
+[[package]]
+name = "gimli"
+version = "0.27.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4"
+
[[package]]
name = "globset"
version = "0.4.8"
@@ -610,6 +649,15 @@ dependencies = [
"autocfg",
]
+[[package]]
+name = "miniz_oxide"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa"
+dependencies = [
+ "adler",
+]
+
[[package]]
name = "nix"
version = "0.23.1"
@@ -658,6 +706,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
+[[package]]
+name = "object"
+version = "0.30.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "once_cell"
version = "1.10.0"
@@ -796,6 +853,12 @@ dependencies = [
"smallvec",
]
+[[package]]
+name = "rustc-demangle"
+version = "0.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4a36c42d1873f9a77c53bde094f9664d9891bc604a45b4798fd2c389ed12e5b"
+
[[package]]
name = "ryu"
version = "1.0.9"
diff --git a/Cargo.toml b/Cargo.toml
index 4833806..e044383 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -27,7 +27,7 @@ itertools = "0.10"
rusqlite = { version = "0.25", features = ["bundled"] }
lz4 = "1.2"
zstd-safe = { version = "6.0", features = ["std", "experimental"] }
-anyhow = "1"
+anyhow = { version = "1.0.69", features = [ "backtrace" ] }
thiserror = "1.0"
libc = "0.2"
getopts = "0.2"
diff --git a/cli-tests/cli-tests.bats b/cli-tests/cli-tests.bats
index 6892022..62ff603 100644
--- a/cli-tests/cli-tests.bats
+++ b/cli-tests/cli-tests.bats
@@ -54,21 +54,21 @@ teardown () {
@test "simple put+get primary key" {
data="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
echo -n "$data" > "$SCRATCH/foo.txt"
- id="$(bupstash put :: "$SCRATCH/foo.txt")"
+ id="$(bupstash put -- "$SCRATCH/foo.txt")"
test "$data" = "$(bupstash get id=$id )"
}
@test "simple put+get put key" {
data="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
echo -n "$data" > "$SCRATCH/foo.txt"
- id="$(bupstash put -k "$PUT_KEY" :: "$SCRATCH/foo.txt")"
+ id="$(bupstash put -k "$PUT_KEY" -- "$SCRATCH/foo.txt")"
test "$data" = "$(bupstash get id=$id )"
}
@test "simple put+get no compression" {
data="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
echo -n "$data" > "$SCRATCH/foo.txt"
- id="$(bupstash put --compression=none -k "$PUT_KEY" :: "$SCRATCH/foo.txt")"
+ id="$(bupstash put --compression=none -k "$PUT_KEY" -- "$SCRATCH/foo.txt")"
test "$data" = "$(bupstash get id=$id )"
}
@@ -78,14 +78,23 @@ teardown () {
mkdir "$SCRATCH/d/f"
echo foo > "$SCRATCH/d/foo.txt"
- id="$(bupstash put name=x.tar "$SCRATCH/d")"
- id="$(bupstash put name=foo "$SCRATCH/d/foo.txt")"
- id="$(bupstash put name=bar.tar "$SCRATCH/d/e" "$SCRATCH/d/f")"
+ id="$(bupstash put -t name=x.tar "$SCRATCH/d")"
+ id="$(bupstash put -t name=foo "$SCRATCH/d/foo.txt")"
+ id="$(bupstash put -t name=bar.tar "$SCRATCH/d/e" "$SCRATCH/d/f")"
bupstash get name=x.tar > /dev/null
bupstash get name=foo > /dev/null
bupstash get name=bar.tar > /dev/null
}
+@test "put --tag CLI option" {
+ data="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+ echo -n "$data" > "$SCRATCH/foo.txt"
+
+ # Same tag twice
+ ! bupstash put -t foo=bar --tag foo=baz -- "$SCRATCH/foo.txt"
+ ! bupstash put -t foo=bar --tag foo=bar -- "$SCRATCH/foo.txt"
+}
+
@test "random data" {
for i in $(echo 0 1024 4096 1000000 100000000)
do
@@ -97,7 +106,7 @@ teardown () {
# Workaround since macOS's head doesn't support a byte count of 0
touch "$SCRATCH/rand.dat"
fi
- id="$(bupstash put -k "$PUT_KEY" :: "$SCRATCH/rand.dat")"
+ id="$(bupstash put -k "$PUT_KEY" -- "$SCRATCH/rand.dat")"
bupstash get id=$id > "$SCRATCH/got.dat"
bupstash gc
cmp --silent "$SCRATCH/rand.dat" "$SCRATCH/got.dat"
@@ -109,7 +118,7 @@ teardown () {
do
rm -f "$SCRATCH/yes.dat"
dd if=/dev/zero of="$SCRATCH/yes.dat" bs=$i count=1
- id="$(bupstash put -k "$PUT_KEY" :: "$SCRATCH/yes.dat")"
+ id="$(bupstash put -k "$PUT_KEY" -- "$SCRATCH/yes.dat")"
bupstash get id=$id > "$SCRATCH/got.dat"
bupstash gc
cmp --silent "$SCRATCH/yes.dat" "$SCRATCH/got.dat"
@@ -119,7 +128,7 @@ teardown () {
@test "key mismatch" {
data="abc123"
echo -n "$data" > "$SCRATCH/foo.txt"
- id="$(bupstash put :: "$SCRATCH/foo.txt")"
+ id="$(bupstash put -- "$SCRATCH/foo.txt")"
bupstash new-key -o "$SCRATCH/wrong.key"
run bupstash get -k "$SCRATCH/wrong.key" id=$id
echo "$output" | grep -q "key does not match"
@@ -136,7 +145,7 @@ teardown () {
fi
data="abc123"
echo -n "$data" > "$SCRATCH/foo.txt"
- id="$(bupstash put :: "$SCRATCH/foo.txt")"
+ id="$(bupstash put -- "$SCRATCH/foo.txt")"
echo 'XXXXXXXXXXXXXXXXXXXXX' > "$BUPSTASH_REPOSITORY/data/"*;
run bupstash get id=$id
echo "$output"
@@ -151,7 +160,7 @@ _concurrent_send_test_worker () {
set -e
for i in $(seq 50)
do
- id="$(bupstash put -e --no-send-log :: echo $i)"
+ id="$(bupstash put --exec --no-send-log -- echo $i)"
test "$i" = "$(bupstash get id=$id)"
done
}
@@ -170,7 +179,7 @@ _concurrent_send_test_worker () {
@test "simple search and listing" {
for i in $(seq 100) # Enough to trigger more than one sync packet.
do
- bupstash put -e "i=$i" :: echo $i
+ bupstash put --exec -t "i=$i" -- echo $i
done
for k in $BUPSTASH_KEY $METADATA_KEY
do
@@ -183,8 +192,8 @@ _concurrent_send_test_worker () {
@test "rm and gc" {
test 0 = $(bupstash list | expr $(wc -l))
test 0 = "$(sqlite3 "$SCRATCH/query-cache.sqlite3" 'select count(*) from ItemOpLog;')"
- id1="$(bupstash put -e :: echo hello1)"
- id2="$(bupstash put -e :: echo hello2)"
+ id1="$(bupstash put --exec -- echo hello1)"
+ id2="$(bupstash put --exec -- echo hello2)"
test 2 = $(bupstash list | expr $(wc -l))
test 2 = "$(sqlite3 "$SCRATCH/query-cache.sqlite3" 'select count(*) from ItemOpLog;')"
if test -n "$BUPSTASH_REPOSITORY"
@@ -223,8 +232,8 @@ _concurrent_send_test_worker () {
@test "rm and recover-removed" {
test 0 = $(bupstash list | expr $(wc -l))
test 0 = "$(sqlite3 "$SCRATCH/query-cache.sqlite3" 'select count(*) from ItemOpLog;')"
- id1="$(bupstash put -e :: echo hello1)"
- id2="$(bupstash put -e :: echo hello2)"
+ id1="$(bupstash put --exec -- echo hello1)"
+ id2="$(bupstash put --exec -- echo hello2)"
test 2 = "$(bupstash list | expr $(wc -l))"
test 2 = "$(sqlite3 "$SCRATCH/query-cache.sqlite3" 'select count(*) from ItemOpLog;')"
if test -n "$BUPSTASH_REPOSITORY"
@@ -264,9 +273,9 @@ _concurrent_send_test_worker () {
}
@test "query sync" {
- id1="$(bupstash put -e :: echo hello1)"
+ id1="$(bupstash put --exec -- echo hello1)"
test 1 = $(bupstash list | expr $(wc -l))
- id2="$(bupstash put -e :: echo hello2)"
+ id2="$(bupstash put --exec -- echo hello2)"
test 2 = $(bupstash list | expr $(wc -l))
bupstash rm id=$id1
test 1 = $(bupstash list | expr $(wc -l))
@@ -279,16 +288,16 @@ _concurrent_send_test_worker () {
}
@test "get via query" {
- bupstash put -e foo=bar echo -n hello1
- bupstash put -e foo=baz echo -n hello2
- bupstash put -e foo=bang echo -n hello2
+ bupstash put --exec -t foo=bar echo -n hello1
+ bupstash put --exec -t foo=baz echo -n hello2
+ bupstash put --exec -t foo=bang echo -n hello2
test "hello2" = $(bupstash get "foo=ban*")
}
@test "rm via query" {
- bupstash put -e foo=bar echo -n hello1
- bupstash put -e foo=baz echo -n hello2
- bupstash put -e foo=bang echo -n hello2
+ bupstash put --exec -t foo=bar echo -n hello1
+ bupstash put --exec -t foo=baz echo -n hello2
+ bupstash put --exec -t foo=bang echo -n hello2
test 3 = $(bupstash list | expr $(wc -l))
if bupstash rm "foo=*"
then
@@ -306,12 +315,12 @@ _concurrent_send_test_worker () {
echo b > "$SCRATCH/foo/b.txt"
mkdir "$SCRATCH/foo/bar"
echo c > "$SCRATCH/foo/bar/c.txt"
- id=$(bupstash put :: "$SCRATCH/foo")
+ id=$(bupstash put -- "$SCRATCH/foo")
test 5 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
# Test again to excercise stat caching.
- id=$(bupstash put :: "$SCRATCH/foo")
+ id=$(bupstash put -- "$SCRATCH/foo")
test 5 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
- id=$(bupstash put :: "$SCRATCH/foo/a.txt" "$SCRATCH/foo/b.txt")
+ id=$(bupstash put -- "$SCRATCH/foo/a.txt" "$SCRATCH/foo/b.txt")
test 3 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
}
@@ -321,19 +330,19 @@ _concurrent_send_test_worker () {
echo b > "$SCRATCH/foo/b.txt"
mkdir "$SCRATCH/foo/bar"
echo c > "$SCRATCH/foo/bar/c.txt"
- id=$(bupstash put --no-send-log :: "$SCRATCH/foo")
+ id=$(bupstash put --no-send-log -- "$SCRATCH/foo")
test 5 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
- id=$(bupstash put --no-stat-caching :: "$SCRATCH/foo")
+ id=$(bupstash put --no-stat-caching -- "$SCRATCH/foo")
test 5 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
}
@test "stat cache invalidated" {
mkdir "$SCRATCH/foo"
echo a > "$SCRATCH/foo/a.txt"
- id=$(bupstash put :: "$SCRATCH/foo")
+ id=$(bupstash put -- "$SCRATCH/foo")
bupstash rm id=$id
bupstash gc
- id=$(bupstash put :: "$SCRATCH/foo")
+ id=$(bupstash put -- "$SCRATCH/foo")
bupstash get id=$id > /dev/null
}
@@ -346,7 +355,7 @@ _concurrent_send_test_worker () {
unset BUPSTASH_REPOSITORY
data="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
echo -n "$data" > "$SCRATCH/foo.txt"
- id="$(bupstash put :: "$SCRATCH/foo.txt")"
+ id="$(bupstash put -- "$SCRATCH/foo.txt")"
test "$data" = "$(bupstash get id=$id )"
}
@@ -354,7 +363,7 @@ _concurrent_send_test_worker () {
export BUPSTASH_KEY_COMMAND="cat $BUPSTASH_KEY"
data="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
echo -n "$data" > "$SCRATCH/foo.txt"
- id="$(bupstash put :: "$SCRATCH/foo.txt")"
+ id="$(bupstash put -- "$SCRATCH/foo.txt")"
test "$data" = "$(bupstash get id=$id )"
}
@@ -365,7 +374,7 @@ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaa\
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaa\
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaa\
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaa
- id=$(bupstash put :: "$SCRATCH/foo")
+ id=$(bupstash put -- "$SCRATCH/foo")
test 7 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
}
@@ -377,7 +386,7 @@ llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll\
llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll\
llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll\
"$SCRATCH/foo/l"
- id=$(bupstash put :: "$SCRATCH/foo")
+ id=$(bupstash put -- "$SCRATCH/foo")
test 2 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
}
@@ -405,38 +414,38 @@ llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll\
touch "$SCRATCH/foo/bar/baz/bang"
# No exclude, everything should be in
- id=$(bupstash put :: "$SCRATCH/foo")
+ id=$(bupstash put -- "$SCRATCH/foo")
test 6 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
# Exclude on multiple levels
# As expected, this also excludes $SCRATCH/foo/bang
- id=$(bupstash put --exclude="$SCRATCH/foo/**/bang" :: "$SCRATCH/foo")
+ id=$(bupstash put --exclude="$SCRATCH/foo/**/bang" -- "$SCRATCH/foo")
bupstash get id=$id | tar -tf -
test 3 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
# Exclude on multiple levels (should be the same)
- id=$(bupstash put --exclude="**/bang" :: "$SCRATCH/foo")
+ id=$(bupstash put --exclude="**/bang" -- "$SCRATCH/foo")
test 3 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
# Exclude on multiple levels (should still be the same)
- id=$(bupstash put --exclude="/**/bang" :: "$SCRATCH/foo")
+ id=$(bupstash put --exclude="/**/bang" -- "$SCRATCH/foo")
test 3 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
# Still the same thing, but using the "match on name" shorthand (no slashes = only file name)
- id=$(bupstash put --exclude="bang" :: "$SCRATCH/foo")
+ id=$(bupstash put --exclude="bang" -- "$SCRATCH/foo")
test 3 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
# Exclude on a single level
# We want /foo /foo/bar /foo/bar/baz /foo/bang /foo/bar/baz/bang (that one's important)
- id=$(bupstash put --exclude="$SCRATCH/foo/*/bang" :: "$SCRATCH/foo")
+ id=$(bupstash put --exclude="$SCRATCH/foo/*/bang" -- "$SCRATCH/foo")
test 5 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
# Exclude on a single level, but wrongly so (nothing gets excluded)
- id=$(bupstash put --exclude="/*/bang" :: "$SCRATCH/foo")
+ id=$(bupstash put --exclude="/*/bang" -- "$SCRATCH/foo")
test 6 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
# Invalid exclusion regex
- ! bupstash put --exclude="*/bar" :: "$SCRATCH/foo"
+ ! bupstash put --exclude="*/bar" -- "$SCRATCH/foo"
}
# Test exclude marker files
@@ -450,7 +459,7 @@ llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll\
touch "$SCRATCH/foo/bar/baz/bang"
# Keep . bang bar
- id=$(bupstash put --exclude-if-present=".backupignore" :: "$SCRATCH/foo")
+ id=$(bupstash put --exclude-if-present=".backupignore" -- "$SCRATCH/foo")
test 4 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
}
@@ -462,7 +471,7 @@ llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll\
n=100000000
export BUPSTASH_CHECKPOINT_SECONDS=0 # Checkpoint as often as possible
head -c $n /dev/urandom > "$SCRATCH/rand.dat"
- id="$(bupstash put :: "$SCRATCH/rand.dat")"
+ id="$(bupstash put -- "$SCRATCH/rand.dat")"
bupstash get id=$id > "$SCRATCH/got.dat"
bupstash gc
cmp --silent "$SCRATCH/rand.dat" "$SCRATCH/got.dat"
@@ -481,14 +490,14 @@ llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll\
echo foo > "$SCRATCH/foo/bar$i/data"
done
export BUPSTASH_CHECKPOINT_SECONDS=0 # Checkpoint as often as possible
- id=$(bupstash put :: "$SCRATCH/foo")
+ id=$(bupstash put -- "$SCRATCH/foo")
test 101 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
}
@test "rm from stdin" {
- id1="$(bupstash put -e echo hello1)"
- id2="$(bupstash put -e echo hello2)"
- id3="$(bupstash put -e echo hello3)"
+ id1="$(bupstash put --exec echo hello1)"
+ id2="$(bupstash put --exec echo hello2)"
+ id3="$(bupstash put --exec echo hello3)"
test 3 = "$(bupstash list | expr $(wc -l))"
echo "${id1}" | bupstash rm --ids-from-stdin
test 2 = "$(bupstash list | expr $(wc -l))"
@@ -548,8 +557,8 @@ _concurrent_modify_worker () {
}
@test "list and rm no key" {
- bupstash put -e echo hello1
- bupstash put -e echo hello2
+ bupstash put --exec echo hello1
+ bupstash put --exec echo hello2
unset BUPSTASH_KEY
test 2 = "$(bupstash list --query-encrypted | expr $(wc -l))"
bupstash rm --allow-many --query-encrypted id='*'
@@ -595,7 +604,7 @@ _concurrent_modify_worker () {
mkdir "$SCRATCH/foo/bang"
echo foo > "$SCRATCH/foo/bar/baz/a.txt"
- id=$(bupstash put :: "$SCRATCH/foo/bar" "$SCRATCH/foo/bar/baz" "$SCRATCH/foo/bang")
+ id=$(bupstash put -- "$SCRATCH/foo/bar" "$SCRATCH/foo/bar/baz" "$SCRATCH/foo/bang")
bupstash get id=$id | tar -tf -
test 5 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
test 5 = "$(bupstash list-contents id=$id | expr $(wc -l))"
@@ -609,7 +618,7 @@ _concurrent_modify_worker () {
mkdir "$SCRATCH/foo/bang"
echo foo > "$SCRATCH/foo/bar/baz/a.txt"
- id=$(bupstash put :: "$SCRATCH/foo")
+ id=$(bupstash put -- "$SCRATCH/foo")
test 5 = "$(bupstash list-contents id=$id | expr $(wc -l))"
test 5 = "$(bupstash list-contents --pick . id=$id | expr $(wc -l))"
test 3 = "$(bupstash list-contents --pick bar id=$id | expr $(wc -l))"
@@ -620,7 +629,7 @@ _concurrent_modify_worker () {
touch "$SCRATCH/foo/a"
ln "$SCRATCH/foo/a" "$SCRATCH/foo/b"
- id=$(bupstash put :: "$SCRATCH/foo")
+ id=$(bupstash put -- "$SCRATCH/foo")
mkdir "$SCRATCH/restore"
bupstash get id=$id | tar -C "$SCRATCH/restore" -xvf -
@@ -635,7 +644,7 @@ _concurrent_modify_worker () {
touch "$SCRATCH/foo/$name"
ln "$SCRATCH/foo/$name" "$SCRATCH/foo/b"
- id=$(bupstash put :: "$SCRATCH/foo")
+ id=$(bupstash put -- "$SCRATCH/foo")
mkdir "$SCRATCH/restore"
if test $(uname) != "Linux"
@@ -660,7 +669,7 @@ _concurrent_modify_worker () {
ln -s "$SCRATCH/foo/a" "$SCRATCH/foo/b"
ln -n "$SCRATCH/foo/b" "$SCRATCH/foo/c"
- id=$(bupstash put :: "$SCRATCH/foo")
+ id=$(bupstash put -- "$SCRATCH/foo")
mkdir "$SCRATCH/restore"
bupstash get id=$id | tar -C "$SCRATCH/restore" -xvf -
@@ -721,14 +730,14 @@ _concurrent_modify_worker () {
bupstash list
bupstash list-contents id=$id
if bupstash init ; then exit 1 ; fi
- if bupstash put -e echo hi ; then exit 1 ; fi
+ if bupstash put --exec echo hi ; then exit 1 ; fi
if bupstash rm id=$id ; then exit 1 ; fi
if bupstash recover-removed ; then exit 1 ; fi
if bupstash gc ; then exit 1 ; fi
if bupstash gc ; then exit 1 ; fi
export BUPSTASH_REPOSITORY_COMMAND="bupstash serve --allow-put $REPO"
- bupstash put -e echo hi
+ bupstash put --exec echo hi
if bupstash init ; then exit 1 ; fi
if bupstash get id=$id > /dev/null ; then exit 1 ; fi
if bupstash list ; then exit 1 ; fi
@@ -742,14 +751,14 @@ _concurrent_modify_worker () {
bupstash list-contents id=$id
if bupstash init ; then exit 1 ; fi
if bupstash get id=$id > /dev/null ; then exit 1 ; fi
- if bupstash put -e echo hi ; then exit 1 ; fi
+ if bupstash put --exec echo hi ; then exit 1 ; fi
if bupstash rm id=$id ; then exit 1 ; fi
if bupstash recover-removed ; then exit 1 ; fi
if bupstash gc ; then exit 1 ; fi
export BUPSTASH_REPOSITORY_COMMAND="bupstash serve --allow-gc $REPO"
if bupstash init ; then exit 1 ; fi
- if bupstash put -e echo hi ; then exit 1 ; fi
+ if bupstash put --exec echo hi ; then exit 1 ; fi
if bupstash get id=$id > /dev/null ; then exit 1 ; fi
if bupstash list ; then exit 1 ; fi
if bupstash list-contents id=$id ; then exit 1 ; fi
@@ -762,7 +771,7 @@ _concurrent_modify_worker () {
bupstash list-contents id=$id
if bupstash init ; then exit 1 ; fi
if bupstash get id=$id > /dev/null ; then exit 1 ; fi
- if bupstash put -e echo hi ; then exit 1 ; fi
+ if bupstash put --exec echo hi ; then exit 1 ; fi
if bupstash recover-removed ; then exit 1 ; fi
if bupstash gc ; then exit 1 ; fi
# delete as the last test
@@ -783,7 +792,7 @@ _concurrent_modify_worker () {
echo -n "abc" > "$SCRATCH/d/a.txt"
id=$(bupstash put $SCRATCH/d)
bupstash restore --into $SCRATCH/restore id=$id
- test 0 = "$(bupstash diff --relaxed $SCRATCH/d :: $SCRATCH/restore | expr $(wc -l))"
+ test 0 = "$(bupstash diff --relaxed $SCRATCH/d -- $SCRATCH/restore | expr $(wc -l))"
}
@test "restore symlink" {
@@ -791,7 +800,7 @@ _concurrent_modify_worker () {
ln -s missing.txt "$SCRATCH"/d/l
id=$(bupstash put "$SCRATCH"/d)
bupstash restore --into "$SCRATCH"/restore id=$id
- test 0 = "$(bupstash diff --relaxed "$SCRATCH"/d :: "$SCRATCH"/restore | expr $(wc -l))"
+ test 0 = "$(bupstash diff --relaxed "$SCRATCH"/d -- "$SCRATCH"/restore | expr $(wc -l))"
}
@test "restore hardlink" {
@@ -800,7 +809,7 @@ _concurrent_modify_worker () {
ln "$SCRATCH"/d/a.txt "$SCRATCH"/d/b.txt
id=$(bupstash put "$SCRATCH"/d)
bupstash restore --into "$SCRATCH"/restore id=$id
- test 0 = "$(bupstash diff --relaxed "$SCRATCH"/d :: "$SCRATCH"/restore | expr $(wc -l))"
+ test 0 = "$(bupstash diff --relaxed "$SCRATCH"/d -- "$SCRATCH"/restore | expr $(wc -l))"
echo -n "xxx" >> "$SCRATCH/restore/a.txt"
test $(cat "$SCRATCH"/restore/a.txt) = $(cat "$SCRATCH"/restore/b.txt)
}
@@ -816,7 +825,7 @@ _concurrent_modify_worker () {
id=$(bupstash put "$SCRATCH"/d)
bupstash restore --into "$SCRATCH"/restore id=$id
- test 0 = "$(bupstash diff --relaxed "$SCRATCH"/d :: "$SCRATCH"/restore | expr $(wc -l))"
+ test 0 = "$(bupstash diff --relaxed "$SCRATCH"/d -- "$SCRATCH"/restore | expr $(wc -l))"
echo -n "xxx" >> "$SCRATCH/restore/a.txt"
test $(cat "$SCRATCH"/restore/a.txt) = $(cat "$SCRATCH"/restore/b.txt)
}
@@ -836,7 +845,7 @@ _concurrent_modify_worker () {
id=$(bupstash put "$SCRATCH"/d)
bupstash restore --into $SCRATCH/restore id=$id
- test 0 = "$(bupstash diff --relaxed $SCRATCH/d :: $SCRATCH/restore | expr $(wc -l))"
+ test 0 = "$(bupstash diff --relaxed $SCRATCH/d -- $SCRATCH/restore | expr $(wc -l))"
}
@test "restore pick" {
@@ -846,7 +855,7 @@ _concurrent_modify_worker () {
echo -n "hij" > "$SCRATCH/d/c/c.txt"
id=$(bupstash put "$SCRATCH"/d)
bupstash restore --pick b --into $SCRATCH/restore id=$id
- test 0 = "$(bupstash diff --relaxed $SCRATCH/d/b :: $SCRATCH/restore | expr $(wc -l))"
+ test 0 = "$(bupstash diff --relaxed $SCRATCH/d/b -- $SCRATCH/restore | expr $(wc -l))"
}
@test "restore sparse" {
@@ -864,7 +873,7 @@ _concurrent_modify_worker () {
truncate -s 16M $sparse_dir/file_with_data2.img
echo data >> $sparse_dir/file_with_data2.img
- id=$(bupstash put :: "$sparse_dir")
+ id=$(bupstash put -- "$sparse_dir")
bupstash restore --into "$restore_dir" "id=$id"
diff -u \
@@ -892,13 +901,13 @@ _concurrent_modify_worker () {
"$BATS_TEST_DIRNAME/mk-random-dir.py" "$rand_dir"
# Put twice so we test caching code paths.
- id1=$(bupstash put :: "$rand_dir")
- id2=$(bupstash put :: "$rand_dir")
+ id1=$(bupstash put -- "$rand_dir")
+ id2=$(bupstash put -- "$rand_dir")
for id in $(echo $id1 $id2)
do
bupstash restore --into "$restore_dir" "id=$id"
- test 0 = "$(bupstash diff --relaxed "$rand_dir" :: "$restore_dir" | expr $(wc -l))"
+ test 0 = "$(bupstash diff --relaxed "$rand_dir" -- "$restore_dir" | expr $(wc -l))"
for i in $(seq 3)
do
@@ -907,7 +916,7 @@ _concurrent_modify_worker () {
rm "$to_delete"
done
bupstash restore --into "$restore_dir" "id=$id"
- test 0 = "$(bupstash diff --relaxed "$rand_dir" :: "$restore_dir" | expr $(wc -l))"
+ test 0 = "$(bupstash diff --relaxed "$rand_dir" -- "$restore_dir" | expr $(wc -l))"
done
bupstash rm id=$id
@@ -927,8 +936,8 @@ _concurrent_modify_worker () {
restore_dir="$SCRATCH/restore_dir"
copy_dir="$SCRATCH/copy_dir"
# Put twice so we test caching code paths.
- id1=$(bupstash put :: "$BUPSTASH_TORTURE_DIR")
- id2=$(bupstash put :: "$BUPSTASH_TORTURE_DIR")
+ id1=$(bupstash put -- "$BUPSTASH_TORTURE_DIR")
+ id2=$(bupstash put -- "$BUPSTASH_TORTURE_DIR")
for id in $(echo $id1 $id2)
do
@@ -965,8 +974,8 @@ _concurrent_modify_worker () {
"$BATS_TEST_DIRNAME/mk-random-dir.py" "$rand_dir"
# Put twice so we test caching code paths.
- id1=$(bupstash put :: "$rand_dir")
- id2=$(bupstash put :: "$rand_dir")
+ id1=$(bupstash put -- "$rand_dir")
+ id2=$(bupstash put -- "$rand_dir")
for id in $(echo $id1 $id2)
do
@@ -1004,8 +1013,8 @@ _concurrent_modify_worker () {
restore_dir="$SCRATCH/restore_dir"
copy_dir="$SCRATCH/copy_dir"
# Put twice so we test caching code paths.
- id1=$(bupstash put :: "$BUPSTASH_TORTURE_DIR")
- id2=$(bupstash put :: "$BUPSTASH_TORTURE_DIR")
+ id1=$(bupstash put -- "$BUPSTASH_TORTURE_DIR")
+ id2=$(bupstash put -- "$BUPSTASH_TORTURE_DIR")
for id in $(echo $id1 $id2)
do
@@ -1047,8 +1056,8 @@ _concurrent_modify_worker () {
"$BATS_TEST_DIRNAME/mk-random-dir.py" "$rand_dir"
# Put twice so we test caching code paths.
- id1=$(bupstash put :: "$rand_dir")
- id2=$(bupstash put :: "$rand_dir")
+ id1=$(bupstash put -- "$rand_dir")
+ id2=$(bupstash put -- "$rand_dir")
for id in $(echo $id1 $id2)
do
@@ -1099,7 +1108,7 @@ _concurrent_modify_worker () {
do
# XXX This timeout scheme is very brittle.
export BUPSTASH_REPOSITORY_COMMAND="timeout -s KILL 0.0$(($RANDOM % 10)) bupstash serve $REPO"
- bupstash put -e echo $(uuidgen) || true
+ bupstash put --exec echo $(uuidgen) || true
bupstash gc > /dev/null || true
bupstash rm --allow-many "id=f*" || true
if test "$(($RANDOM % 2))" = 0
@@ -1144,7 +1153,7 @@ _concurrent_modify_worker () {
bupstash exec-with-locks sh -c \
"rmdir \"$BUPSTASH_REPOSITORY/items\" && sleep 0.5 && mkdir \"$BUPSTASH_REPOSITORY/items\"" &
sleep 0.2
- bupstash put -q -e echo foo
+ bupstash put -q --exec echo foo
}
@test "parallel thrash" {
@@ -1193,7 +1202,7 @@ _concurrent_modify_worker () {
# The indexer must handle EACCES in any path element, i.e. gracefully stop at
# the first inaccessible parent directory.
- id=$(bupstash put --ignore-permission-errors :: "$SCRATCH/userdir" /root/rootfile)
+ id=$(bupstash put --ignore-permission-errors -- "$SCRATCH/userdir" /root/rootfile)
rootuid=$(bupstash list-contents --format=jsonl1 --pick root/ id=$id | jq .uid)
test 0 = $rootuid
}
diff --git a/cli-tests/diod-thrash.sh b/cli-tests/diod-thrash.sh
index 66c9e9c..8e0e301 100644
--- a/cli-tests/diod-thrash.sh
+++ b/cli-tests/diod-thrash.sh
@@ -67,7 +67,7 @@ thrash_worker () {
for i in $(seq 15)
do
expected=$(uuidgen)
- id=$(bupstash put -q -e --no-send-log thrash_test=yes :: echo $expected)
+ id=$(bupstash put -q -e --no-send-log -t thrash_test=yes -- echo $expected)
if test "$?" = 0
then
@@ -97,7 +97,7 @@ thrash_worker () {
fi
expected=$(uuidgen)
- id=$(bupstash put -r "$SCRATCH/sync-source-repo" -q -e --no-send-log thrash_test=yes :: echo $expected)
+ id=$(bupstash put -r "$SCRATCH/sync-source-repo" -q -e --no-send-log -t thrash_test=yes -- echo $expected)
bupstash sync -r "$SCRATCH/sync-source-repo" --to "$BUPSTASH_REPOSITORY" -q id="$id" >&2
if test "$?" = 0
then
diff --git a/cli-tests/parallel-thrash.sh b/cli-tests/parallel-thrash.sh
index 940bb4a..da2b74d 100644
--- a/cli-tests/parallel-thrash.sh
+++ b/cli-tests/parallel-thrash.sh
@@ -36,7 +36,7 @@ thrash_worker () {
for i in $(seq 15)
do
expected=$(uuidgen)
- id=$(bupstash put -q -e --no-send-log thrash_test=yes :: echo $expected)
+ id=$(bupstash put -q -e --no-send-log -t thrash_test=yes -- echo $expected)
if test "$?" = 0
then
@@ -66,7 +66,7 @@ thrash_worker () {
fi
expected=$(uuidgen)
- id=$(bupstash put -r "$SCRATCH/sync-source-repo" -q -e --no-send-log thrash_test=yes :: echo $expected)
+ id=$(bupstash put -r "$SCRATCH/sync-source-repo" -q -e --no-send-log -t thrash_test=yes -- echo $expected)
bupstash sync -r "$SCRATCH/sync-source-repo" -q id="$id" >&2
if test "$?" = 0
then
diff --git a/cli-tests/s3-parallel-thrash.sh b/cli-tests/s3-parallel-thrash.sh
index 3cd4583..6ef4643 100644
--- a/cli-tests/s3-parallel-thrash.sh
+++ b/cli-tests/s3-parallel-thrash.sh
@@ -38,7 +38,7 @@ thrash_worker () {
do
expected=$(uuidgen)
- id=$(bupstash put -q -e --no-send-log thrash_test=yes :: echo $expected)
+ id=$(bupstash put -q -e --no-send-log -t thrash_test=yes -- echo $expected)
if test "$?" = 0
then
diff --git a/doc/cli/put.txt b/doc/cli/put.txt
index 8f5264b..03c3dcd 100644
--- a/doc/cli/put.txt
+++ b/doc/cli/put.txt
@@ -1,5 +1,5 @@
-bupstash put [OPTIONS] TAGS... [::] PATHS...
-bupstash put -e [OPTIONS] TAGS... [::] CMD...
+bupstash put [OPTIONS] [--] PATHS...
+bupstash put -e [OPTIONS] [--] CMD...
`bupstash put` encrypts a file, directory, or command output and stores it
in a bupstash repository such that only the primary backup key can decrypt it.
@@ -19,17 +19,21 @@ Examples:
# To avoid resending data needlessly during backups, create job specific send log.
$ bupstash put --send-log /root/backup-sendlog ./to-backup
- # Specify arbitrary metadata as KEY=VALUE before.
- $ bupstash put host=$(hostname) ./file.txt
+ # Specify arbitrary metadata as KEY=VALUE using '-t'/'--tag'.
+ # Can be specified multiple times.
+ # Note that the `name` tag has an automatically generated default value (which is overidden in this example),
+ # this can be deactivated with `--no-default-tags`.
+ $ bupstash put -t name=MyBackup -t host=$(hostname) ./file.txt
# Multiple files and directories can be saved at once.
$ bupstash put ./file1.txt ./file2.txt ./some-dir
+ # You can use '--' to separate options and paths.
+ $ bupstash put -- --weird-file-name.txt
+
# Use --exec to save the output of commands.
- $ bupstash put --exec name=files.tar tar -C ./files -cvf - .
+ # '--' here is optional but helps with readability.
+ $ bupstash put --exec -t name=files.tar -- tar -C ./files -cvf - .
# Put from stdin (does not check error codes).
$ echo data | bupstash put -
-
- # You can use '::' to terminate the tag list.
- $ bupstash put :: foo=bar.txt
diff --git a/doc/man/bupstash-put.1.md b/doc/man/bupstash-put.1.md
index 73653a8..7c36bfc 100644
--- a/doc/man/bupstash-put.1.md
+++ b/doc/man/bupstash-put.1.md
@@ -5,8 +5,8 @@ bupstash-put(1)
Put data into a bupstash repository.
-`bupstash put [OPTIONS] [TAG=VAL...] [::] PATHS...`
-`bupstash put --exec [OPTIONS] [TAG=VAL...] [::] COMMAND`
+`bupstash put [OPTIONS] [--] PATHS...`
+`bupstash put --exec [OPTIONS] [--] COMMAND...`
## DESCRIPTION
@@ -21,14 +21,6 @@ Data stored in a bupstash repository is automatically deduplicated
such that the same or similar snapshots take minimal additional disk space.
For efficient incremental operation, use the --send-log option described in the usage notes section.
-All puts can associated with a set of arbitrary encrypted metadata tags, which
-can be queried using bupstash-list(1). Tags are specified in a simple
-`KEY=VALUE` format on the command line. Valid tag keys *must* match the
-regular expression `^([a-zA-Z0-9\\-_]+)=(.+)$`, that means tag keys must be alpha numeric
-with the addition of `-` and `_`. Tag processing ends at the first argument that does not match the pattern.
-
-The special marker argument `::` may be used to force the end of tag parsing, but is usually not necessary.
-
Note that multiple concurrent uploads to the same repository are safe and supported provided that all clients
are accessing the repository from the same server and thus respect the repository file locks.
Some network filesystems (like NFS and sshfs) do not always respect remote file locks and are therefore not supported.
@@ -76,24 +68,32 @@ snapshots include ZFS, BTRFS and also LVM snapshots
Another choice is to perform a put operation at a time when the files are less likely to be modified,
this will provide backups that are good enough for many people without extra complications.
-### Default tags
+### Tags
+
+All puts can associated with a set of arbitrary encrypted metadata tags, which
+can be queried using bupstash-list(1). Tags are specified in a simple
+`KEY=VALUE` format on the command line via the `--tag`/`-t` flag. Valid tag keys *must* match the
+regular expression `^([a-zA-Z0-9\\-_]+)=(.+)$`, that means tag keys must be alpha numeric
+with the addition of `-` and `_`.
+
+#### Default tags
`bupstash` automatically sets default tags.
Currently they are:
-- name, set to the `FILENAME`, or `DIRNAME.tar`, omitted when putting in --exec mode.
+- `name`, set to the `FILENAME`, or `DIRNAME.tar`, omitted when putting in --exec mode.
Default tags can be overidden manually by simply specifying them.
-### Reserved tags
+#### Reserved tags
The following tags are reserved and cannot be set manually:
-- id
-- decryption-key-id
-- size
-- timestamp
+- `id`
+- `decryption-key-id`
+- `size`
+- `timestamp`
### File actions
@@ -163,6 +163,9 @@ With possible types:
This will still backup the folder itself, containing the marker file. Common marker file names are `CACHEDIR.TAG`, `.backupexclude`
or `.no-backup`.
+* -t, --tag KEY=VALUE:
+ Add a tag which the backup will be associated with. May be passed multiple times. No duplicate tag keys allowed.
+
* --send-log PATH:
Path to the send log file, defaults to one of the following, in order, provided
the appropriate environment variables are set, `$BUPSTASH_SEND_LOG`,
@@ -254,7 +257,7 @@ deduplicating repeated files.
# Snapshot a directory.
$ ID="$(bupstash put ./data)"
# List snapshot contents.
-$ bupstash list-contents id="$ID"
+$ bupstash list-contents -t id="$ID"
# Fetch the snapshot.
$ bupstash get id="$ID" | tar -xf -
```
@@ -276,7 +279,7 @@ $ bupstash put ./multiple-files.txt ./directory
```
# Snapshot a postgres database with pgdump
-$ bupstash put --exec name=dbdump.sql pgdump mydb
+$ bupstash put --exec -t name=dbdump.sql pgdump mydb
```
### Connecting to an ssh server with a specific ssh config.
diff --git a/src/fstx2.rs b/src/fstx2.rs
index 4ab4f19..beae314 100644
--- a/src/fstx2.rs
+++ b/src/fstx2.rs
@@ -799,8 +799,8 @@ mod tests {
let txn = ReadTxn::begin_at(&fs).unwrap();
assert_eq!(txn.read("append").unwrap(), vec![1, 2, 3]);
assert_eq!(txn.read("write").unwrap(), vec![4, 5, 6]);
- assert!(txn.metadata("renamed").is_ok());
- assert!(txn.metadata("some_dir").is_ok());
+ assert!(txn.fs.metadata("renamed").is_ok());
+ assert!(txn.fs.metadata("some_dir").is_ok());
txn.end();
}
}
diff --git a/src/main.rs b/src/main.rs
index 6ae041a..d2b8dbb 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -40,6 +40,7 @@ pub mod xglobset;
pub mod xid;
pub mod xtar;
+use anyhow::Context;
use plmap::PipelineMap;
use std::collections::{BTreeMap, HashMap};
use std::fmt::Write as FmtWrite;
@@ -436,11 +437,7 @@ fn cli_to_serve_process(
let mut proc = match std::process::Command::new(bin)
.args(serve_cmd_args)
- .stderr(if progress.is_hidden() {
- std::process::Stdio::inherit()
- } else {
- std::process::Stdio::piped()
- })
+ .stderr(std::process::Stdio::piped())
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.spawn()
@@ -449,21 +446,39 @@ fn cli_to_serve_process(
Err(err) => anyhow::bail!("error spawning serve command: {}", err),
};
- let stderr_reader = if progress.is_hidden() {
- None
- } else {
+ let stderr_reader = {
let progress = progress.clone();
let proc_stderr = proc.stderr.take().unwrap();
let stderr_reader = std::thread::spawn(move || {
let buf_reader = std::io::BufReader::new(proc_stderr);
for line in buf_reader.lines().flatten() {
- progress.println(&line);
- // Theres a tiny race condition here where we may print an
- // error line twice, I can't see how to fix this unless we
- // rewrite the progress bar library to report if the print happened.
- if progress.is_finished() || progress.is_hidden() {
+ if progress.is_hidden() {
+ /*
+ * Workaround for SSH rogue behavior.
+ * It sets its file descriptors to O_NONBLOCK, and because these
+ * are shared on Unix it would also affect our stderr if we simply
+ * inherited that file descriptor. Instead, we always use `piped` and
+ * simply read and forward the bytes in a thread.
+ *
+ * See also:
+ * https://github.com/andrewchambers/bupstash/issues/378
+ * https://public-inbox.org/git/20190909170403.GB30399@sigill.intra.peff.net/T/
+ * https://lists.nongnu.org/archive/html/bug-cvs/2005-07/msg00008.html
+ * https://www.spinics.net/lists/openssh-unix-dev/msg06047.html
+ * https://github.com/rust-lang/rust/issues/13336
+ * https://github.com/rust-lang/rust/issues/100673
+ */
let _ = writeln!(std::io::stderr(), "{}", line);
+ } else {
+ progress.println(&line);
+
+ // Theres a tiny race condition here where we may print an
+ // error line twice, I can't see how to fix this unless we
+ // rewrite the progress bar library to report if the print happened.
+ if progress.is_finished() || progress.is_hidden() {
+ let _ = writeln!(std::io::stderr(), "{}", line);
+ }
}
}
});
@@ -885,7 +900,7 @@ fn put_main(args: Vec) -> Result<(), anyhow::Error> {
opts.optflag("q", "quiet", "Be quiet, implies --no-progress.");
opts.optflag(
- "e",
+ "",
"exec",
"Treat arguments as a command to run, ensuring it succeeds before committing the item.",
);
@@ -926,6 +941,12 @@ fn put_main(args: Vec) -> Result<(), anyhow::Error> {
or `.no-backup`.",
"FILENAME",
);
+ opts.optmulti(
+ "t",
+ "tag",
+ "Add a tag which the backup will be associated with. May be passed multiple times. No duplicate tag keys allowed.",
+ "KEY=VALUE",
+ );
opts.optflag(
"",
"ignore-permission-errors",
@@ -951,35 +972,25 @@ fn put_main(args: Vec) -> Result<(), anyhow::Error> {
let matches = parse_cli_opts(opts, &args);
- let tag_re = regex::Regex::new(r"^([a-zA-Z0-9\\-_]+)=(.+)$").unwrap();
+ let source_args = matches.free.clone();
+ let tag_re = regex::Regex::new(r"^([a-zA-Z0-9\\-_]+)=(.+)$").unwrap();
let mut tags = BTreeMap::::new();
- let mut source_args = Vec::new();
- {
- let mut collecting_tags = true;
-
- for a in &matches.free {
- if collecting_tags && a == "::" {
- collecting_tags = false;
- continue;
- }
- if collecting_tags {
- match tag_re.captures(a) {
- Some(caps) => {
- let t = &caps[1];
- let v = &caps[2];
- tags.insert(t.to_string(), v.to_string());
- }
- None => {
- collecting_tags = false;
- source_args.push(a.to_string());
- }
- }
- } else {
- source_args.push(a.to_string());
- }
- }
+ for tag in matches.opt_strs("tag") {
+ let Some(caps) = tag_re.captures(&tag) else {
+ anyhow::bail!(
+ "Invalid tag '{tag}'. Tags must match the regex ^([a-zA-Z0-9\\-_]+)=(.+)$"
+ );
+ };
+ let key = &caps[1];
+ let val = &caps[2];
+ // Insert but check for duplicates
+ // TODO switch to try_insert once stable
+ anyhow::ensure!(
+ tags.insert(key.to_string(), val.to_string()).is_none(),
+ "Duplicated tag key '{key}'"
+ );
}
let want_xattrs = matches.opt_present("xattrs");
@@ -1158,7 +1169,8 @@ fn put_main(args: Vec) -> Result<(), anyhow::Error> {
action,
ty,
path.as_os_str().to_string_lossy()
- )?;
+ )
+ .context("failed to write to stderr")?;
Ok(())
}))
} else {
@@ -1288,7 +1300,8 @@ fn put_main(args: Vec) -> Result<(), anyhow::Error> {
&progress,
ServeProcessCliOpts::default(),
protocol::OpenMode::ReadWrite,
- )?;
+ )
+ .context("could not start serve process")?;
let mut serve_out = serve_proc.proc.stdout.as_mut().unwrap();
let mut serve_in = serve_proc.proc.stdin.as_mut().unwrap();
@@ -1314,7 +1327,8 @@ fn put_main(args: Vec) -> Result<(), anyhow::Error> {
threads,
};
- let (id, stats) = client::put(ctx, &mut serve_out, &mut serve_in, tags, data_source)?;
+ let (id, stats) = client::put(ctx, &mut serve_out, &mut serve_in, tags, data_source)
+ .context("failed to backup the data")?;
client::hangup(&mut serve_in)?;
serve_proc.wait()?;
@@ -3133,9 +3147,10 @@ fn main() {
// Support unix style pipelines, don't print an error on EPIPE.
match err.root_cause().downcast_ref::() {
Some(io_error) if io_error.kind() == std::io::ErrorKind::BrokenPipe => {
- std::process::exit(1)
+ // Use distinct exit code here for diagnostic
+ std::process::exit(2)
}
- _ => die(format!("bupstash {}: {}", subcommand, err)),
+ _ => die(format!("bupstash {}: {:?}", subcommand, err)),
}
}
}
diff --git a/src/put.rs b/src/put.rs
index 40557e3..db6c8e5 100644
--- a/src/put.rs
+++ b/src/put.rs
@@ -22,6 +22,7 @@ use super::oplog;
use super::protocol;
use super::rollsum;
use super::sendlog;
+use anyhow::Context;
use plmap::{PipelineMap, ScopedPipelineMap};
use std::borrow::Cow;
use std::collections::BTreeMap;
@@ -590,7 +591,8 @@ impl<'a, 'b> plmap::Mapper, anyhow::Err
&mut self,
file_batch: Result, anyhow::Error>,
) -> Self::Out {
- self.process_batch(file_batch?)
+ self.process_batch(file_batch.context("received `Err` batch for processing")?)
+ .context("failed to process a batch")
}
}
@@ -672,7 +674,8 @@ impl<'a, 'b> BatchFileProcessor<'a, 'b> {
.unwrap()
.lock()
.unwrap()
- .stat_cache_lookup_and_update(&stat_cache_key.unwrap())?
+ .stat_cache_lookup_and_update(&stat_cache_key.unwrap())
+ .context("failed to update stat cache")?
} else {
None
};
@@ -682,7 +685,10 @@ impl<'a, 'b> BatchFileProcessor<'a, 'b> {
let mut uncompressed_data_size = 0;
for (i, (ref path, ref mut ent)) in file_batch.iter_mut().enumerate() {
uncompressed_data_size += ent.size.0;
- self.log_file_action('~', ent.type_display_char(), path)?;
+ self.log_file_action('~', ent.type_display_char(), path)
+ .with_context(|| {
+ format!("failed to log file action for {}", path.display())
+ })?;
ent.data_hash = cache_entry.hashes[i];
ent.data_cursor = cache_entry.data_cursors[i];
}
@@ -707,16 +713,23 @@ impl<'a, 'b> BatchFileProcessor<'a, 'b> {
let file_batch_len = file_batch.len();
for (i, (ref path, ref mut ent)) in file_batch.iter_mut().enumerate() {
- self.log_file_action('+', ent.type_display_char(), path)?;
+ self.log_file_action('+', ent.type_display_char(), path)
+ .context("failed to log file action")?;
let ent_data_chunk_start_idx = data_addresses.len() as u64;
let ent_start_byte_offset = self.data_chunker.buffered_count() as u64;
if ent.is_file() {
match file_opener.next_file().unwrap() {
- (_, Ok(f)) => {
+ (ent_path, Ok(f)) => {
let stat_size = ent.size.0;
- self.chunk_and_hash_file_data(f, &mut data_addresses, ent)?;
+ self.chunk_and_hash_file_data(f, &mut data_addresses, ent)
+ .with_context(|| {
+ format!(
+ "failed to process file data of {}",
+ ent_path.display()
+ )
+ })?;
if stat_size != ent.size.0 {
// The files size changed, don't cache
// this result in the stat cache.
@@ -750,7 +763,9 @@ impl<'a, 'b> BatchFileProcessor<'a, 'b> {
crypto::keyed_content_address(&chunk, &self.ctx.data_hash_key);
let chunk = compression::compress(self.ctx.compression, chunk);
let chunk = self.ctx.data_ectx.encrypt_data(chunk);
- self.sender.write_chunk(&address, chunk)?;
+ self.sender
+ .write_chunk(&address, chunk)
+ .context("failed to write chunk")?;
data_addresses.push(address);
self.ctx.progress.inc(chunk_len);
}
@@ -793,7 +808,8 @@ impl<'a, 'b> BatchFileProcessor<'a, 'b> {
data_cursors: Cow::Borrowed(&data_cursors),
hashes: Cow::Borrowed(&content_hashes),
},
- )?;
+ )
+ .context("failed to add stat cache data")?;
}
data_addresses
diff --git a/src/repository.rs b/src/repository.rs
index 2e2de2f..3bceb7f 100644
--- a/src/repository.rs
+++ b/src/repository.rs
@@ -274,7 +274,7 @@ impl Repo {
schema_version = txn.read_string("meta/schema_version")?;
if schema_version != CURRENT_SCHEMA_VERSION {
anyhow::bail!(
- "the current version of bupstash expects repository schema version {}, got {}",
+ "the current version of bupstash expects repository schema version {}, got {} (if the latter number is greater than the former, this means that you need to update bupstash)",
CURRENT_SCHEMA_VERSION,
schema_version
);