diff --git a/.github/auto-assignees.yml b/.github/auto-assignees.yml index d98cefe57c..2380b5c9e7 100644 --- a/.github/auto-assignees.yml +++ b/.github/auto-assignees.yml @@ -16,6 +16,7 @@ reviewers: - qiuming-best - shubham-pampattiwar - Lyndon-Li + - anshulahuja98 tech-writer: - sseago diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 3e01a97d7f..db634e8320 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -48,7 +48,10 @@ jobs: version: latest - name: Build - run: make local + run: | + make local + # Clean go cache to ease the build environment storage pressure. + go clean -modcache -cache - name: Test run: make test @@ -73,7 +76,7 @@ jobs: run: | sudo swapoff -a sudo rm -f /mnt/swapfile - docker image prune -a --force + docker system prune -a --force # Build and push Velero image to docker registry docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASSWORD }} diff --git a/Dockerfile b/Dockerfile index 167dc5f040..43994141c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,8 @@ RUN mkdir -p /output/usr/bin && \ go build -o /output/${BIN} \ -ldflags "${LDFLAGS}" ${PKG}/cmd/${BIN} && \ go build -o /output/velero-helper \ - -ldflags "${LDFLAGS}" ${PKG}/cmd/velero-helper + -ldflags "${LDFLAGS}" ${PKG}/cmd/velero-helper && \ + go clean -modcache -cache # Restic binary build section FROM --platform=$BUILDPLATFORM golang:1.20.7-bullseye as restic-builder @@ -65,10 +66,11 @@ COPY . /go/src/github.com/vmware-tanzu/velero RUN mkdir -p /output/usr/bin && \ export GOARM=$(echo "${GOARM}" | cut -c2-) && \ - /go/src/github.com/vmware-tanzu/velero/hack/build-restic.sh + /go/src/github.com/vmware-tanzu/velero/hack/build-restic.sh && \ + go clean -modcache -cache # Velero image packing section -FROM gcr.io/distroless/base-nossl-debian11@sha256:f10e1fbf558c630a4b74a987e6c754d45bf59f9ddcefce090f6b111925996767 +FROM paketobuildpacks/run-jammy-tiny:latest LABEL maintainer="Xun Jiang " @@ -76,5 +78,5 @@ COPY --from=velero-builder /output / COPY --from=restic-builder /output / -USER nonroot:nonroot +USER cnb:cnb diff --git a/MAINTAINERS.md b/MAINTAINERS.md index d9f920683f..14abb441f5 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -4,16 +4,16 @@ ## Maintainers -| Maintainer | GitHub ID | Affiliation | -|---------------------|---------------------------------------------------------------|-------------------------------------------| -| Dave Smith-Uchida | [dsu-igeek](https://github.com/dsu-igeek) | [Kasten](https://github.com/kastenhq/) | -| Scott Seago | [sseago](https://github.com/sseago) | [OpenShift](https://github.com/openshift) | -| Daniel Jiang | [reasonerjt](https://github.com/reasonerjt) | [VMware](https://www.github.com/vmware/) | -| Wenkai Yin | [ywk253100](https://github.com/ywk253100) | [VMware](https://www.github.com/vmware/) | -| Xun Jiang | [blackpiglet](https://github.com/blackpiglet) | [VMware](https://www.github.com/vmware/) | -| Ming Qiu | [qiuming-best](https://github.com/qiuming-best) | [VMware](https://www.github.com/vmware/) | -| Shubham Pampattiwar | [shubham-pampattiwar](https://github.com/shubham-pampattiwar) | [OpenShift](https://github.com/openshift) | -| Yonghui Li | [Lyndon-Li](https://github.com/Lyndon-Li) | [VMware](https://www.github.com/vmware/) | +| Maintainer | GitHub ID | Affiliation | +|---------------------|---------------------------------------------------------------|--------------------------------------------------| +| Scott Seago | [sseago](https://github.com/sseago) | [OpenShift](https://github.com/openshift) | +| Daniel Jiang | [reasonerjt](https://github.com/reasonerjt) | [VMware](https://www.github.com/vmware/) | +| Wenkai Yin | [ywk253100](https://github.com/ywk253100) | [VMware](https://www.github.com/vmware/) | +| Xun Jiang | [blackpiglet](https://github.com/blackpiglet) | [VMware](https://www.github.com/vmware/) | +| Ming Qiu | [qiuming-best](https://github.com/qiuming-best) | [VMware](https://www.github.com/vmware/) | +| Shubham Pampattiwar | [shubham-pampattiwar](https://github.com/shubham-pampattiwar) | [OpenShift](https://github.com/openshift) | +| Yonghui Li | [Lyndon-Li](https://github.com/Lyndon-Li) | [VMware](https://www.github.com/vmware/) | +| Anshul Ahuja | [anshulahuja98](https://github.com/anshulahuja98) | [Microsoft Azure](https://www.github.com/azure/) | ## Emeritus Maintainers * Adnan Abdulhussein ([prydonius](https://github.com/prydonius)) @@ -25,12 +25,12 @@ * Carlisia Thompson ([carlisia](https://github.com/carlisia)) * Bridget McErlean ([zubron](https://github.com/zubron)) * JenTing Hsiao ([jenting](https://github.com/jenting)) - +* Dave Smith-Uchida ([dsu-igeek](https://github.com/dsu-igeek)) + ## Velero Contributors & Stakeholders | Feature Area | Lead | |------------------------|:------------------------------------------------------------------------------------:| -| Architect | Dave Smith-Uchida [dsu-igeek](https://github.com/dsu-igeek) | | Technical Lead | Daniel Jiang [reasonerjt](https://github.com/reasonerjt) | | Kubernetes CSI Liaison | | | Deployment | | diff --git a/changelogs/unreleased/5211-cleverhu b/changelogs/unreleased/5211-cleverhu new file mode 100644 index 0000000000..1ed39117cf --- /dev/null +++ b/changelogs/unreleased/5211-cleverhu @@ -0,0 +1 @@ +fix run preHook and postHook on completed pods \ No newline at end of file diff --git a/changelogs/unreleased/6475-nilesh-akhade b/changelogs/unreleased/6475-nilesh-akhade new file mode 100644 index 0000000000..3db273f29a --- /dev/null +++ b/changelogs/unreleased/6475-nilesh-akhade @@ -0,0 +1 @@ +Add `orLabelSelectors` for backup, restore commands diff --git a/changelogs/unreleased/6637-Lyndon-Li b/changelogs/unreleased/6637-Lyndon-Li new file mode 100644 index 0000000000..b816b976bc --- /dev/null +++ b/changelogs/unreleased/6637-Lyndon-Li @@ -0,0 +1 @@ +Add CSI snapshot data movement doc \ No newline at end of file diff --git a/changelogs/unreleased/6680-dzaninovic b/changelogs/unreleased/6680-dzaninovic new file mode 100644 index 0000000000..b4d735d145 --- /dev/null +++ b/changelogs/unreleased/6680-dzaninovic @@ -0,0 +1 @@ +Add support for block volumes with Kopia \ No newline at end of file diff --git a/changelogs/unreleased/6686-ywk253100 b/changelogs/unreleased/6686-ywk253100 new file mode 100644 index 0000000000..d2a79be6db --- /dev/null +++ b/changelogs/unreleased/6686-ywk253100 @@ -0,0 +1 @@ +Make Kopia support Azure AD \ No newline at end of file diff --git a/changelogs/unreleased/6712-kaovilai b/changelogs/unreleased/6712-kaovilai new file mode 100644 index 0000000000..e597187560 --- /dev/null +++ b/changelogs/unreleased/6712-kaovilai @@ -0,0 +1 @@ +Kubernetes 1.27 new job label batch.kubernetes.io/controller-uid are deleted during restore per https://github.com/kubernetes/kubernetes/pull/114930 \ No newline at end of file diff --git a/changelogs/unreleased/6715-nilesh-akhade b/changelogs/unreleased/6715-nilesh-akhade new file mode 100644 index 0000000000..756432ad12 --- /dev/null +++ b/changelogs/unreleased/6715-nilesh-akhade @@ -0,0 +1 @@ +Remove schedule-related metrics on schedule delete \ No newline at end of file diff --git a/changelogs/unreleased/6751-Lyndon-Li b/changelogs/unreleased/6751-Lyndon-Li new file mode 100644 index 0000000000..6f614cb681 --- /dev/null +++ b/changelogs/unreleased/6751-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #6647, add the --default-snapshot-move-data parameter to Velero install, so that users don't need to specify --snapshot-move-data per backup when they want to move snapshot data for all backups \ No newline at end of file diff --git a/changelogs/unreleased/6757-Lyndon-Li b/changelogs/unreleased/6757-Lyndon-Li new file mode 100644 index 0000000000..1e095ee673 --- /dev/null +++ b/changelogs/unreleased/6757-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #6753, remove the check for read-only BSL in restore async operation controller since Velero cannot fully support read-only mode BSL in restore at present \ No newline at end of file diff --git a/changelogs/unreleased/6760-blackpiglet b/changelogs/unreleased/6760-blackpiglet new file mode 100644 index 0000000000..db9b5a118c --- /dev/null +++ b/changelogs/unreleased/6760-blackpiglet @@ -0,0 +1 @@ +Fix #6752: add namespace exclude check. \ No newline at end of file diff --git a/changelogs/unreleased/6770-ywk253100 b/changelogs/unreleased/6770-ywk253100 new file mode 100644 index 0000000000..6e55ffcd49 --- /dev/null +++ b/changelogs/unreleased/6770-ywk253100 @@ -0,0 +1 @@ +Update restore controller logic for restore deletion \ No newline at end of file diff --git a/changelogs/unreleased/6827-Lyndon-Li b/changelogs/unreleased/6827-Lyndon-Li new file mode 100644 index 0000000000..1aac387e08 --- /dev/null +++ b/changelogs/unreleased/6827-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #6786, always delete VSC regardless of the deletion policy \ No newline at end of file diff --git a/changelogs/unreleased/6833-Lyndon-Li b/changelogs/unreleased/6833-Lyndon-Li new file mode 100644 index 0000000000..9ec3673147 --- /dev/null +++ b/changelogs/unreleased/6833-Lyndon-Li @@ -0,0 +1 @@ +Bump kopia to v0.14 \ No newline at end of file diff --git a/changelogs/unreleased/6838-yanggangtony b/changelogs/unreleased/6838-yanggangtony new file mode 100644 index 0000000000..4f48f8fefb --- /dev/null +++ b/changelogs/unreleased/6838-yanggangtony @@ -0,0 +1 @@ +change the metrics backup_attempt_total default value to 1. \ No newline at end of file diff --git a/changelogs/unreleased/6872-Lyndon-Li b/changelogs/unreleased/6872-Lyndon-Li new file mode 100644 index 0000000000..e5db5e6b42 --- /dev/null +++ b/changelogs/unreleased/6872-Lyndon-Li @@ -0,0 +1 @@ +Fix #6861. Only Restic path requires repoIdentifier, so for non-restic path, set the repoIdentifier fields as empty in PVB and PVR and also remove the RepoIdentifier column in the get output of PVBs and PVRs \ No newline at end of file diff --git a/changelogs/unreleased/6875-Lyndon-Li b/changelogs/unreleased/6875-Lyndon-Li new file mode 100644 index 0000000000..8d11bca2c9 --- /dev/null +++ b/changelogs/unreleased/6875-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #6859, move plugin depending podvolume functions to util pkg, so as to remove the dependencies to unnecessary repository packages like kopia, azure, etc. \ No newline at end of file diff --git a/changelogs/unreleased/6883-ywk253100 b/changelogs/unreleased/6883-ywk253100 new file mode 100644 index 0000000000..bc8d80b92e --- /dev/null +++ b/changelogs/unreleased/6883-ywk253100 @@ -0,0 +1 @@ +Replace the base image with paketobuildpacks image \ No newline at end of file diff --git a/changelogs/unreleased/6885-Lyndon-Li b/changelogs/unreleased/6885-Lyndon-Li new file mode 100644 index 0000000000..f879897c0e --- /dev/null +++ b/changelogs/unreleased/6885-Lyndon-Li @@ -0,0 +1 @@ +Set ParallelUploadAboveSize as MaxInt64 and flush repo after setting up policy so that policy is retrieved correctly by TreeForSource \ No newline at end of file diff --git a/cmd/velero-helper/velero-helper.go b/cmd/velero-helper/velero-helper.go index 5991531a60..fc8dd25125 100644 --- a/cmd/velero-helper/velero-helper.go +++ b/cmd/velero-helper/velero-helper.go @@ -1,3 +1,19 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import ( diff --git a/config/crd/v1/bases/velero.io_podvolumebackups.yaml b/config/crd/v1/bases/velero.io_podvolumebackups.yaml index aa582b1d2d..3922fdc076 100644 --- a/config/crd/v1/bases/velero.io_podvolumebackups.yaml +++ b/config/crd/v1/bases/velero.io_podvolumebackups.yaml @@ -35,10 +35,6 @@ spec: jsonPath: .spec.volume name: Volume type: string - - description: Backup repository identifier for this backup - jsonPath: .spec.repoIdentifier - name: Repository ID - type: string - description: The type of the uploader to handle data transfer jsonPath: .spec.uploaderType name: Uploader Type diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index a375900ac3..12bb1e42d5 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -34,7 +34,7 @@ var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YKs\xe3\xb8\x11\xbe\xebWt\xed\x1e|YQ3\x9bKJ\x97\x94FN\xaa\xa6\xe2\x89]#ǹ\xe4\xb0\x10Д\xb0\x06\x01\x06\x0fi\x94T\xfe{\xaa\x01\xf0!\x92\xb2\xe4\xa9$\x8b\x8bM\xb2\xd1\xe8\xfe\xfa\r\xcd\xe7\xf3\x19\xab\xe5\vZ'\x8d^\x02\xab%~\xf3\xa8\xe9\xc9\x15\xaf\xbfw\x854\x8b\xc3\xc7٫\xd4b\t\xeb༩\xbe\xa23\xc1r\xbc\xc7Rj\xe9\xa5ѳ\n=\x13̳\xe5\f\x80im<\xa3\u05ce\x1e\x01\xb8\xd1\xde\x1a\xa5\xd0\xcew\xa8\x8bװ\xc5m\x90J\xa0\x8d̛\xa3\x0f\x1f\x8a\x8f?\x17\x1ff\x00\x9aU\xb8\x84-㯡v\xdeX\xb6CexbY\x1cP\xa15\x8543W#\xa7\x13vքz\t݇\xc4!\x9f\x9e$\xff\x14\x99m\x12\xb3\x87\xcc,~W\xd2\xf9?_\xa6y\x90\xceG\xbaZ\x05\xcb\xd4%\xb1\"\x89\xdb\x1b\xeb\xff\xd2\x1d=\x87\xadS\xe9\x8bԻ\xa0\x98\xbd\xb0}\x06ษq\tqw\xcd8\x8a\x19@\x86&r\x9b\x03\x13\"\x82\xcdԓ\x95ڣ]\x1b\x15*ݞ%\xd0q+k\x1f\xc1L\xba@V\x06\x1am\xc0y\xe6\x83\x03\x17\xf8\x1e\x98\x83ՁIŶ\n\x17\x7fլ\xf9?\xf2\x03\xf8\xd5\x19\xfd\xc4\xfc~\tE\xdaU\xd4{暯\xc9FO\xbd7\xfeD\n8o\xa5\xdeM\x89\xf4\xc0\x9c\x7faJ\x8a(ɳ\xac\x10\xa4\x03\xbfGP\xccy\xf0\xf4\x82\x9e\x12B@\x10!4\b\xc1\x91\xb9|\x0e\xc0!q\x89\x18MK\xaaFg\x9d\x89M\xa2\xc0ˀK\x92\x9f\xded\xe9{l\x1b\xff.\xb8Ŗ\xa5\xf3\xac\xaa\xcf\xf8\xaevx\x89\xd9\x19\x14\xf7X\xb2\xa0|_U\xb2\x92\xea\xfb\xe5\xb9Z5\xf2B\xa4]g'ޟ\xbdK\xa7n\x8dQ\xc8\x12\x97Du\xf8\x98\xbc\x90\xef\xb1b\xcbLljԫ\xa7\xcf/\xbfۜ\xbd\x86)G\x1a\x04\x05\x19\x8e\xf5l\xb3G\x8b\xf0\x12\xe3/\xd9\xcde\xd5Z\x9e\x00f\xfb+r\xdf\x19\xb1\xb6\xa6F\xebe\x13,i\xf5rQ\xef\xed@\xa6;\x12;Q\x81\xa0$\x84ɏr\xbc\xa0Ț\x82)\xc1\xef\xa5\x03\x8b\xb5E\x87\xda\xf7\xe1m\x05+\x81\xe9,^\x01\x1b\xb4Ćb9(A\xb9\xeb\x80փEnvZ\xfe\xb3\xe5\xed\xc0\x9b\xec\xbc\x1e\x9d\x1f\xf0\x8c\xf1\xa9\x99\"W\r\xf8\x130-\xa0b'\xb0H\xa7@\xd0=~\x91\xc4\x15\xf0\x85\xfc]\xea\xd2,a\xef}햋\xc5N\xfa&\asSUAK\x7fZ\xc4t*\xb7\xc1\x1b\xeb\x16\x02\x0f\xa8\x16N\xee\xe6\xcc\xf2\xbd\xf4\xc8}\xb0\xb8`\xb5\x9cG\xd1uJ\x9a\x95\xf8\xd1\xe6\xac\xed\xee\xced\x1dEmZ1k\xbea\x01ʘ\xc9\v\xd2֤E\a4\xbd\"t\xbe\xfeq\xf3\f\xcd\xd1\xd1\x18C\xf4#\xee\xddFי\x80\x00\x93\xbaD\x9b\x8cXZSE\x9e\xa8Em\xa4\xf6\xf1\x81+\x89z\b\xbf\v\xdbJz\xb2\xfb?\x02:O\xb6*`\x1d\v\x13l\x11B\x1d㾀\xcf\x1a֬B\xb5f\x0e\xff\xe7\x06 \xa4ݜ\x80\xbd\xcd\x04\xfd\x9a:$N\xa8\xf5>4\xb5\xf0\x82\xbd&\xa3xS#?\x8b\x1f\x81NZ\xf2p\xcf<Ƹ\x18\xe0\x9aC\xfcr1m\xd6tp\xd3b\x9c\xa3s_\x8c\xc0ᗁȫ\x96\xf0L\xc6\x1am%],\x8bP\x1a;\xac\x18\xac\xcd\xc0\xfd\xd5d\xaab\xf4\ru\xa8Ƃ\xcc\xe1+2\xf1\xa8\xd5\xe9§\xbfY\xe9\xc7\a]0$\xad$\xe2\xe6\xa4\xf9\x13Zi\xc4\x15\xe5?\r\xc8[\b\xf6\xe6\betk\xedՉr\x90;i>ζ\xcdZ=}n2o\n\xa0\x1co\x19\xab\x02V9rM\t\x1f@HG\r\x80\x8bL\xc7`\xe9\xa0b\x83\xb0\x04oû\xd4\xe7F\x97r7V\xba\xdf\xd3\\\xf2\x98+\xac\aȭ\xe3I\x94\x9a\xc8;jk\x0eR\xa0\x9dS|\xc8R\xf2,I\xb0\xa9r\x95\x12\x95pcM/DYTŢ\xa0\xa8f\xea\x8a\r\xd7-a쀙\xd4Ƀ;\x061\xd9\xd8*\x97T\xedQ\x8b\xb6\x1b9\x93\xc6Ĭ\xe5P\xc0Q\xfa}J\x87j*\xee\xe0\xcdأ\xf5\x8a\xa7\xa9\xd7\x03ٟ\xf7H\x94\xa9\x80\"8\xe4\x16}\xf46T\xe4>\xe4J\x05\xc0\x97\xe0bB\x1d\xe6\x89f\xc5F\xad\xd9\xfd\x8a\xa71\xd0p\u0378\xb9\x85\xb9.\xf2\x1d\xb5\u038d\xc0\x16K\xb4\xa8\xfddR\xa7\x01\xc4j\xf4\x18\xf3\xba0\xdcQJ\xe7X{\xb70\a\xb4\a\x89\xc7\xc5\xd1\xd8W\xa9ws\x02|\x9e#h\x11NJŏ\xf1\xcf\x05\x95\x9f\x1f\xef\x1f\x97\xb0\x12\x02\x8cߣ%\xab\x95A5\x8e\xd6\xebo~\x8a5\xf6'\bR\xfc\xe1\xee{p1u\x8a\x9c\x1b\xb0\xd9D\xef?Q\xa3\x16\x85\"\x886\xc9*\xc6\x02UJ2v\x95\xad\x99r͔#Nu\x98\xfdE\x89\x89*\xc8TF}\xc5q2}#\xcc\x00\xbe\xcd;C\xcd+V\xcf\x135\xf3\xa6\x92|6\xd46\xb6\xc1W\"\xb2i\xbb\xa5\x16\x92S\xdbv\x1eI\xcd8\"κ\xf3\t\x18\x86\xfd\xfa\xa5\xfc1\rSR7W\xcf+\x12?\xf6i\xbb!.%\xb3\\\x11\x1dzj\xb7\x1ch\xa4\x8a\xc9\xec\x18\xe7\x98B\xb8њb\xd7\x1b`mb\xbcsÊ\xf0\xce|\xb2\r\xfc\x15'\x80\x1f\xa9\xf2)\x126\x18\xa7m$Kp\x18S\xf551\xe0zDp\xb6F{\x8b,\xeb\x15\x11\xb6E\x95\xc1z\x05۠\x85\xc2F\xa2\xe3\x1e5\xcd\x13\xb2\x14?\xfcf3\x93b\xce\xd3\b\x84\xe2+\x1e\xe4\xf8Nh\x8c\xee\xc3hG\x13\xf8m8\xd0\xc3/\xcdh\xbd\xb0\x99\xec\x97\t0J\xa9\xa8s\x9c\xc8\x13]\xc70\xbe\xbd\xfc\xb4y\xb8s\xb1\xe1G\xed\xa7\x9a\xc4#Z\x8c\xf3\x15\n\xea\xf9M\xbe\xc5\bΣ\x9dp\x80\xd6z\xd1栌\xde\r\x02'\xad|\xa7A\xfd\\r(cA\xa0\xa7Ҥw\xc0\xf7Lﰻ\xb3\xca\xf2\xbf-)\xb9\xcf\xc0g:\x0f\x91\xfa\x92{\xdcd\xd1g9\xd5ԏ\xee\x8b;\xe2\xe9\xbb\xe2F\xfaƲ\x17\x87\xa2+\xb8\x8f\xe8\x9b*M\xa0\xce}w\x7fܭ\xef\x1f\x86Ǘ\xd37 \xf1ޛ\xf37nA\xe0\xc8\\w\x87\xfe\xdb\xe1PQ\xb7z\xb5\x05\xfe\x92\xa8\xd2ec\xde\x02lk\x82\x7f+2\xef\xa6\x1c:\xff8\xf0\x1e\x19\xe3O\x1eך\f\xa2i,\u0083\xa5\xc1\xb3\xbbC\x8bIa\xaa\xb6\xdc~\x19\xb5\x1a\xfc2\xd3\xff6\xfe\xdd\xe6\x06\xbd&k\xed\xe8e\xaa\x97=\xbbf\x90\xfbo¶\xbdW^¿\xfe=\xfbO\x00\x00\x00\xff\xff\x80.\x12\xd3P\x1c\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4\x96M\x93\xdb6\x0f\xc7\xef\xfe\x14\x98y\x0e\xb9<\x92\xb3\xed\xa5\xa3[\xb3\xc9a\xa7mƳ\x9bɝ&a\x8bY\x8ad\x01\xd0[\xb7\xd3\xef\xde!)\xf9E\xb67\xdbCy\x13\t\x02\x7f\xfe@\x80j\x9af\xa1\xa2\xfd\x8a\xc46\xf8\x0eT\xb4\xf8\x87\xa0\xcf_\xdc>\xffĭ\r\xcb\xdd\xdd\xe2\xd9z\xd3\xc1}b\t\xc3#rH\xa4\xf1#n\xac\xb7b\x83_\f(\xca(Q\xdd\x02@y\x1fD\xe5iΟ\x00:x\xa1\xe0\x1cR\xb3E\xdf>\xa75\xae\x93u\x06\xa98\x9fB\xef\u07b7w?\xb4\xef\x17\x00^\r\u0601A\x87\x82k\xa5\x9fS$\xfc=!\v\xb7;tH\xa1\xb5a\xc1\x11u\xf6\xbf\xa5\x90b\aDž\xba\x7f\x8c]u\x7f,\xae>\x14W\x8f\xd5UYu\x96\xe5\x97[\x16\xbf\xda\xd1*\xbaD\xca]\x17T\f\xd8\xfamr\x8a\xae\x9a,\x00X\x87\x88\x1d|β\xa2\xd2h\x16\x00㱋\xcc\x06\x941\x05\xa4r+\xb2^\x90\xee\x83K\xc3\x04\xb0\x01\x83\xac\xc9F)\xa0\xbe\xf4X\x8e\ba\x03\xd2#\xd4p \x01\xd68*0e\x1f\xc07\x0e~\xa5\xa4\xef\xa0ͼ\xdaj\x9a\x85\x8c\x06\x15\xf5\x87\xf9\xb4\xec\xb3`\x16\xb2~{K\x02\x8b\x92ē\x88\x12\xd7\x06\x0ft\xc2\xf7\\@\xb1oc\xaf\xf8<\xfaSY\xb8\x15\xb9\xda\xec\xee*i\xdd㠺\xd16D\xf4?\xaf\x1e\xbe\xfe\xf8t6\r\xe7Z\xaf\xa4\x16,\x83\x9a\x94fp\x95\x1a\x04\x8f\x10\b\x86@\x13Un\x0fN#\x85\x88$v\xbaZu\x9c\x14\xcf\xc9\xecL»\xac\xb2Z\x81\xc9U\x83\\\xa0\x8d\x97\x00\xcdx\xb0\n\xd32\x10FBF_\xeb\xe8\xcc1d#\xe5!\xac\xbf\xa1\x96\x16\x9e\x90\xb2\x1b\xe0>$gr\xb1\xed\x90\x04\bu\xd8z\xfb\xe7\xc17\xe7s\xe6\xa0N\xc91?\xd3(\x97\xce+\a;\xe5\x12\xfe\x1f\x9470\xa8=\x10\xe6(\x90\xfc\x89\xbfb\xc2-\xfc\x961Y\xbf\t\x1d\xf4\"\x91\xbb\xe5rkej\x1a:\fC\xf2V\xf6\xcbR\xffv\x9d$\x10/\r\xee\xd0-\xd9n\x1bE\xba\xb7\x82Z\x12\xe1RE\xdb\x14\xe9\xbe4\x8ev0\xff\xa3\xb1\xcd\xf0\xbb3\xad\x17\x17\xa4\x8eR\xe8\xafd \x97yM{\xddZOq\x04\x9d\xa72\x9d\xc7OO_`\n]\x921\xa7_\xb8\x1f7\xf21\x05\x19\x98\xf5\x1b\xa4\x9a\xc4\r\x85\xa1\xf8Dob\xb0^ʇv\x16\xfd\x1c?\xa7\xf5`\x85\xa7+\x99s\xd5\xc2}餹\xa8S4Jд\xf0\xe0\xe1^\r\xe8\xee\x15\xe3\x7f\x9e\x80L\x9a\x9b\f\xf6m)8}\x04\xe6ƕ\xda\xc9\xc2Ծo\xe4\xebJ\xd1>E\xd49\x83\x19b\xdem7V\x97\xf2\x80M x\xe9\xad\ue9e2\x9d\xd1=\x14x{\xb6p\xbd\xa0\xf38\xb6\xc9\xf9\xca\xcd\xc3Cɝ%\x9c\xdd\xc2\x06.z\xee\xeb\\J3\xfc\x97dj'\x1e\xd9\xe8D\x84^N\xfa\xb3\xba\xb6\xe9\xad,\x90(\xd0\xc5\xecLԧbT^ze=\x83\xf2\xfbq#H\xaf\x04^\x90r\x19\xe8\x90r\x9fA\x03&]\xf0\x1b\xb1\x9c\xbe%\x91\x82F\xe6\xf6\xc2\xce\n\x0eW4\xbd\x92\x9d<|rN\xad\x1dv \x94\xf0Ff\x15\x91\xda\xcf\xd6ʛ\xf5\x1d\x04\xabls-\a\x87w\xfa\xbbI(\xb8}\x1a.#5\xf0\x19_\xae\xcc>\xf8\x15\x85-!ϯ|^\\Uz\x87\x9f\x817P\xbaz)/&9\xf7;sB\x91%\x90ڞr\xe5\xb4>\xf4\xef\x0e\xfe\xfa{\xf1O\x00\x00\x00\xff\xff\x045\f\xc6i\n\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4VM\x93\xdb6\f\xbd\xfbW`\xa6\x87\xb43\x91\x9c\xb4\x97\x8eo\xad\x93\xc3N\xd24\xb3N\xf7NS\xb0\xc4.E\xb2\x04\xe8\xcd\xf6\xd7w@J\xfe\x94\xbd\xdeCu\x13\t\x82\x8f\x0f\x0f\x8f\xac\xaaj\xa6\x82y\xc0Hƻ\x05\xa8`\xf0;\xa3\x93?\xaa\x1f\x7f\xa5\xda\xf8\xf9\xf6\xfd\xecѸf\x01\xcbD\xec\xfb{$\x9f\xa2\xc6\x0f\xb81ΰ\xf1n\xd6#\xabF\xb1Z\xcc\x00\x94s\x9e\x95\f\x93\xfc\x02h\xef8zk1V-\xba\xfa1\xadq\x9d\x8cm0\xe6\xe4\xe3\xd6\xdbw\xf5\xfb\x9f\xebw3\x00\xa7z\\@㟜\xf5\xaa\x89\xf8OBb\xaa\xb7h1\xfa\xda\xf8\x19\x05Ԓ\xbb\x8d>\x85\x05\xec'\xca\xdaa߂\xf9Ð澤\xc93\xd6\x10\x7f\x9a\x9a\xfdl\x86\x88`ST\xf6\x1cD\x9e$\xe3\xdadU<\x9b\x9e\x01\x90\xf6\x01\x17\xf0E`\x04\xa5\xb1\x99\x01\fG̰\xaa\xe1t\xdb\xf7%\x95\xee\xb0W\x05/\x80\x0f\xe8~\xfbz\xf7\xf0\xcb\xeah\x18\xa0A\xd2\xd1\x04\xceD\x9d`\x06C\xa0`@\x00\xecw\xa0@9P\x91\xcdFi\x86M\xf4=\xac\x95~La\x97\x15\xc0\xaf\xffF\xcd@\xec\xa3j\xf1-P\xd2\x1d(\xc9WB\xc1\xfa\x166\xc6b\xbd[\x14\xa2\x0f\x18ٌ,\x97\xef@C\a\xa3'\xc0\xdf\xc8\xd9J\x144\"\x1e$\xe0\x0eG~\xb0\x19\xe8\x00\xbf\x01\xee\fA\xc4\x10\x91\xd0\x159\x1d%\x06\tRn8A\r+\x8c\x92\x06\xa8\xf3\xc96\xa2\xb9-F\x86\x88ڷ\xce\xfc\xbb\xcbM\u0090lj\x15\x8fr\xd8\x7f\xc61F\xa7,l\x95M\xf8\x16\x94k\xa0W\xcf\x101\xf3\x94\xdcA\xbe\x1cB5\xfc\xe1#\x82q\x1b\xbf\x80\x8e9\xd0b>o\r\x8f\xbd\xa3}\xdf'g\xf8y\x9e\xdb\xc0\xac\x13\xfbH\xf3\x06\xb7h\xe7d\xdaJE\xdd\x19F\xcd)\xe2\\\x05Se\xe8.\xf7O\xdd7?ġ\xdb\xe8\xcd\x11V~\x16\x99\x11G\xe3ڃ\x89\xac\xf9+\x15\x10\xd5\x17\xc1\x94\xa5\xe5\x14{\xa2eHع\xff\xb8\xfa\x06\xe3ֹ\x18\xa7\xec\x17\xe5\xec\x16Ҿ\x04B\x98q\x1b\x8c\xa5\x88Yy\x92\x13]\x13\xbcq\x9c\x7f\xb45\xe8N駴\xee\r\xd3(f\xa9U\r\xcbl(\xb0FH\xa1Q\x8cM\rw\x0e\x96\xaaG\xbbT\x84\xff{\x01\x84i\xaa\x84\xd8\xdbJp腧\xc1\x85\xb5\x83\x89\xd1\xc9.\xd4\xeb\xa4\xd5W\x01\xb5TO\b\x94\x95fctn\r\xd8\xf8\bj\xdf\xf9\x03\x81\xf5Q\xe6\xe9\xce\xcd\xe0Tl\x91OGO\xb0|\xcbA\xb2\xfdS\xa7\x8e\x8d\xe6G\xac\xdbZ\xbc\x82\x06 \xc5=~\xaa\xcf2^\xc6\x00\x93\xea\x9dD2\x8aXh\x10^\xc5\nĤ\x0e1\x9do-\x1f\xba\xd4OoP\xc1\xef\x19\xf3g\xdf^\x9d_z\xc7\"\xf7\xabA\x0fަ\x1eWN\x05\xea\xfc\v\xb1w\x8c\xfd\x9f\x01c\xb91\xaf\x86\x8e\x17\xef\ue5ba\x12\x98\xec\xc5}\xefQ\xfc\x1e/\x9ft\b\xb8)\xcb\r\x98\x86ț\x0e\xba\\ݽ\x86\xc2\v\xe1W\x8bt\xa1m\xc7/_\xcf/kP.\xf8Q\x83\xb2\xa4\xdcY\b\x9f\xd2\x1a\xa3CF\xda\xdb\xe7\x93\xe1n2#\xc0Sgt\x97\x17f\x01\x8b3\x13ym\xb2Ͻ\x1e\xbe\xf4\xbd\x898\xd1DUn\xae\x89a\x01\x7f6|\xc1\xad.mP\r\x0er\x93\xe3\xb1\xe2D\xaf\xf0\xbc\x1c?R\xadS\x8c\xe8xȒ\xdf\x00\xa7\vn5\xbd\xd1)\xfe\xba\xff\xfc\x82\xf3}\xd8G\xe6Ǭ2\xae\xa0\t\x11+2\xad\xbc\\dN\xbc/{\xd29\x19\xe5;~I\x1d\x135YQ\xfc\x1eLi\x98\x17 ~\xdc\x05\x16\x83FW.\xdfӷbN\x88\x94\x1f6Z\x9d>\xa9\xe4[#4h\x91\xb1\x81\xf5s\xb9i\x9e\x89\xb1?ǽ\xf1\xb1W\xbc\x00\xb9\x94+6\x132r\xc9Z\xb5\xb6\xb8\x00\x8e\xe9\x92\xca&\x0f\x1e:E\x13mxt\xe6\xaf\x123%\x8c]3^U\x06\\\xbc\x0f*\xf8\x82O\x13\xa3_\xa3\xd7H\x84\xe7mt\xf1$\x93Mp6H\xf2rj\x0eX\x1a\x1e\xe4\xc3\xc8\x7f\x01\x00\x00\xff\xff\xa7\x94\xfb\xf9\xa5\r\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YKs\x1c\xb7\x11\xbe\xf3Wt\xc9\a\xc6U\x9aYKI\xa5R{\x93\xc98\xc5ĦX\xa2\xa4\x8b\xcb\a\xec\xa0w\x06\xe6\f\x80\x00\x98\xa56.\xff\xf7T\xe31;\x0f\xec.Ɋ\x9c\xb9\x90\x8bG\xe3C\xbf\xbbQ\x14\xc5\x05\xd3\xe23\x1a+\x94\\\x03\xd3\x02\xbf8\x94\xf4˖\x0f\x7f\xb3\xa5P\xabݛ\x8b\a!\xf9\x1a\xaez\xebT\xf7\x01\xad\xeaM\x85\u05f8\x15R8\xa1\xe4E\x87\x8eq\xe6\xd8\xfa\x02\x80I\xa9\x1c\xa3aK?\x01*%\x9dQm\x8b\xa6\xa8Q\x96\x0f\xfd\x067\xbdh9\x1aO<\x1d\xbd\xfb\xae|\xf3\xb6\xfc\xee\x02@\xb2\x0eנ\x15ߩ\xb6\xefpê\x87^\xdbr\x87-\x1aU\nua5VD\xbb6\xaa\xd7k8L\x84\xbd\xf1܀\xf9N\xf1Ϟ\xcc\xf7\x9e\x8c\x9fi\x85u\xff\xca\xcd\xfe(\xac\xf3+t\xdb\x1b\xd6.A\xf8I+dݷ\xcc,\xa6/\x00l\xa54\xae\xe1\x96`hV!\xbf\x00\x88W\xf4\xb0\n`\x9c{\xa6\xb1\xf6\xce\b\xe9\xd0\\\x11\x85Ĭ\x028\xda\xca\b\xed\x8a\x0e\xe1\xb1A\t\xae\x11\x16\xc2m\xe1\x91Y\x82c\x9c\xbfe\xfe`?Oۭc\x9d\x9e \xb82\xc8\x0e[\x03\x04\xce\x1c\xe6\x00\f\xfc\x04\xb5\x05\xd7 q\xde+\x16\x13R\xc8\xda\x0f\x05I\x80S\xb0A\x0f\x119\xf4:\x83LcUj\xc5K\x99\x88N`\xdd\xceF\xcf\xf1\x86\xd6\xff\xafQM\x00\xdd)\xfe\x02(\xcf:7,\x9e\x9c\xfay\xbc\x9e\x90\x00\x05X\x10\x16X\xdc\x1anq`tr\x90\x1f\xfe~\xff\x11\xd2\xd1^\x18s\xee{\xbe\x1f6ڃ\b\x88aBn1:\x98\xadQ\x9d\xa7\x89\x92k%\xa4\xf3?\xaaV\xa0\x9c\xb3\xdf\xf6\x9bN8\x92\xfb\xbf{\xb4\x8edU\u0095\xcf]\xc8a\xf6\x9a4\x97\x97p#\xe1\x8au\xd8^1\x8b_]\x00\xc4i[\x10c\x9f&\x82q\xda5_\x1c\xb86\x9aHI\xd3\x11y\xcd2\xa1{\x8d\x15I\x8f\x18H;\xc5VD\x0fE\xee\x9c͗\x97\x13\xc2yå/\xeb\x9d\xe6\x8b \x17\\f{\x1269\xf2\xa9\xc9a\x86\x95\v\xa2\x00\xed\xdc\xcb\x0e{Ƒ\xcbF\a[.(\x1c\x11\x03}Rq\xdb\a\xf5q\n\xaa\x86\xc9\x1a\xc3}\x11\xb6=E\xc7\xf2\xf2%v\xbcLIҗIM\xe6\x8e\xe3\xff\x16ܟx9\x9fA?\xe1r\xe3*\xe3\xe4\xe5\x1e\xfa\r\x1a\x89\x0e\xfd\xfd\xb8\xaa,]\xadB\xed\xecJ\xed\xd0\xec\x04>\xae\x1e\x95y\x10\xb2.H5\x8b\xa0\x03v\xe5\v\xe7\xd57\xfeϋ\xef\xe2k\xec\xa7^hR\xfb\x7f\xcd[\xd19v\xf5\xa2K\xa5\x1c\xf6\xe9q\xec\xf2>fV\xf3\xbdd\x16\x8f\x8d\xa8\x9aT\x9cD\x1f{Ę\x04e\xc2<\xb8f&\xf7_]\x95\x89\xa1\xbd!D\xfb\"v\xf7\n&9\xfdo\x85u4\xfe\"\x0e\xf6\xe2I\xe6\xfb\xe9\xe6\xfa\x8fQ\xf0^\xbc\xc8V\x8f$\xe0\xe1\xfbR\x1c`\x15\x1d\xd3EX͜\xeaD5[=퉜I\xe3>L\x16\xa7D3\x93\xdf\x0ek\x9e\x95G:Vg\x12\xb7q3\xf3Tzw\x92_\xd3\xc6\r\xab-0\x83\xc0\xa0c\x9a\xe4\xfc\x80\xfb\"$\x04\x9a\t\x8a\xe6\x14\xb0\x87\xae\b0\xad[\x91\r\xdc1\xecǔ5r\x82\xcarV\xdbcw\xcfJm\xdc\x05:#\x85O\xa3\xa5I\x06g\xfaP\xae\xc9\xd9\xf5\xa4;\xb5D\x8b\xb2\xef\x96P\nxPZ\xb0̸A\xeb\x16\xfaE\x13\xaf\x96y\xc9\ta\x05^\x9e\xe1AlXgJ\x9d(\x8a\x90\x17\x0e\xe5\x8e\xefQ\xe6\xea\x89\xe3\xc5\xc4Q\x88T\xcfS\x96;\x85X\xe4\v\xcf\xd9\x1a*\xc4fCZ\xf1\x8b9#3}\xca49风\x91.\xabq\xdf\x1e\x7fF=\x1e\xda\xfe\x91\xa7\xc1\xfb\xba\xf4\x18@\xa5\xc7K+\xf2JQV?\xe9\xe8\x9d\x11\xef\xd5r\x87o~\x19\x1e\xd5]td\xbd\xa3G\x82xF\xae\xa4\x86\x11\xb9\xb0\xd3\xc7;\xa2\x86ܧ\xdcT\x11l\x99h\x91Cz\t\x9a\xef\xc9P\x1dS\xd9\xe0\x96\x82C0\xbdT\xc8FxCZ\xdb X\xdfU\xba\xb4'h\xf6\x16\xb9\xef\x80d\x98\xb0Lu\xb7\xcat̅.h\x91%*\xfb\xb6e\x9b\x16\xd7\xe0L\xbf\x9c>a\x89\x1dZ\xcb\xeas\xa6\xf8SX\x15\xea\xfb\xb8\x05\xd8F\xf5n(\xf0'\xee\xf1\xd2F\x9dz^\x8f![:OՙQicc\x8a߶~\xcf\xd8\x11\x1c^\t=\xaa\r\xe6S\x84\x97\xf8\x04\x00\xff\xfcu\x0e!\xad\xc9\x19\xd8\xe0\xbdNZ\x18\x9cpʷ\xf8\x98\x19]<ۍ'\xaf\x92\xc9d\xe6~\xf0\xd6\xf0\xac\xfbǃα .\x83F\xb5ɘ\x95c-Ⱦ۠!>l\xf6\x0e\xedԝ\xe7\xba9\xbe\n<\xb0q\xb4?\xc9/P\x8a\x85mŤﺒu9\x05\\Xݲ}\x86p\xba\x88\xcf\xf4ȸ\xc8\x05\x1c\xf49\x19\xb5F㧞ۅ\U00098b95\x18\x9f\x83:Y|&\nŧ\xea\\\f\xbaG\xcd\fY\xba\x7fA\xb8\x9a?q\xbd\x06+|[\x942ϐ\x8a\x86\xa6\x85\xa5\xe0D\xa9\x952\x98q\x99\xb0\f+\x93 2\x85\xffGƏ\xac\x9e,\x06=r>\xa2\x1d[\xeb\xe3\x91~3<\x1b\xad\xe1\xb7\xdf/\xfe\x1b\x00\x00\xff\xffÊ\xc5\x01R\"\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YKs\x1c\xb7\x11\xbe\xf3Wt\xc9\a\xc6U\x9aY[I\xa5R{\x93\xa88\xc5ĦX\xa2\xa4\x8b\xcb\a\xec\xa0g\x06\xe6\f\x80\x00\x98%7.\xff\xf7T\xe3\xb1;\x0f\xec.Ɋ\x1c\\\xc8ţ\xf1\xa1\xdf\xddS\x14\xc5\x05\xd3\xe2\v\x1a+\x94\\\x03\xd3\x02\x1f\x1dJ\xfae\xcb\xfb\xbf\xd9R\xa8\xd5\xf6\xfb\x8b{!\xf9\x1a\xae\x06\xebT\xff\x11\xad\x1aL\x85\xef\xb1\x16R8\xa1\xe4E\x8f\x8eq\xe6\xd8\xfa\x02\x80I\xa9\x1c\xa3iK?\x01*%\x9dQ]\x87\xa6hP\x96\xf7\xc3\x067\x83\xe88\x1aO<]\xbd\xfd\xae\xfc\xfeM\xf9\xdd\x05\x80d=\xaeA+\xbeU\xdd\xd0\xe3\x86U\xf7\x83\xb6\xe5\x16;4\xaa\x14\xea\xc2j\xac\x88vcԠ\xd7pX\bg\xe3\xbd\x01\xf3\xad\xe2_<\x99w\x9e\x8c_\xe9\x84u\xffʭ\xfe(\xac\xf3;t7\x18\xd6-A\xf8E+d3t\xcc,\x96/\x00l\xa54\xae\xe1\x86`hV!\xbf\x00\x88O\xf4\xb0\n`\x9c{\xa6\xb1\xee\xd6\b\xe9\xd0\\\x11\x85Ĭ\x028\xda\xca\b\xed\x89\x1e\xe1\xa1E\t\xae\x15\x16\xc2k\xe1\x81Y\x82c\x9c\x7fe\xfeb\xbfNǭc\xbd\x9e \xb82\xc8\x0eG\x03\x04\xce\x1c\xe6\x00\xec\xf9\t\xaa\x06\xd7\"q\xde+\x16\x13R\xc8\xc6O\x05I\x80S\xb0A\x0f\x119\f:\x83LcUj\xc5K\x99\x88N`\xdd\xccf\xcf\xf1\x86\xf6\xff\xafQM\x00\xdd*\xfe\x02(Ϻ7l\x9e\xdc\xfae0\x96=\xff\xc6\xc40j/'X\x17\x8a\x11\x86\x0ff'$@\xe1\f\x84\x05\x16\x8f\x86W\x1c\x18\x9d\xdc\xd1ǿ\xdf}\x82t\xb5\x17Ɯ\xfb\x9e\uf1c3\xf6 \x02b\x98\x905\x995\t\xb16\xaa\xf74Qr\xad\x84t\xfeG\xd5\t\x94s\xf6\xdba\xd3\vGr\xff\xf7\x80֑\xacJ\xb8\xf2\x99\x02\xb9\xa7A\x93\xe6\xf2\x12\xae%\\\xb1\x1e\xbb+f\xf1\xab\v\x808m\vb\xec\xd3D0Nr\xe6\x9b\x03\xd7F\v)E9\"\xafY\xdeq\xa7\xb1\"\xe9\x11\x03館E\xf4P\xb52\xc0\xe6\xdb\xcb\t\xe1\xbc\xe1\xd2\xc8z\xa7\xf9\xa6\x19\xb2w\xb93\t\x9b\x1c\xf9\xd4\xe40\xc3\xce\x05Q\x80n\xeee\xf7g\fje\x85SfG\x84\x83\x83-\x17\x14\x8e\x88\x81\x86T\x1cϼ\xe3Fq\xcc\xc1\xa6\xa3\xe0Z\x16\xb4\x95\xf2+\xf2G\x83\x94\xcb[h(\xf9,`Z\xf13\xb8\xe2\x8d\f\f\xd6hPV\x98\x1cש\xe4!\x83l\x1c֗\x18\x8f+\x05\x9c\xf0\xeaY\xc4oo\xaf\x93'OL\x8c\xd8\xdd\xf2\xde3\xfc\xa1Q\v\xec\xb8\x0ft\xe7ﾼ\xae\xc3eާ9\x05\f\xb4\xc0\x90\x06\xee\x83\x04\bi\x1d2\x0e\xaa\xceR\xa4\x9a\x04\xc8\xf0\r\xc6\x13\xaf\x83\a\x8b\xae\xf2\x10Z\x88\xf7\xc0\xc8w\n\x0e\xff\xbc\xfbp\xb3\xfaG\x8e\xf5\xfbW\x00\xab*\xb4>\vvأt\xaf\xf7\x899G+\frJ\xb3\xb1\xec\x99\x145ZW\xc6;\xd0؟\xdf\xfc\x92\xe7\x1e\xc0\x0f\xca\x00>\xb2^w\xf8\x1aD\xe0\xf8\xde-'\xa5\x116\xb0cO\x11\x1e\x84k\xc5<\x98\xee9@\xea\x15\x9f\xfd\xe0\x9f\xeb\xd8=\x82\x8a\xcf\x1d\x10:q\x8fkx\xe5Ӛ\x03\xcc\xdf\xc8v~\x7fu\x84Ꟃi\xbf\xa2M\xaf\x02\xb8}\x1c\x1e\x1b\xdd\x01d\xb0<#\x9a\x06\x0fY\xd5|\xf8\xa0B\xae\xfa[P\x868 Ո\x84'L\xd2\v\x8e\x12\xf9\x02\xf4\xcfo~9\x8ax\xca/\x10\x92\xe3#\xbc\x01\x11K\x1b\xad\xf8\xb7%|\xf2ڱ\x93\x8e=\xd2MU\xab,\x1e㬒\xdd.\xe4\xb9[\x04\xab\xa8P®+B\x1e\xc4\xe1\x81\xed\x88\vIp\xa4o\f43\ue936\xa6\xec\xe7Ӈ\xf7\x1f\xd6\x01\x19)T\xe3=1E\xcdZP6CiL\x88\xc5^\x1b\x17\xc1<\r;\x04\xf5q\n\xaa\x96\xc9\x06\xc3{\x11ꁢcy\xf9\x12;^\xa6$idR\x93\xb9\xe3\xf8\xbf\x05\xf7'>\xceg\xd0Oxܸ\xca8\xf9\xb8\xfba\x83F\xa2C\xff>\xae*KO\xabP;\xbbR[4[\x81\x0f\xab\ae\xee\x85l\nR\xcd\"\xe8\x80]\xf92u\xf5\x8d\xff\xf3\xe2\xb7\xf8\x8a\xf6\xa9\x0f\x9aT\xda_\xf3Ut\x8f]\xbd\xe8Q)\x87}z\x1c\xbb\xbc\x8b\x99\xd5\xfc,\x99\xc5C+\xaa6\x15'\xd1\xc7\x1e1&A\x990\x0f\xae\x99\xc9\xddWWeb\xe8`\bѮ\x88\xbd\xb4\x82IN\xff[a\x1dͿ\x88\x83\x83x\x92\xf9~\xbe~\xff\xc7(\xf8 ^d\xabG\x12\xf00\x1e\x8b\x03\xac\xa2g\xba\b\xbb\x99S\xbd\xa8f\xbb)+\xbd\xe6\xc4\xf8Z\xa09\x93\xc6}\x9clN\x89f&\xbf\xdd\xefyV\x1e\xe9X\x93I\xdcƭ\xc3S\xe9\xddI~M\x1b7\xac\xb1\xc0\f\x02\x83\x9ei\x92\xf3=\ue290\x10h&(\x9aS\xc0\xdewE\x80i݉l\xe0\x8ea?\xa6\xac\x91\x13T\x96\xb3\xc6\x1e{{Vj\xe3.\xd0\x19)|\x1emM28Ӈrmή'ݩ%Z\x94C\xbf\x84R\xc0\xbd҂e\xe6\rZ\xb7\xd0/Zx\xb5\xccKN\b+\xf0\xf2\f\x0fb{8S\xeaDQ\x84\xbcp_\xee\xf8\x8e`\xae\x9e8^L\x1c\x85H\xf5a\x89=Z˚s\xa6\xf8S\xd8\x15\xea\xfbx\x04\xd8F\rn_\xe0O\xdc㥍:\xf5\xbc\x1eC\xb6t\x9e\xaa3\xa3\xd2\xc6\xc6\x14\xbf\xeb\xfc\x99\xb1#8|\x93\xf3\xa86\x98O\x11^\xe2\x13\x00\xfcǦs\biO\xce\xc0\xf6\xde뤅\xc1\t\xa7|\x83\x0f\x99\xd9\xc5G\xb2\xf1\xe2U2\x99\xcc\xda\x0f\xde\x1a\x9e\xf5\xfex\xd19\x16\xc4mЪ.\x19\xb3r\xac\x039\xf4\x1b4ć\xcdΡ\x9d\xba\xf3\\7\xc7W\x81\a6\x8e\xce'\xf9\x05J\xb1\xb0\xad\x98\xf4]W\xb2.\xa7\x80\v\xab;\xb6\xcb\x10N\x0f\xf1\x99\x1e\x19\x17\xb9\x80\x83>'\xa3\xd6h\xfc\xd2s\xbbP\x1e\xd3{%\x8f\xd4%ɞ\x85t\x7f\xfdˉ\xbcPH\x87\xcd,8\xc4ub\xe7;\xba\xe5\xeb\xdcp\"\x89\xb1\x92i\xdb*w\xfd\xfe\x8c\x16\xdc\xed7&k8\xa4\x8c\xde\xf7\xf9\x9ep\xdc\x14U!'\xaa\xbdoy\x96\xa9N?Ϟ\x83:\xd9|&\n\xc5\x0fù\x18t\x87\x9a\x19\xb2t\xff\x05\xe1j\xfe\x89\xeb5X\xe1ۢ\x94y\x86T44-,\x05'J\xad\x94\xc1\x8c˄eX\x99\x04\x91)\xfc?2~d\xf5d1\xe9\x91\xf3\x11\xed\xd8Z\x1f\xcf\f\x9b\xfdg\xa35\xfc\xf6\xfb\xc5\x7f\x03\x00\x00\xff\xffY\xc0\xfaX\xc0!\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Y_\x93۶\x11\x7fק\xd8q\x1e\xae\x991\xa9\xd8\xedt:z\xb3\xef\x9aε\xc9Yc\x9d\xfd\x92\xc9\x03D\xacHD$\x80\x02\xa0tj&߽\xb3\x00A\xf1\x9f\xa4\xd3M.\xe1\x8b}\xc0b\xf1\xc3\x0f\xfb\x0f\xab$IfL\x8b\xafh\xacPr\x01L\v|r(\xe9/\x9bn\xffaS\xa1\xe6\xbbw\xb3\xad\x90|\x01\xb7\xb5u\xaa\xfa\x8cV\xd5&\xc3;\xdc\b)\x9cPrV\xa1c\x9c9\xb6\x98\x010)\x95c4l\xe9O\x80LIgTY\xa2Ir\x94\xe9\xb6^\xe3\xba\x16%G\xe3\x95ǭwߥ\xefާ\xdf\xcd\x00$\xabp\x01Z\xf1\x9d*\xeb\n\rZ\xa7\f\xdat\x87%\x1a\x95\n5\xb3\x1a3R\x9e\x1bU\xeb\x05\x1c'\xc2\xe2f\xe3\x00z\xa9\xf8W\xaf\xe7s\xd0\xe3\xa7Ja\xdd\x7f&\xa7\x7f\x10\xd6y\x11]ֆ\x95\x138\xfc\xac\x152\xafKf\xc6\xf33\x00\x9b)\x8d\vx (\x9ae\xc8g\x00\xcd9=\xb4\x04\x18\xe7\x9e9V.\x8d\x90\x0e\xcd-\xa9\x88\x8c%\xc0\xd1fFh\xe7\x99i\xf5\x80ڀ+\x90\xb6\xf4\xac2!\x85\xcc\xfdP\x80\x00N\xc1\x1a\xa1A½2\x80_\xac\x92K\xe6\x8a\x05\xa4D\\\xaa\x15Oe\xd4\xd9\xc8\x04\xce\x1f\x06\xa3\xee@\xe7\xb0\xce\b\x99\x9fB\xf6;\x83\xea\xe1Y*\xfeL$\x8f\x05z\x99\x88\xa6֥b\x1c\rm^0\xc9K\x042Pp\x86I\xbbAs\x02E\\\xf6x\xd0}$_\xa2\xbe\xce\xcc5\xec\\CE\x90\xedm\xff\xb5;tiߥ\xe2\xcd\x02h\x8c\x1a\xacc\xae\xb6`\xeb\xac\x00f\xe1\x01\xf7\xf3{\xb94*7h\xed\x04\f/\x9e\xea\x82\xd9>\x8e\x95\x9fx]\x1c\x1be*\xe6\x16 \xa4\xfb\xfb\xdfNck\x16\xa5N9V~<8\xb4=\xa4\x8f\xc3ဖ\x9c-o\xae\xffO\x81\xbb&HwJ\xf6y\xfd8\x18\x9d\x02\xdbQ\x1a\xe3m\x9a\x19\xf4\xa1\xf6QTh\x1d\xabtO뇼\xaf\x8f3\x17\x06\xc2\xf4\xee]\beY\x81\x15[4\x92J\xa3\xfc\xb0\xbc\xff\xfa\xd7Uo\x18@\x1b\xa5\xd18\x11\xa3k\xf8:ɣ3\n}foHa\x90\x02NY\x03mp\x8a0\x86\xbc\xc1\x10\x9cEX0\xa8\rZ\x94!\x8f\xf4\x14\x03\t1\tj\xfd\vf.\x85\x15\x1aR\x03\xb6Pu\xe9#\xd0\x0e\x8d\x03\x83\x99ʥ\xf8_\xabے\xefѦ%s\u0604\xf8\xe3\xe7c\xb0d%\xecXY\xe3[`\x92C\xc5\x0e`\x90v\x81Zv\xf4y\x11\x9b\u008fd!Bn\xd4\x02\n\xe7\xb4]\xcc\xe7\xb9p1if\xaa\xaaj)\xdca\xee\xf3\x9fX\xd7N\x19;\xe7\xb8\xc3rnE\x9e0\x93\x15\xc2a\xe6j\x83s\xa6E\xe2\xa1K\x9f8ӊ\x7fc\x9a4kozXGN\x17>\x9f\xeb\xce\xdc\x00%;\x10\x16X\xb34\x9c\xe2Ht\fٟ\xff\xb9z\x84\xb8\xb5\xbf\x8c!\xfb\x9e\xf7\xe3B{\xbc\x02\"L\xc8\r\x05]\xbačQ\x95\u05c9\x92k%\xa4\xf3\x7fd\xa5@9\xa4\xdf\xd6\xebJ8\xba\xf7\xff\xd6h\x1d\xddU\n\xb7\xbe\x92\xa0xYk\xb2\\\x9e½\x84[Vay\xcb,\xbe\xfa\x05\x10\xd36!b\x9fw\x05\xdd\"h(\x1cX\xebL\xc4\n\xe6\xc4}\r\xab\x92\x95ƌ\xae\x8f\x18\xa4\xa5b#2\xef\x1b\x14~\x80\x8d\xe4Ӟ\xeaiץoͲm\xadWN\x19\x96\xe3\x0f*\xe8\x1c\n\r\xb0}\x9cZ\x13\xc1\xc9N\xce\v\xca\xc1\x06ɑR\x802.\xde\x17h\xb0\xbbƠVV8e\x0e\xa48d\xcbt\xa4\xe1\xc4E\xf8#+~\xe1\x18\x14\xee\xbdC\x18ܠA\x99a\x8c\x10\xe7*\x99\x89St\x12\xfa\x18\xe2i\xea\xe1L\xf4\x9c\x04\xfcay\x1f#fd\xb8\x81\xee\xc6\xfb^\xa0\x87\xbe\x8d\xc0\x92\xfb\x84ry\xef\x9b\xfbM\xd8\xcc\xc7\x0e\xa7\x80\x81\x16\x18*\xd26\x18\x83\x90\xd6!\xe3\xa06\x93\x1a\xe9m\x00\xe4`\x06\x9b\x15oC\xa4hB\xd21\x84\x13\xf5\xc0(F\t\x0e\xff^}z\x98\xffk\x8a\xf9\xf6\x14\xc0\xb2\f\xad\xf5\xf9\x1a+\x94\xeem\x9b\xb39Za\x90S\xe1\x82iŤؠui\xb3\a\x1a\xfb\xd3\xfb\x9f\xa7\xd9\x03\xf8^\x19\xc0'V\xe9\x12߂\b\x8c\xb7\xe1/ڌ\xb0\x81\x8eV#\xec\x85+\xc40i\xb5\f\x90u5\xc7\xde\xfb\xe3:\xb6EP\xcdqk\x84Rlq\x01o|%x\x84\xf9+9\xd6ooNh\xfdKp\xa07$\xf4&\x80k\xf3]\xd7#\x8f ]\xc1\x1c8#\xf2\x1c\x8f\x85\xe8\xf0\xf3\xc1\x9bBⷠ\f1 UG\x85WL\xb7\x17\xe2\x11\xf2\x11\xe8\x9f\xde\xff|\x12q\x9f/\x10\x92\xe3\x13\xbc\a!\x037Z\xf1oSx\xf4\xd6q\x90\x8e=\xd1NY\xa1,\x9ebV\xc9\xf2\x10\xaa\xfd\x1d\x82U\x15\xc2\x1e\xcb2\t\xf5\x06\x87=;\x10\v\xf1\xe2\xc8\xde\x18hf\xdcYk\x8dU\xc6㧻O\x8b\x80\x8c\f*\xf7\xf1\x8e\xb2\xd3FP\xd5@\xe5B\xc8y\xde\x1aGI3~\xb6\x0e\xe6\xe3\x14d\x05\x939\x86\xf3\"lj\xcaB\xe9\xcdK\xfcx\x9c\xfa\xe37Q\x02\f\x03ǟ\x96D\x9fy8_\xa9>\xe3pݷ\xd6\xd9\xc3m\xeb5\x1a\x89\x0e\xfd\xf9\xb8\xca,\x1d-C\xed\xec\\\xed\xd0\xec\x04\xee\xe7{e\xb6B\xe6\t\x99f\x12l\xc0\xce\xfd\x93y\xfe\x8d\xff\xe7\xc5g\xf1\xaf\xeb\xe7\x1e\xa8\xf7\xe8\x7f\xcdS\xd1>v\xfe\xa2C\xc5Z\xf1\xf9y\xecf\xd5\x140õ\xe4\x16\xfbBdE|\x0441\xf6\x843\t\xaa8y\b\xcdL\x1e^ݔ\x89\xd0\xda\x10\xa2C\xd2\xf4\xb4\x12&9\xfd\xdf\n\xebh\xfcE\f\xd6\xe2Y\xee\xfb\xe5\xfe\xee\x8f1\xf0Z\xbc\xc8WO\x14\xba\xe1{J\x8e\xb0\x92\x8a\xe9$H3\xa7*\x91\r\xa4\xa9\xf6\xbb\xe7D\xfcF\xa0\xb9P\xc5}\xee\t\xc7*t\xa2\x8ale\xae*#\xadd\xda\x16\xca\xdd\xdf]\xc0\xb1j\x05#\x86\xe3u5\xc5c\xd45h\x02]\x87\xc7\xfb\xcb\xc3\xe9@\xd2\a\u0557\x8eȔ\x11\xb9O[\xad\xef\xfbW\x84d\x15\xeb6\xff\xba_Ŵ\x162\xbf\nk\xb7\x97v\x01藎hDy\xa1\x9b\xe7\x8a)\x9c\xbd\x1e\xdf\x18-ʺ\x1aCI`\xab\xb4`\x13\xe3tG#\xfb\xa4\x897\xe3\xba\xe6\f\x13\xc1\x00.pд\x9e&\xdeQ\x8d\xfd\x84\xbaҏ\xd0\xdb\xc5[\xd1t@\xbe֮\xe8\xd9MEr\x1fa2\xfd:\x1c\xc8h\xc5gCҺ.9\x98<:\xd4p\xa2o\xab\x83\xd9^K\xb4{\x9a\xf1\xc3\xda\xf7ۮyZ\x87\x1e_\xc3{\x88\xf0.v\xfe\xe8y\xf3\xe2\xc7u\xa6\xe8\xe9\xd0k\xcf]\xb0\x81\xdb\xf1\n\xdf\xc92\xbc\xf1\tQ\xa1\x7f\xb1\x86\xf6\xe4\x9eٸ\xc9\xd4}CG_X\xea\xb3*\xa9C\xee\v{zwl\x98(\x91C\xfb+\x8bo\xa5[\xdfҹ\x99\xaac\xa3\xa2\xda\"\xf7qc\x02\xf4x]\xec\x92r\xe60!\x15#\tY\x97%[\x97\xb8\x00g\xea\xf1\xf4\x19\xf7\xaa\xd0Z\x96_\xf2\xaf\x1f\x83Tx\xf37K\x80\xadU\xed\xdaG\x7f\xe3h\r\x157\xb6\xb1\x82\xeb\x1a\x0f\x05\xb3\x97\xa0,If\xca\xe2Z\x97?orp&\x94=\xe0~btԵ\xeeN\xdeF\x13\x9a\x98\xfb\xde[\xc7U\x044\x1b]\xe2\xa0\x11\x83B\x95Ѻ\x95\xa3\xa4TWk4D\x84o\x95GFb\xe0\x98\xea\xa2\xf8\xd7בɣ\x86\x18\v\x83\xaa\xe6=\x991雊d\xbfN\x01\x17V\x97\xec0\xa17\x9e\xc4\x17Xd\xbe\xe4GG\x8b\x89^H\xee\xef\xe7\xae\xed\xfe\xb4?\x05L\x97\x7fS?,L\xddB\xf7W\x82\xc1|\xfb\x1b\xc8\xeb\xecp\xa6䳎\x19\xf7ܰ\xb7\xea\t_\x8ax^\xf5t\xbc놮q\xa0\xeao\xf3GƨI\xa2F\x83\x1e9\xef\xe8n:\xa7ݑz\xdd\xfe.\xb0\x80_\x7f\x9b\xfd?\x00\x00\xff\xffg\b\x17r\xc1\x1f\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xdc{j\xec\xf2\x9e!\xb2%b\x87\x04\xf8\x00Pc%\x95\xff\x9ej|\xf1\v$A\xcd\xe8\xad_p\x13\x054\x80\xeeF\x7f\x03\x9b\xcd\xe6\x86\xd6\xec;H\xc5\x04\xbf#\xb4f\xf0C\x03\xc7_j\xfb\xf4\xefj\xcbĻ\xd3\xfb\x9b'\xc6\xf3;\xf2\xa1QZT\x8f\xa0D#3\xf8\x05\x0e\x8c3\xcd\x04\xbf\xa9@ӜjzwC\b\xe5\\h\x8a\x9f\x15\xfe$$\x13\\KQ\x96 7G\xe0ۧf\x0f\xfb\x86\x959H\x03\xdcO}\xfa\xd3\xf6\xfd\xbfn\xfftC\b\xa7\x15\xdc\x11\tJ\v\tj{\x82\x12\xa4\xd82q\xa3j\xc8\x10\xe6Q\x8a\xa6\xbe#\xed\x1fv\x8c\x9bϮ\xf5\xd1\x0e7_J\xa6\xf4_\xba_\xffʔ6\xff\xd4e#i\xd9Nf>*ƏMIe\xf8|C\x88\xcaD\rw\xe43NS\xd3\f\xf2\x1bB\xdc\xd2ʹ\x1b\xb7\xea\xd3{\v\"+\xa0\xa2v=\x84\x88\x1a\xf8\xfd\xc3\xee\xfb\xbf}\xed}&$\a\x95IVk\x83\x00\xb76\xc2\x14\xa1\xe4\xbb\xd9\x1b.\xc0\xe0\x9a\xe8\x82j\"\xa1\x96\xa0\x80kEt\x01\x84\xd6u\xc92\x83\xea\x00\x91\x10q\b\xa3\x149HQ\xb5\xd0\xf64{jj\xa2\x05\xa1DSy\x04M\xfe\xd2\xecArРHV6J\x83\xdc\x06X\xb5\x145H\xcd\xdf\x12\xa6\xfd\xd7%\x88\xb4,;\xf3\xff\x81\t\xb3\x9e\xe3wÑ\xaf\xca\xf1\xb3TY\x82\x88T\t\xd3\xff\x01\x89b\x94\xc5W\xa7+\x92\t\xf2\xd7\xee\xa8[\xc2\x0e\x81 \xf9-9\xb0R\x83\x1cP\xe6E\xe7\xe55\x90\x91\xa2\xef\xb0UTg\xc5\xc7\x1fh٨6Δ\x88\x97\xe1`k\x12{\x1f\xa1\xaf\x98\x17\xe0\x12\xe3\xbe2\t\x95u\x8b\xbf\x19l\xb6_\x8c?q\xff\xf9\x17\xc8\xe7\xd0C\xd28o\xb4\x91\xfb\xc1b\xbbS;;?u\x1b\xce\xf4\t>\x93\rx\xdc\x12J\x9e\xe0l-\x16\xca\t\x12\x87jc\xefF\xbd\xa71rL\xe4\xc50\xd9\x13\x9c\r\x18\x17JY\x1c\x9d\xca\n\xb6=A\xc4\u070f\xb5\x1e\x02qM\xce\xc1\xb5\x98\xc4\x0f\x06\x11\xc6\xf1NG\x1e1a1/\x8b\x967G\xd2\x05\x89o\x1e\xf7\x17l3\x90\xad\x13>4\x84}\xab,\x89\xf0\x14\x14\xacNܨ\x89\x1e*0\xa7\xc5\aƾӒ\xe5a\"\xcb\xf7;>m\r\xf7\xdbg\xa1w\xfc\xd6zd\xcap\xc9/\x02\xd4g\xa1͗\xab\xa0\xd3.\xfc\x02dځ\xe6xq+\xb6\x11\x0f\xdd\b[\x02s۶\xb3\x81\x94@\x1e\xa6Ȏ\xa3\xe3\xe2\xf0a\xe2\xa5v\xbay\xfd\xd0oU\xa3L\b\x8d\v\xbe1\xaar\x1b\x9b\xc9\";\x11\xa4\x90=\x8a\x8c\x97\x16&\xb5\x13&\x82\xfd\x86\x9aĎ\xb7\x11\xe0\x92f\x90{o\xd3\xc4-\xa9\x86#\xcbH\x05\xf28\xa78\xba\xadF\xf9\x9e\xb6\x84D\xa9k\xdbJ\x0eKS\xed\xbe9ѝ//f\x83'7\xa1\x97'\xf6b\u05c9p\xe5t\xd7\xe5\x1d\x19\x15k\xec\x8fE\xec\xd2<7)$Z>\xac\x90\xf8+h1\xd6\xfdvaVCV\xb4\xc6\xf3\xfb?\xa8\xe6\fC\xff/\xa9)\x93\tg\xf8ޤ\x89J\xe8\x8du\x81\xb1\xee48\x03S\x04\xe9{\xa2\xe58\x10\x1eٜ@\xd9\x02\xa5U\xe4\xe20\xb2Xn\xc9s!\x94թ\a\x06e,d\xd3oL\x917Op~s;\x92\x03ov\xfc\x8dU\xf0\xab\xc5M\xb0\x16\x04/\xcf\xe4\x8d\x19\xfb\xe6%FP\"'&v\xfb\xb1y\n!\xb9ME\xeb\x8d\xe3^-*\x96M\x8e\xe3\xd1\xf0x\xdbz\xec\xd4\r\x91\xb7\xb1qg\x1e\xcf\xed6\x89\x7fk\xa1\xf4\x9fま\x89\xf5<\xf8\x11}\x9b6\x12/[\xb4\xf5]\xec+\bc\xb4\x00\x0f\x1a\xa4\v\xfeY\x01\xed=\x87\x17\xfaTK\xc1\xbd\x10أ! \x8b\b^\xe0&\x9b*IY\xe2\x1ak\x13\xf1\xb2\xd2N\xff\xf8\xa3\x13\x9bē\x8d\xbf\xbb\x1bymk8\x13UE\x87\xc9\xc1\xa4\xa5~\xb0#=O;@\x96\xfa\xf2ؘ\xf3\x9cn&z\x1e2i\xc1g\xa6\v\xc6\t\xf5b\x03\xa4c(Jj\xb1,\xc1l+\xa8\"{\x00\x1eb\xea?\x83\x9e\xaf\x18ߙ\t\xc8\xfbW\xb7\vH\x8b\xae\x8b\xc8\xe9Q\x1d\b\x1a>\x18M\x95jR\x89\x9c<\x17 \xa1\xc7\x15\xe3@9Z\x9a\x89 \xb9\xd0\xddx\x04\u00adE\xfeV\x91\x03\x93Jw\x17\x9a\xcap\x8dJe\x87\x95\x14\xc6\xdd}c\x15\x88F_@\x83\x8f\xed\xe8^^\xb7\xa2?X\xd5T\x84V\xa2I0\nlC\xfdª\x90|u\x14x\xa6L\x87<\x94\x89\xcch\x81T\xaaKЩ$\xde\xc3\x01\xc5Q&\xb8b9H_\x1c`)\xcb\x04\x1e\xdc\x03ee\x13K\xfb\xc4\xdaZ\xf7\x96\x7f\x94\xf2\"\xef\xf6\x8b\x1dى6\x16⹏\xa0d\x14\x14\xf4\x04\x84\x1d\b\xd3\x04x\x86t\x01iE\xb6\x99\xc2!à&\x99-\xd3\x04<6\xe0M\x95\x86\x80\x8d9ٌ\xcf\x06Ӻ\xdd?QV^\x83l\xc8y\x97\x1f\x8d\xbf\xb5\xa3\x7f\x97\xa3\x11\x84J\xba\n\xdb\x03y\x04\x9a\x9f\xfd\xf9\xa0Z\xa3\x8bkx@\x10\xd9\xf0\xaeD\xbc\xc2\xc9X\xe3\x17\xbaU\xbc\xa6\xc3\xc78K \xec \x0f\xc0t\xd7\xdaA\x10W\xb5vp\x82\xa0\xe8.\t\xe9\xecz\x00PUz\xc3٬=p\xcd\n\xcbg\x0f\xe8\xd8Bn\x83e\xa8>\x9d\x1dmK\x9e&\xd2\xe7\xd1ݭ7]\x92(\xeb[\xcfK2\xe1Ay\x82Mß\xb8x\xe6\x1b\xe3]\xaaŸ\xfdp\x05\xe9\xe1\xaaW\x9d^_,\x89~O)\xd4\xe7\xd7t\x9e\xf2\n\xfd\nR&\x99oVy\xe8s\\\xb0$\xd7l9\xedğ\x8b\xab\x98\x9b\x7ff\xb0K~~\xb0u\xb0\xa95V\xbb\xf8\xa8\x8eA\xf2\\\x80.@\xfa\x02ۍ\xa9%\x8e\xc9\xe96G\xda\xda֡\xe8\n\xf9Ǜg\xb6\x18pP\x86\x157\xbeyS\x96\xb7\xc8ش)\xb5-\x89\x95M\x84\x89\x92\x8a\x91\xf6B\x94@\x87\xe5\xb9)\xd9\xfc\xa5\x1c~\xbf.-\xe4\xd0}a\x9a\xf0\x93Dvhii\xabO\xbb\t\xe2~2ބ\xa1\xfcJ\xff\xe1%k\ty\xf6\x85\xec\xfa|!\xdf\x1c\xbe\xc6l\xd3\xc5X˃\xae\x9f\xab\xf0\xfc\xb9Ч\xa1\xfaR\xbbs0i\x80\xf61\x18\x192(Q0\x92\x1b\xddH\x93\xec\xa6,&\\\xf0\x14\xba\x10\x15B\xbc\xcf\xccI\x14\x1e\xb02\xe1Ow\xda\\\xc55S\xe4=)D\x13)\xf3\x9a\xc1\xceB\xd2\x7f:\xd5\xef\x02۠\xe9\xe9\xfd\xb6\xff\x8f\x16.\xf1o\xa21\x91\xdd=\x17!\xb6b\xac\x15\x9e\xb3\x13\xcb\x1bZ\xf6\x0eY\x87-Z\xee!B\x12\xce\xcaX\xce\x0f\xd9ʏ\xef\xb1\x11\xf9R\xdb\xd8\xffjq4o\"\xa6\xd5\a\\\\\x15\xd0\xcf\xfaO(\xa9\xb5a\xf0\xf4\xf2\xc7\xf4\xbc\xff|\xa2~M\xb6\x7f\x98˟\x04\xba\x9c\xe3O\xb1\xee\x17\xf2\xf9\x17d\xf1\x13+\xb8^\x1c\xb4O\xc9\xd3_\x94\x9d_,rJ\xcc\xc9\xf7\xb3\xed\xf3 Wdⓐ\xb3\x9cu_\x9dkw\xb9\xed\xd9}$g\xd8#\xb9\xf3Y\xc0\x93y\xf5\xb9\x8c\xf9<\xca#\xd9\xf4\xf4<\xf9,h\x93C_Ύ\xbf^\r\xdckx\x01Ӣf1\xc3\xfd\"/!!\x87\xbd&s\xbd\x88\xb1\v\xb3\xd4!\v=1\xef\xda\xdct?\xf7<\x014%#=\x91q\x9e\x808\x9b\x87N\xcd3O\xc0^P\xbb\xb3\\2\xfb\xe7\x9a\xfcrpC~\xa5u\xcd\xf8q\xcc'\xa9\xdc4\xcbI\xa3\xe4tw\xce\x1e+u\xbd\x85\x9e\x9f\x15\x9b\xd2\xde\x14\x8d\xf8d>\xacǸ\x16[r\xcf\xcf#\xb8\xa6L=\xea\x83\xf4\xaf\x12ᲞYYv\xef\xcb\x18\xb0]P\xee晊G\x06\xb0㔅\x1d%\xa1\x90=\xebx\xc9\x05\xfb2\xe8\xde\r\x14\xce[\xdb1C\x9b\xe9\xe2Bk\xbbjJ\xcd\xea葯\xa581\x13v,\xe0\x1c\xf0\xf9wan\xaa\xec\xcf\x06җ\xc7p\x1a\xb7\x03ǁ\xc6\xce\xd03\x94%\xa1j\xbc\xfd\xcc^\xd6\xcc\xc4\xc6ܿBJz~p\x97:o͉\x8dy\xec\xdc_#\xac\x10\x8c\xb9\xf0\xa9\"\x11\x91I]4o\x0f[\xd3\xdd|\xfb\xad\x01y&\xe2d\xf2\xcc\xce@Z(\x05\xb7rE\xa1\xff\xe6%\x9d\x13\x97\xf6\x8a\xf0\xc0Oh\xe5\v\xb9\xe7VcG\xc1\x0e\xd6hࠈk}#\x94\xe6\xe8\xf6Lt\x8dB\xe5\"\x8c\x8e\xf3ìbJ\xad\xa3\xbe\xae\xa7\xb4\xdeWZ\xb4R\xae\xe2/]\xee1̀L\xad\x8bNˉ,\xd6A_\xcbsZ\U0009d48dƴ:\xe7k\xd47\xaf\xa8k^\xe1C\xad\xf3\xa2\x92єR\xbf|\x15_\xea\x8a\xde\xd45\xfc\xa9\xcb<\xaa\x05\x90\x83\xba䔊\xe3\xa44^r\xce&%˶\x9c9\x9e\xaf$N\xa8 N\xc8\x06-\xad4\xa1Rx]\x85p\x02\x0e\xaf\xe4k]\xc9ۺ\x86\xbfu]\x8fk\xd1\xe7Z䜅\xbf\xd7U\xf6^\x9cd\xf0\xe9\xe8\xcf\"\x87\a!\xf5\x92\x83\xf00\xec\x1fI\x01v\x9c&Q\xe6\x84\xfb\xae\xb1L\x03\xda\xfe\xce\xee\xbflS\xf1l\x9d7\x7f\x7f\x159\xaem)\xb7\xf08\xe8>\xba\xd6y\x00\t\xdc>v\xf0__\xbf|\x0e\xf0c\xf6\xa83z\a\xf7쭁\x91;\xe4\xb8\xec\x93+\xb8\xb1\xd82:\xfc\x95\x93\x04\xb4f\xffiޑZ\x0e\xc8\xdc?\xecLWo-\x99\xf7\xa7BB?\xe4\xde\xf6\x80\xda#`d\x92\xfbw\x87\x1e\xc4H)d\xf8I\xcc+>^{\xb1!-}\xb3EHx\xea\x1evvu[\xf2\tM7~&\xc22^\xc1d\xbe\xa9\xa9\xd4g\xc3\x1d\xea6\xaca:(\xe3u\xc8\\\xe8dRԎ\xdf'\x8a\xe2\xd6?Sd\x12p纟\xcd\x1cb\xf4\x92uLW\xf4/\xd6\xf2\xbf\xe2:\xa6\xd5\xf1\xc6`*\xf29Z\x01\xf1j!)'\x86\x1e\xbe/\x89\xb5\xc7\xd0q^\x9e\xa1'\xeb\xc3:\x11\xfc\xe0x#\xd2\x14\xa7\xb5*\"Q\xa1\x97\xc94\xf3x\x92\xa6\xbaI\u070f\xed\xdb\xdb\x12ˊ\x8e\x00z\x06/\xa2d\xe7)\xb9\xc1\x9a\xf0\xacZ@F\r\x1b\x13\x9a\xb3\xf2\xb6\xe3\x98\xff>)\xcfħ*.~\xa4¢gBT\x98H\x13\x8a1\xcf\v-^.Hv.\x9ap\t\xf7\xc9\xe7\xed\xce\xc4W\x0e.~\xdf`\x19Y\x11DM=m\x90\xf2|\xc1?\x14\x9f3\"Ie\x05\xe4M\t\t\x8f\x8e}\xedt]~v\xcc\x03\x8e\x9dI\xd1\x7fx\f\xf1\xdaQ\xafh\xf0\xf6\x1f8sHw\x90\x91<\x11\xa8]\x90ֲ\xb7/!ehǫ&\xcb@\xa9CS\xfaj\x99L\x02Ր\xfb\xee\xd1jq\xbf\x87\x15e!q-\xb2\xe9<\xedv\x93@\x19\x15\x11\x933\"2\xa3\xb5n\xa4c\xf3\xac\x91\xd2l\xd9\xfe'\x0e\xa3\xe7\xe8z`\xa7\x85\x96+gt\xc58J\xd3*b\x89\r/'\rG\x98G\x1fe\xde)\xdf\xe9\xbe\xda\x15\xaarbam\xaaBEe\xbe\xed\xc0\xb6`\x8c\xf1\x83\xa0!'p\x02N\x047\xf7\x1f \x9f+\xdf\xfdf\xc2f\xf2\x04\xf2\xad\npLA\x11ڊ_5\x95:,}\xcc\x11\a!+\xaa\xefHN5lp\xf4e\x1a2\xfe\xa2\x9e\x94\xcb\x19\x0es\x91¹\xc1\xe6\xf6\x83!oY\xba\xeb\x0f\x15(E\x8f\xde|\x7f\x06\t\xe4\b\x1cQ<\xf7hX{\x83ĝ\xe0Pw\x86آ\x99n\xa8\x9b\xc0jʐ\xfb\x89\xe5\r\xecK\x94\xc6#8N\x9e\x1b\xc65\x1cGY\x17w{\xe5\x11\xa8\x1a\xbe\\:Bħn_\x173\xb38\xb0\xcfdP[$f\x1e\xba\xd4,x)\x13\xd2\bg^U\xfaU\x17T-\x89\xcb\a\xec\x13\xaeuu\x0ee\x90\x94\x8f\x13k\x8a_3ِ\xcf\xf0\x1c\xf9\xfa\xc90\xbd\x89\x83Ə҆\xec\xf8\x83\x14G\tj\xcc\xd2\x1bsǃ\xf1\xe3'!\x1f\xca\xe6\xc8x(\xc1[\xd7\xf9\x81J\xcdhY\x9e\xedz\"c?\xf8\xc3\x1c\xf9oy\xf4\xc4\x1fsDr{^\f\n\xd8nmL\x85q{\xd0ͥ\xaa\xbdht\xf7T\xbcU큉9\xd0\x0eږ|\x16\x1a|\xac\x9e\xf5\x812t\x9e\x95\xde\xc0\xe1 \xa4\xb61\x9c͆\xb0\x83\x13ԱX\x03e\xa5\xb15컫h\x80\x84꒠\xf9\x9c?)ͩ0FJE\xcf\xd6-\xa5Y֠\x1cx\xa74\x8d)\xb4\x17\x99\xb6Ƹqܜ\xe2Q\xee\xba\xfd\x83K\xd7T{\x90\xe6R\a\xfemQg\xee{Y\x11\x14\xcdS\x12s+\xacsݔ(<\xce\xf1\xb0ڜ\xf01\xff\vM\xcbݴ\xa1ֿ\x90\x10:\a\xdf\x18\xbf\x8c\xb7\xd1{ar\xba\x16\x91)?\x14i\x96\x15\x94\x1f\x91}\xa4h\x8e\x85g\xc1)I=\x15FmĻ6'U\xf9\x9c\x97n$\xef\x84l]\x16,o\x97;\at\x1e\x853v\xa6l+r[\x99qo\xefj\xc5x&f\xedL\f\x9e\xc0\x7f,\xa0\x14\x86Pu\xe6\xd9|\x99\xb0\r\x1e\xb1\x99\xdb@sȈ\xee7H\xc0K\xf6\x1b\x06\xa7ﷵz\xcbskK\xad\xd9\xfc\xb4\x9d\xfd\n\xe8\xb0\"\xfd\x12\\ؑS\a\xcf\xec/\xb2\xf2U\xe4v\xd1\x06\xe0h`\x9ab\x90QL\x83x\xb3m\x1d.T\xcf\xca\\r\xbaz\x9d_fM\x9b\x89і\xfey\xad\xe0S0c>\xa6\xd8\xc3\xdf\a\xdd\aw.\xd02n!:\x1b6\x82\x9c\x7ff\a\xffT\xff\xbe\x84\x7f\x19\xf5\xf8\x9d\xefN\xcc\x0f\x0f\x90\x00\x8cHT\xf1ڐp\xb4\xado\xbb\x9f\x1c\x80\xadR\x05py\xd1V:\xbcw\xb2\x97\xed\xa1\xe4\u05fe\xb2\xaa@~\xb8\xdd|\xfb\xf7\xbb\xdeg6\x90%O)&\f\xe3\xec\x1bM\f\xa6\xfdLev\xcf-Ӏ\x9c\ai\xb1F\xa5a\x15\xa8\x9b7 \x19S\x9aU\xa0\x85\xcaE\x16\xb8B\x8d\xcd^\xd5Eζ\x80\fZ7\r*\xad*\xd0V\x84\xa9\xe7JG\xa3t\xbe\x0e0\xfe\x01\a\xe5j9I\x04C\xc2\xe7'\x14\xe4\x9e\x0en~\b\xd3\xe2OL\xea\x01fX\x89K\xa6\xb6\xbfBf\xd7\xec\x0e4\x82\tXgJ\x1e@#\x052\xf5 \xc5\xff6\xb0\rJ\xbd%a\xb4\xe0\xf5A[h\x02K^\xb0\x03/j\xb8b\\\xe6\xac\xe4G\xa6\x01{a\xb5\xec\xc0\xa3*f\xcd~Q\x1a\x98\x90;u\xcd\xf6\xd6V\xe6\xfaݻ\aa\x83&\xcdTY\xd6R\xd8\xe3;R\x8ab[[\xa5ͻ\x1c\x0eP\xbc3\xe2a\xc5u\xb6\x17\x162[kx\xc7+\xb1\"\xd4%i\xd3u\x99\xffS\xe0\xa8\xf9\xa1\x87\xeb\xc9|s\x85\x14\xe1\x04\aP#:\x81qM\xdd(ZB\xe3'\xa4\xce\xd7Ow\xf7]a\x12fH}\xa2{G\xc2Z\x16 \xc1\x84܁\x9f\xd1;\xadJ\x82\t2\xaf\x94\x90\x96\xfe\xc8\n\x01rH~SoKa\x91\xef\x7f\xa9\xc1X\xe4՚ݐyA9\xac+\x9c\x81\xf9\x9am$\xbb\xe1%\x147\xdc\xc0\xab3\x00)mVH\xd84\x16t-㰲\xa3Z\xe7\x87`\xde\"\xfc\ns\xfc\xae\x82\xac7e\xb0\x9d؉\x8c&\x06i\xcfF\x05\f4\xa8+㳖~!55\xfc:\xc0\xc3\xe9\xb2\xd0+\x18\xb4\x1fvO\x1cn\xcd\x18ʕ\x83\x86:E\xaa!wǴ`\x87\x12\x1e\xca\f&}\xad\x97j\xdfN`2\xaf\xea\xd6\x11\x1cO\xb8J?AY\xa1ژA\xf1\xdeWC\x14\x91>y\xe35\x05\xc3\x1fԬ\xf2ڕ\x9d(7\xean\x0fȷ\x83Ƚ\xf6:\xe1*\x9b\xe4,\x96̈;\xc9+\xb3W\x16m\x9c\xaa\xedX\xad\xc1\x00n\xee6\x83F\x1d\xce#VdÉ\xd1V\xb1'.N9\xed\n\xca\xe5\xcd݆}C\x97\b\x02L\xe6,9\xb3\xb5\x96\xa4\x8e\xbf\x02Ϗ\xf7\xea\xcf\x06X^\x93V\nv\xf9*\x02x\v;\x9c\xf4\x1a\x10\x066\x00\xadq\x0e\x18BM\xd5vM\x0eG\x0e;^\x17\xd6+9a\xd8\xfb\x1fY)dm\xe1\x94\xefl\x9a\xf7D$ny\xa9\x0e\xa0\x13h\xf8\x91[\xfe\v\xd6\x1d\x90\x0ea0\x02\xe2\xd9Od\xdc\x1e#\x03\xc5&['\xa9l\xb3\xeb@\x15\x86]^\xe2<\xbbt.\xf1啫[\x8b®\x84\xa4~\"0]\xefO\xa2(B\xff\xe7Q\xc3\x11\xd7\xf1\xd6ܫ\x9f\x8c\x13\xeb\x14\xe2D\x9a\x8e(\x98J\xe5\xec@\xf5b2&\n`\xe6h,\x94\x9eR\x1dυ\x88Kڱ(<\x18öǀ\xfb\xf8\xb8e]\x14|[\xc05\xb3\xba\x1e\xefvJ\x91\x8d\xd1\xe6+\x18+\xb2\x04\xca\\\x0eI\xe3Z\x8e\x10F\xd3\x0f\x11\xa2\f(\x80.\x0f\x7fD\xb7\xdbS\b}\xa7\xa2\xe8\x10w\x9e*\x8c\xfd\x8fd\x1f\xd1\xdcgh\x84\xaf\xbdq\x17P\x90C!\x15+\x94|\x00\xedzD\xc7)H\x98\x06\x94\xb8<\x02\x15-\xad\x86\x02]\x06\xb6\xab\xd1\b\xaf\x19j\x82\xa8\x8c\bi,\xf0|}\xf9Z̃\xefYQ\xe7\x90\xdf\x14\xb5\xb1\xa0\xef0\x04\xccC\b<\xaae\aL\xfc4\t\xc0\xbb_\x85\xc8\x00\xf9\x90\xb9J+\x8a4cDj=\xb1c\x05.\xf0E\xa6zL[\x17\xab\xa3*\fX\xacr\xf9\xaf\x971%\x8a\x12\xd0\xef\xbdߏa\\CC\x8d\x9eF\x8d@l\xf4,\x94\x95=\x8eˑ\xb0PF\x888\xabr\x16\xb0\x97k\xcdǔj\x18N\x13џ\xcf\xde\x18\x88\x01\x83e\xa8\xf6\x1b\xb1x\xd8\xff?\"\x93\xcfb\xab\xa1u,.$\xb2\xb3\x10\xc6\xf6\xb89\f\x88\x1a\xcc0vF\x9ab\xd0\"\xa4\x83\x89ʭü\xdf3\xcdΙ\t1\xd1o$͋\xf3\x9eDŽ\xea\x0fH\xb0\xbdR\x8f)D\xfao\xac\xd7\x06\xca,\xa3%U\xb6\x85=?\b\xa5\xcdp\xb5\x05\xbeCVۨ\x9e\xe0\x96\xe5b\xb7\x03\x8d\xb0h\x81\xb0YO\x9c\"\xd6t\x98\xc0:\n(Za0\xae\x96\xe9\xc8<\xa2Fl(\x14\x8eE\xa12B\x1c\xbdx\xb2\xee\xb98\x88\xbc\xe6\x05\x19z.37>\xde\xe0\x17sOf\x04\xe2\x04\x7f\xe7N\x84Q \x97zQ\xb6\x92\x80\xeeu\xa9t\xcc\xf3t\xe5\x14L\x9c\f[N\xc1q,$m\x8b\xae\v0\x1e\x15\xe7\xc0\xb6z\xe7\xaa\xe5\x94[\xa0*\xf8\x16\nf\xa0\x80\xcc*\x1d'O\x8a\x10\xb8\x92\xaa?#\x94\x1dѤ\xfd hV\x89\xb6\x05\x03̽\xc8\xf6\xce\xddD)#X,W`Hc\xf0\xaa*\"V\xa8-\xb3\x92\xe1;\x9bS\x1amIP\x1fC\xb81EҖD\x1dܖ\x19mܧz#6oD\xef\xa1)\x9f%웓\xe6//\xecHn\x01\x86\x9c>\U000bab98\xb0\xe1k\nԞ\x1fh\xfe\xce\x18w\xdel\xd9\f[\xbf\xf8ly\x11\xae5h\xfc\x9d0\x8d\x8c՝\xb7U\x8b\x18\xf6s\xb7\xe5\x15\x13\xbb\x86a\xf9\x15ۉ\xc2\x02\xf9Rs\x88v\x1c\x9dYν$\x81Rm/\x96\x92\xdbl\xff\xa9Y\xd6Nh1\xa0\xd5\x10\x80\xf3\xcbC\fC\f<\x9d.\n4\xc0$\x90\x9dA\x91\x9b\xd6\xc4xn?\xef\x8aq\xf6\bG\xe7Y\x8d.\x0f\x8d\x15d-o@j\xa0\xcdER#\x8fp$P~\xb70\t\xde\x12Qq\xe5\x11\x8e\xa9U\aDE\xfc\xfc>\x85\xa3.~\xa0Q\xa4L\xa5\xb64D\xf5s\x87Y\x956X\xb6L)\x85\x12(~\xe6\xb0\x1b\x86\xf5\xb6\xc8\x1f\xe1\xf8\x83q\xec\xc3Y\xb3\x17\xd5\x02\n\xa0¦%\x19\xb5k\xf6\x86\xbf\xf1B\xe4Mg4O\x16@\xdc\xc8+\xf6YY\xfc\xe7\xd3wa\x10E\x99\xb3\x8f\n\xccge\xe9˫\x92\xd8\r\xe2L\x02\xbb\xc64-\xa53\vH\x97E\xfd\xb78\x90\tE\x11m\xd8&\f\xdbH\x8c\xcf\x1c}\x96\xb0i\x0f\x019\x87VY\x1b\xda]\x96J\xaeܒ\x96\xefm\x01\xd0.^\x9eUJ\xf78u\xb5\x10\xe2(\x8a\x1e\xbd{\xb4V\ue5d3}\xf9\xa9\xa2\xa1*x\x06y\xd8e\xa3$\x00n\xe1Ad\xac\x04\xfd\x00\xacB\xbb\x91.T\v4\xb9+gHa\xbak\x11\x8a7\v#{\xdace\x85\xb3>\xb1f`sR\xf5Ȏ\xfft\xf5\xb4Q\x92y'\x7f(\x89\xfa\xdd\x14\xb5e\x96e!\xbfN}\x10\x87\xa4s?JN\x1bO\x7fE\xf3J\xe2\xfd\xb74kȅ6k\xf6\x81\x12\xf4\n\xe8\xb6\x0f\xab\x84\x9d\xae\x92@\"&\xc20\x94\x93\x03/\xd0}@\xe5-\x19\x14ΙP\xbb\x13\x0f*M\xc5<\xed\x95q6\xbf\xd9\x18\xbb|\x84\xa3ߜ\xedj\x89ˍ\x8c\xae\xda\xf7\v\xea\xfc\x13\xa5\xd5x-J\x16GvI\xbf]\x92c\xb6d\x8a\x9c\xe1\xbc-\x90\xea\x05U\xbf\xaf\x1e\xeb-h\t\x16̪\xe4\xd5\xca\xcf\x06\xab\xca\xe8\x1e\xa7+\x94F\xb7$\x8c\xc08=x<ظI6C\xf7\x7f\x8e\x02\xc9\xf3\xa1R&\x92i\x11A\xebV\x19\xeb\x16\x0f{\xae\xfa\xc8\xeabJ\xe4\xe8W\x1c\x19\xdfY\xd0\xccX\xa5Cb\x17\xaa\xec\xc1\xe2:J\x8d\x99\x97\x1b\xb7O\xe4W2\x1d`\fP/[\xed\xe2\xec\xc1\xa5۫\xc2\xff\xcf\xc3\xcc\xc8\xd1\"ؕV\x19\x98h6B[\x12\xad\xce\xccbo\xb3\xd0\xcb]\xe0\xb7KR\xeb)\xcbС,s㑴g\x04E\x9f\xbew֬Q\x85\xe1\xdf)\xa2|\x0e\x8e\x8cr\xbb˒\x0f\x93\f\x93ѽq\xad\xc3\x04\xf4\xc0\\\xb0\xa5\x1fjRH\xcb|n/\x92\xbf7\xa7\xa5\x14rC\x1d\xb1\xf7\xaf\xe6\xe8\xb0`\x06b\x19Ice\xc0\x0e߾eH\xf3!5\xf6e!UM\xd1>\x8f\x86\x1egOwA\xd29\xc5\xd0\x11\x97\xcav\x17z|O?\x18\xb6\x13\xda\xd8\x16\xe1\x05P\x85\x99\xc8z\x1a\x1d\xde\x19\xf1\xa9\xfc\xa4\xf5\xd9\xe1\xe9\x17\u05fa\xb3$\xb9WO>\xc1sIP\x1e\x88\xbf\xe7\a`bDŽe 3UKZ,Cu\x81\xdd,\x80\xe8\x98\xe8\x8cI\xa2\xcd\xec4\x96u\x99N\x90\x15I\xa7\x90\xb3+k\xdd&?q\x91\xb6\xb2\xc5\xcec\xab\x9dJ\xa2\x1c+\xfd\xccP\x9fM\xd9\xcd\xe4-\xf9wQ\xd6%\xe3%\xb2eI̹sy\x98!\xed\xd7\xf1\xfa\x89\v\xebOS\xb8M\xd9e\xda4SeU\x80\x85\x90a\x99)iD\x0e\x8d\xfb\xe0\xf9?\x9a\xaf\x1a+\x9c\xed\xb8(j\xbd@G/\xe6\xccҘϫ\xa7\x97\x0f\xe4\xd2\x11Y\x111\x13\x17\xec\x178\xdc\xf3\xf6\xa3\xd2\xcb\\\xe6[\r/\xef\x9aVZ(ʁ\x9d\xf1Nga\x92\xf7\xda\xf7N\xbd\xf0ry\x8c\xb9\xa7\xb3P\t\x937\xf7\xb4)o\xee\xe9\x9b{\xfa\xe6\x9e\x0eʛ{\xfa枾\xb9\xa7\xe3\xe5\xcd=\xed\x947\xf74\xd9~\xa4`\xb8\xa2\x95ۉ\nIX%\xa6o̡=ӗ\xcfR\xf2gA\x96dWo\xc6[\x8e\x9c\x05Zt\x86\xc4t\x8c^\x93n\x8dS2L&w\xa64\xc1\v\x7f\x81\xb36\x01\x81\xb3\xcf\xdal&\x01\xbc\xe0Y\x1b\x8f\xe9p\xed\xfc\x05O\xda\x04Z,?\x84q\xe5ӘJ\xe0aK\xc8\xe5\xa0\xe4\xb1nc^l\x0f\x8f\xd1:\xbfq\xd6\xfdI\xb6\xe6\xf9\"\xf3\xffr~'\"6'\xa7S#P\x85A\xb9\xfacp\xe2,\xdaG\xa9\xed\xfe\x17\x1b]KX\xa7x\xdd5\x03\xddT\xcb~\xca\xeb\x1fG\xb0ϑ\xe4\xd4\xf371\xe7<\xae\xdb:\xc4Խ\xf3\x1e\xbfoZZ(\xbfTޒ\xa5\x9f{ߌ4{\xc6\xc9wn\x8e2\xdbk%Um\xfc\n\x0f\xf6\xf0!sW\x01\x84\x8e\xcc\x12e\xf0\x9e\xedU\x1d9\xe31Cׄ\xcc\xdbx\xbe\xad\xcf\xe0\x00\xcb\x0f\xef\xd7\xfd_\xac\xf2ٷ\x11\xac\x9f\x84ݻ\xfb\x18x\x9e\xa3\xa3\xde9\xe2\x13&\xaf\xbf\x93e(x\x11\x88J3)\n'\x95\x01B߀~\xa9ܒ\xdf\xd9~\xcb\xfc\xc2Sz\x8e\xee\xd2\xcc\xdc&\x97r\xdeK~F>\xee\xb2\xc3R\xb3\xb9\xb7)H\xb3\x94\x8c\xdb\xf1\\\xda\x19\xa8K\xf2lS\xd7\x14\x13rj\xd33i\xd3\xc8\xc3\xe8&\xa5\xd4\xfc\xd9\xe4(45W\xf6u2d\x13\xf3b;ٮ\xb3 \xcf̆M&XZ\xe6kr\xbek'\x8bu\x9eZ\x13Y\xae㹫\xb3 \xc7r[S2V\x93pM\xceSm\xb2O\xe7wF\x9e\x95\x9d\xfa\xf2\xe7`^r\xddb:\xd74)\xc34imc\x1e\xe7\xa4\x1cҥ\x99\xa3IT]\x9a%\xdad\x80Nt\x9c\x94\x1bz\x9a\xf795\x94ٌ\xd0x\xb6\xe7\x14ر<Є\x1c\xcf\t\x90\xdd\xec\xcf\xc5n\xc0\xac4\xcdVX\x9a\xbb9~?Z(\xf3ֹ\xf8-d\xf6\xb9dR\xba\xe74\xa7\x04w_\x06MPZ\x82\x9f8\xe6\x88\xc7Ce瞟\xe1\x88G@nv\xac\xac\v+\xaa\xa2sA\x99\xddñ\xb9\xf2\xe7WE\a\u05f7G\x82\xf6\xe5k#\xf21\x90\xfd\x90\x82\x1b\xf6\x04E\x81\xff\x9eP!s\xd7\x01fj\x05h\xa5\xe2\x1b\x81\xfe\xaa#\x7f\x97\xe0\x95[\x16\xa3S\xfdd\x01K\x844}\x01֤)\x99v\x8f\x9dWO\xdf\xfeR\x83>2\xbas+\xf8AQ1kO{\xfa\xc9l0&\f\xca\xc7k1w)e_\x19\xc5gC\xa3\x02\xd8\a\xe9\f\xf3\x10W\x82\x85Z\xa7\r\xa7\xa6\x94-FO1\x10R5\x10\"\xedS\xbc\xef%\xc7\x1f_#\xb8z\x89\xf0*\xc9\x11y\x8d\x10뵂\xac\xa5a֒䍤㋯\x11l-\t\xb7\x16\xf9\x8c\xe9\xc7\x13_\xebX\xe2+\x84]g\a^\x8bH\x97z\xecpq\xf8\x950\xbe\x99c\x86'>Z\x02\xc8\xe8\xf1\xc2\xf1\x10,\x01\xe2ɱ\xc2\xd9 ,e\x1e\fôg\x1f\x12LNdZ\xb4\x9b\x9e\x9a\x84\x94\xb6\xd1=\x7f\xf8/\xf1\xd0_\xe26x\n\xf6\x89\x87\xfb\x96\x1f\xeaK\xa4\xf3\x99\xe1\xd9d\u05c9\x87\xf7\x16\x05hg\x86h\x93\x10\xa7\x0e\xebM\ai\xd3\vp\xc3Czg\xb8\x13\t\x12\x96Pe\xf9A\xbbgo\xc6(\x9d\x83\x9e\xdd\xd7Z\"γ\x82<\x88\xa3\xfa\xfd\x0fvt\u008d\xa8X\xab\xbbg\x16\xe3\xa8j\xee\x1d\xc9؟\x84\xf4\xbb\xf5(\xb8\x1d\x9f\xa4\xb7\xf1\xd6:L\xf1}\x9d\xd6K\xf5\x17\xab\xbb\x1d;\x03\x15״\x8f\xbf=\xba\xa4 \xb3f\x9fx\xb6oz\x88\x80\xa4~\xf7ܰ\x9d\xd2%\xb7\xec\xb2\xd9\n}\xe7:\xc0\xbf/\u05cc\xfd\xa4\x9a\xf4\x91νb\x11\xa8F\x94UqĈ\x89]v\xc1ti^\xd6ɂ\x88\xc5(\xd8\\\xf8\v\v\a\x97\n\xefTQ\xa8\xa73\x97.x%\xfe\x8b\xde1I[\x1b\xfbp\xbb\xa1\xeaA\xaa\xe8\r\x94&{\xae\x91\xb1-L+\xf4v\xe0\xe4zt\xa1\x8ed\xaf6\x7fN@\xa4W\x04\x82\x9f\xe1\xd5x\xa6P\x8b\xddn\x1c\x96k\x12,.\x8fL\xf9{\xe2\x85\xceW\x15\xd7\xd1M=\xe6\xe5\xc1\\\xf50\fv|n\x05kҬ\x9d\xbe\x8a\xd0-=\x9a\x87\a\x12h\xb3\xf7X\xf5\xb7щ\xd2\x1dz>\a\xa7\xe9\x83˳G\x96_\x01\xa7i\x97iET\x8c\xfc\x14M\xc7{\xf1\xd5C\xe3/\x91\xffE\x1d\xe0ct\x15\xb1\xffd\xc0\xa0\xc9H\x02]\x80:umz\x9b5\x17\xbf\xce\xfa\x052\xe2\x02*\xfe\xe2\xeb\x05\xe3\xf3-\xc6^i\xf0\xf7\x7f\a\xd8\x13\xb6\r\xa7\xec\xed7\n7\x1bu\xe9g\xb8\x0f&\xc3R\xe1\xe0\xaa\xd6\b\xc8\xd83\v/E-\xab4\x7f\x80\x9f\x95{\t#\x85Z\xfd\x16\xbd\xc7P\xbc3\x17\xb2\x89\xfd\\\x8b)s?\xb6!\xc0\xf6\x90\xc1\xc9u\xf4\x88홷\xed[[$\f\xee\xfe\xfeg7 +JX\x7f\xac]\x86\t\xea]\x03H\xe90P\xd7h\x1b\xd7N{\xf5D\xf7\xb9w\x9f\xab\xe8<\b\x04t\xa8\x81\xd2F\xcf\x1a͡\xf7 D ]\x8a\xb0\x7f\x1bo\xd9\xf1\xa1:L\x9cJ!S\xbb(,n\x8c\xca\x04\xb9]\xb4fNg\t^\xef&\xe2)\xf7yB\x7f\xd6\x06\xbe{\xd6.\x87\xce>\xb26¿\xe6Y\x9d\xe8;3.\x90p\x8f\xa0\xad\x10\xfey\xec\x1c\x9d\at\x8d\xf5\xdc\x13DX\xa79\a\xe6\tM\r\xc3\xf5\xd7w1\xd4\xc7\x0f\xf6\xac\xd8g8u\xe4W\xec\x93\xc4A\x9c\xdawwz\arZD\x1d{\x90lr\x88\x87\xa6\x15\x1d\x9d\x1a\xd1\x16}57\xa8>H\xec\xa4\xe7L\x9a*\xee\x98\xd4\x18[\xffY\xec\xdc\nw\x86c\xfa\x97\x93\x1aQ\xc55\xa9\xb4b\nktJ\x9d|4\xa0\x0f\xf4~H\x10\x12oû_\xeam{\x1b9\xfb\xeb\xdf.\xfe/\x00\x00\xff\xff\x80\xea<õr\x00\x00"), diff --git a/design/CLI/PoC/overlays/plugins/node-agent.yaml b/design/CLI/PoC/overlays/plugins/node-agent.yaml index dbb4ce18db..0db26e2c54 100644 --- a/design/CLI/PoC/overlays/plugins/node-agent.yaml +++ b/design/CLI/PoC/overlays/plugins/node-agent.yaml @@ -49,6 +49,9 @@ spec: - mountPath: /host_pods mountPropagation: HostToContainer name: host-pods + - mountPath: /var/lib/kubelet/plugins + mountPropagation: HostToContainer + name: host-plugins - mountPath: /scratch name: scratch - mountPath: /credentials @@ -60,6 +63,9 @@ spec: - hostPath: path: /var/lib/kubelet/pods name: host-pods + - hostPath: + path: /var/lib/kubelet/plugins + name: host-plugins - emptyDir: {} name: scratch - name: cloud-credentials diff --git a/design/volume-snapshot-data-movement/volume-snapshot-data-movement.md b/design/volume-snapshot-data-movement/volume-snapshot-data-movement.md index d006b29e21..6a88b8b48e 100644 --- a/design/volume-snapshot-data-movement/volume-snapshot-data-movement.md +++ b/design/volume-snapshot-data-movement/volume-snapshot-data-movement.md @@ -703,33 +703,38 @@ type Provider interface { In this case, we will extend the default kopia uploader to add the ability, when a given volume is for a block mode and is mapped as a device, we will use the [StreamingFile](https://pkg.go.dev/github.com/kopia/kopia@v0.13.0/fs#StreamingFile) to stream the device and backup to the kopia repository. ```go -func getLocalBlockEntry(kopiaEntry fs.Entry, log logrus.FieldLogger) (fs.Entry, error) { - path := kopiaEntry.LocalFilesystemPath() +func getLocalBlockEntry(sourcePath string) (fs.Entry, error) { + source, err := resolveSymlink(sourcePath) + if err != nil { + return nil, errors.Wrap(err, "resolveSymlink") + } - fileInfo, err := os.Lstat(path) + fileInfo, err := os.Lstat(source) if err != nil { - return nil, errors.Wrapf(err, "Unable to get the source device information %s", path) + return nil, errors.Wrapf(err, "unable to get the source device information %s", source) } if (fileInfo.Sys().(*syscall.Stat_t).Mode & syscall.S_IFMT) != syscall.S_IFBLK { - return nil, errors.Errorf("Source path %s is not a block device", path) + return nil, errors.Errorf("source path %s is not a block device", source) } - device, err := os.Open(path) + + device, err := os.Open(source) if err != nil { if os.IsPermission(err) || err.Error() == ErrNotPermitted { - return nil, errors.Wrapf(err, "No permission to open the source device %s, make sure that node agent is running in privileged mode", path) + return nil, errors.Wrapf(err, "no permission to open the source device %s, make sure that node agent is running in privileged mode", source) } - return nil, errors.Wrapf(err, "Unable to open the source device %s", path) + return nil, errors.Wrapf(err, "unable to open the source device %s", source) } - return virtualfs.StreamingFileFromReader(kopiaEntry.Name(), device), nil + sf := virtualfs.StreamingFileFromReader(source, device) + return virtualfs.NewStaticDirectory(source, []fs.Entry{sf}), nil } ``` In the `pkg/uploader/kopia/snapshot.go` this is used in the Backup call like ```go - if volMode == PersistentVolumeFilesystem { + if volMode == uploader.PersistentVolumeFilesystem { // to be consistent with restic when backup empty dir returns one error for upper logic handle dirs, err := os.ReadDir(source) if err != nil { @@ -742,15 +747,17 @@ In the `pkg/uploader/kopia/snapshot.go` this is used in the Backup call like source = filepath.Clean(source) ... - sourceEntry, err := getLocalFSEntry(source) - if err != nil { - return nil, false, errors.Wrap(err, "Unable to get local filesystem entry") - } + var sourceEntry fs.Entry - if volMode == PersistentVolumeBlock { - sourceEntry, err = getLocalBlockEntry(sourceEntry, log) + if volMode == uploader.PersistentVolumeBlock { + sourceEntry, err = getLocalBlockEntry(source) + if err != nil { + return nil, false, errors.Wrap(err, "unable to get local block device entry") + } + } else { + sourceEntry, err = getLocalFSEntry(source) if err != nil { - return nil, false, errors.Wrap(err, "Unable to get local block device entry") + return nil, false, errors.Wrap(err, "unable to get local filesystem entry") } } @@ -766,6 +773,8 @@ We only need to extend two functions the rest will be passed through. ```go type BlockOutput struct { *restore.FilesystemOutput + + targetFileName string } var _ restore.Output = &BlockOutput{} @@ -773,30 +782,15 @@ var _ restore.Output = &BlockOutput{} const bufferSize = 128 * 1024 func (o *BlockOutput) WriteFile(ctx context.Context, relativePath string, remoteFile fs.File) error { - - targetFileName, err := filepath.EvalSymlinks(o.TargetPath) - if err != nil { - return errors.Wrapf(err, "Unable to evaluate symlinks for %s", targetFileName) - } - - fileInfo, err := os.Lstat(targetFileName) - if err != nil { - return errors.Wrapf(err, "Unable to get the target device information for %s", targetFileName) - } - - if (fileInfo.Sys().(*syscall.Stat_t).Mode & syscall.S_IFMT) != syscall.S_IFBLK { - return errors.Errorf("Target file %s is not a block device", targetFileName) - } - remoteReader, err := remoteFile.Open(ctx) if err != nil { - return errors.Wrapf(err, "Failed to open remote file %s", remoteFile.Name()) + return errors.Wrapf(err, "failed to open remote file %s", remoteFile.Name()) } defer remoteReader.Close() - targetFile, err := os.Create(targetFileName) + targetFile, err := os.Create(o.targetFileName) if err != nil { - return errors.Wrapf(err, "Failed to open file %s", targetFileName) + return errors.Wrapf(err, "failed to open file %s", o.targetFileName) } defer targetFile.Close() @@ -807,7 +801,7 @@ func (o *BlockOutput) WriteFile(ctx context.Context, relativePath string, remote bytesToWrite, err := remoteReader.Read(buffer) if err != nil { if err != io.EOF { - return errors.Wrapf(err, "Failed to read data from remote file %s", targetFileName) + return errors.Wrapf(err, "failed to read data from remote file %s", o.targetFileName) } readData = false } @@ -819,7 +813,7 @@ func (o *BlockOutput) WriteFile(ctx context.Context, relativePath string, remote bytesToWrite -= bytesWritten offset += bytesWritten } else { - return errors.Wrapf(err, "Failed to write data to file %s", targetFileName) + return errors.Wrapf(err, "failed to write data to file %s", o.targetFileName) } } } @@ -829,42 +823,43 @@ func (o *BlockOutput) WriteFile(ctx context.Context, relativePath string, remote } func (o *BlockOutput) BeginDirectory(ctx context.Context, relativePath string, e fs.Directory) error { - targetFileName, err := filepath.EvalSymlinks(o.TargetPath) + var err error + o.targetFileName, err = filepath.EvalSymlinks(o.TargetPath) if err != nil { - return errors.Wrapf(err, "Unable to evaluate symlinks for %s", targetFileName) + return errors.Wrapf(err, "unable to evaluate symlinks for %s", o.targetFileName) } - fileInfo, err := os.Lstat(targetFileName) + fileInfo, err := os.Lstat(o.targetFileName) if err != nil { - return errors.Wrapf(err, "Unable to get the target device information for %s", o.TargetPath) + return errors.Wrapf(err, "unable to get the target device information for %s", o.TargetPath) } if (fileInfo.Sys().(*syscall.Stat_t).Mode & syscall.S_IFMT) != syscall.S_IFBLK { - return errors.Errorf("Target file %s is not a block device", o.TargetPath) + return errors.Errorf("target file %s is not a block device", o.TargetPath) } return nil } ``` -Of note, we do need to add root access to the daemon set node agent to access the new mount. +Additional mount is required in the node-agent specification to resolve symlinks to the block devices from /host_pods/POD_ID/volumeDevices/kubernetes.io~csi directory. ```yaml -... - mountPath: /var/lib/kubelet/plugins mountPropagation: HostToContainer name: host-plugins - .... - hostPath: path: /var/lib/kubelet/plugins name: host-plugins +``` + +Privileged mode is required to access the block devices in /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/publish directory as confirmed by testing on EKS and Minikube. -... +```yaml SecurityContext: &corev1.SecurityContext{ - Privileged: &c.privilegedAgent, + Privileged: &c.privilegedNodeAgent, }, - ``` ## Plugin Data Movers diff --git a/go.mod b/go.mod index ab1c6a5149..51cd719132 100644 --- a/go.mod +++ b/go.mod @@ -3,46 +3,50 @@ module github.com/vmware-tanzu/velero go 1.20 require ( - cloud.google.com/go/storage v1.30.1 + cloud.google.com/go/storage v1.32.0 github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-sdk-for-go v67.2.0+incompatible + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Azure/go-autorest/autorest v0.11.27 github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 github.com/Azure/go-autorest/autorest/to v0.3.0 - github.com/aws/aws-sdk-go v1.44.253 + github.com/aws/aws-sdk-go v1.44.256 github.com/bombsimon/logrusr/v3 v3.0.0 github.com/evanphx/json-patch v5.6.0+incompatible github.com/fatih/color v1.15.0 github.com/gobwas/glob v0.2.3 github.com/golang/protobuf v1.5.3 github.com/google/go-cmp v0.5.9 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.3.1 github.com/hashicorp/go-hclog v0.14.1 github.com/hashicorp/go-plugin v1.4.3 github.com/joho/godotenv v1.3.0 - github.com/kopia/kopia v0.13.0 + github.com/kopia/kopia v0.14.1 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.20.1 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.15.0 + github.com/prometheus/client_golang v1.16.0 github.com/robfig/cron v1.1.0 - github.com/sirupsen/logrus v1.9.0 + github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.6.0 github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.4 github.com/vmware-tanzu/crash-diagnostics v0.3.7 - go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f - golang.org/x/mod v0.10.0 - golang.org/x/net v0.9.0 - golang.org/x/oauth2 v0.7.0 - golang.org/x/text v0.9.0 - google.golang.org/api v0.120.0 - google.golang.org/grpc v1.54.0 - google.golang.org/protobuf v1.30.0 + go.uber.org/zap v1.25.0 + golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 + golang.org/x/mod v0.12.0 + golang.org/x/net v0.14.0 + golang.org/x/oauth2 v0.11.0 + golang.org/x/text v0.13.0 + google.golang.org/api v0.138.0 + google.golang.org/grpc v1.57.0 + google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.25.6 k8s.io/apiextensions-apiserver v0.24.2 @@ -58,13 +62,11 @@ require ( ) require ( - cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/compute v1.19.0 // indirect + cloud.google.com/go v0.110.6 // indirect + cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.13.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 // indirect + cloud.google.com/go/iam v1.1.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect @@ -72,6 +74,7 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect @@ -81,7 +84,7 @@ require ( github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/emicklei/go-restful/v3 v3.8.0 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.2.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -90,31 +93,34 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/s2a-go v0.1.2 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.8.0 // indirect + github.com/google/s2a-go v0.1.5 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect + github.com/hashicorp/cronexpr v1.1.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.5 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect - github.com/klauspost/reedsolomon v1.11.7 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect + github.com/klauspost/reedsolomon v1.11.8 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-ieproxy v0.0.1 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.52 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/minio/minio-go/v7 v7.0.63 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/moby/spdystream v0.2.0 // indirect @@ -125,30 +131,32 @@ require ( github.com/nxadm/tail v1.4.8 // indirect github.com/oklog/run v1.0.0 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect - github.com/rogpeppe/go-internal v1.9.0 // indirect - github.com/rs/xid v1.4.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect + github.com/rs/xid v1.5.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/vladimirvivien/gexe v0.1.1 // indirect github.com/zeebo/blake3 v0.2.3 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel v1.14.0 // indirect - go.opentelemetry.io/otel/trace v1.14.0 // indirect + go.opentelemetry.io/otel v1.17.0 // indirect + go.opentelemetry.io/otel/metric v1.17.0 // indirect + go.opentelemetry.io/otel/trace v1.17.0 // indirect go.starlark.net v0.0.0-20201006213952-227f4aabceb5 // indirect - go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.8.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/term v0.7.0 // indirect + golang.org/x/crypto v0.12.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/term v0.12.0 // indirect golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect @@ -159,4 +167,4 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) -replace github.com/kopia/kopia => github.com/project-velero/kopia v0.13.0-velero.1 +replace github.com/kopia/kopia => github.com/project-velero/kopia v0.0.0-20230918080509-48b07dfffc74 diff --git a/go.sum b/go.sum index c6c1f12017..bc4c0ed7c0 100644 --- a/go.sum +++ b/go.sum @@ -19,24 +19,23 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= -cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.110.6 h1:8uYAkj3YHTP/1iwReuHPxLSbdcyc+dSBbzFMrVwDR6Q= +cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.19.0 h1:+9zda3WGgW1ZSTlVppLCYFIr48Pa35q1uG2N1itbCEQ= -cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= -cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= -cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= +cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= +cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -46,19 +45,25 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= -cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= +cloud.google.com/go/storage v1.32.0 h1:5w6DxEGOnktmJHarxAOUywxVW9lbNWIzlzzUltG/3+o= +cloud.google.com/go/storage v1.32.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-sdk-for-go v67.2.0+incompatible h1:Uu/Ww6ernvPTrpq31kITVTIm/I5jlJ1wjtEH/bmSB2k= github.com/Azure/azure-sdk-for-go v67.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 h1:qoVeMsc9/fh/yhxVaA0obYjVH/oI/ihrOoMwsLS9KSA= -github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= -github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 h1:E+m3SkZCN0Bf5q7YdTs5lSm2CYY3CK4spn5OmUIiQtk= -github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 h1:/iHxaJhsFr0+xVFfbMr5vxz848jyiWuIEDhYq3y5odY= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 h1:LNHhpdK7hzUcx/k1LIcuh5k7k1LGIWLQfCjaneSj7Fc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1/go.mod h1:uE9zaUfEQT/nbQjVi2IblCG9iaLtZsuYZ8ne+PuQ02M= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 h1:ECsQtyERDVz3NP3kvDOTLvbQhqWp/x9EsGKtb4ogUr8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0 h1:LcJtQjCXJUm1s7JpUHZvu+bpgURhCatxVNbGADXniX0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0/go.mod h1:+OgGVo0Httq7N5oayfvaLQ/Jq+2gJdqfp++Hyyl7Tws= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= @@ -104,9 +109,11 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 h1:KeNholpO2xKjgaaSyd+DyQRrsQjhbSeS7qe4nEw8aQw= +github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -129,11 +136,11 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.44.253 h1:iqDd0okcH4ShfFexz2zzf4VmeDFf6NOMm07pHnEb8iY= -github.com/aws/aws-sdk-go v1.44.253/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4= +github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -188,7 +195,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= +github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -197,9 +204,7 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -261,8 +266,8 @@ github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= @@ -303,6 +308,8 @@ github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -382,21 +389,21 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.2 h1:WVtYAYuYxKeYajAmThMRYWP6K3wXkcqbGHeUgeubUHY= -github.com/google/s2a-go v0.1.2/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= +github.com/google/s2a-go v0.1.5 h1:8IYp3w9nysqv3JH+NJgXJzGbDHzLOTj43BmSkp+O7qg= +github.com/google/s2a-go v0.1.5/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= +github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.8.0 h1:UBtEZqx1bjXtOQ5BVTkuYghXrr3N4V123VKJK67vJZc= -github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= @@ -414,9 +421,11 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hanwen/go-fuse/v2 v2.3.0 h1:t5ivNIH2PK+zw4OBul/iJjsoG9K6kXo4nMDoBpciC8A= +github.com/hanwen/go-fuse/v2 v2.4.0 h1:12OhD7CkXXQdvxG2osIdBQLdXh+nmLXY9unkUIe/xaU= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/cronexpr v1.1.2 h1:wG/ZYIKT+RT3QkOdgYc+xsKWVRgnxJ1OJtjjy84fJ9A= +github.com/hashicorp/cronexpr v1.1.2/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.14.1 h1:nQcJDQwIAGnmoUWp8ubocEX40cCml/17YkF6csQLReU= @@ -479,20 +488,19 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/klauspost/reedsolomon v1.11.7 h1:9uaHU0slncktTEEg4+7Vl7q7XUNMBUOK4R9gnKhMjAU= -github.com/klauspost/reedsolomon v1.11.7/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= +github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kopia/htmluibuild v0.0.0-20230326183719-f482ef17e2c9 h1:s5Wa89s8RlPjuwqd8K8kuf+T9Kz4+NsbKwR/pJ3PAT0= +github.com/kopia/htmluibuild v0.0.1-0.20230917154246-98806054261e h1:XogFUFI4mcT5qyywKiGY5WqLi7l4b/eMi7BlEzgLTd0= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -507,6 +515,7 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 h1:nHHjmvjitIiyPlUHk/ofpgvBcNcawJLtf4PYHORLjAA= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -530,8 +539,8 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -540,10 +549,10 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps= -github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= +github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -567,7 +576,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -606,6 +614,8 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -615,48 +625,47 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/project-velero/kopia v0.13.0-velero.1 h1:rjiP7os4Eaek/gbyIKqzwkp2XLbJHl0NkczSgJ7AOEo= -github.com/project-velero/kopia v0.13.0-velero.1/go.mod h1:Iic7CcKhsq+A7MLR9hh6VJfgpcJhLx3Kn+BgjY+azvI= +github.com/project-velero/kopia v0.0.0-20230918080509-48b07dfffc74 h1:dLMdxdRegzp3DhTa7nc9c37n8cukHLIK8SXRTOh/3pg= +github.com/project-velero/kopia v0.0.0-20230918080509-48b07dfffc74/go.mod h1:eFPSn8Dy0XlRUbIDZjEaDKmFdJMh5lySvtF6eyl5j/s= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= -github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/robfig/cron v1.1.0 h1:jk4/Hud3TTdcrJgUOBgsqrZBarcxl6ADIjSC2iniwLY= github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -668,8 +677,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -713,8 +722,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tg123/go-htpasswd v1.2.1 h1:i4wfsX1KvvkyoMiHZzjS0VzbAPWfxzI8INcZAKtutoU= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -739,7 +748,7 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zalando/go-keyring v0.2.2 h1:f0xmpYiSrHtSNAVgwip93Cg8tuF45HJM6rHq/A5RI/4= +github.com/zalando/go-keyring v0.2.3 h1:v9CUu9phlABObO4LPWycf+zwMG7nlbb3t/B5wa97yms= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= @@ -774,17 +783,19 @@ go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUz go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= -go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= +go.opentelemetry.io/otel v1.17.0 h1:MW+phZ6WZ5/uk2nd93ANk/6yJ+dVrvNWUjGhnnFU5jM= +go.opentelemetry.io/otel v1.17.0/go.mod h1:I2vmBGtFaODIVMBSTPVDlJSzBDNf93k60E6Ft0nyjo0= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/metric v1.17.0 h1:iG6LGVz5Gh+IuO0jmgvpTB6YVrCGngi8QGm+pMd8Pdc= +go.opentelemetry.io/otel/metric v1.17.0/go.mod h1:h4skoxdZI17AxwITdmdZjjYJQH5nzijUUjm+wtPph5o= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= -go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= +go.opentelemetry.io/otel/trace v1.17.0 h1:/SWhSRHmDPOImIAetP1QAeMnZYiQXrTy4fMMYOdSKWQ= +go.opentelemetry.io/otel/trace v1.17.0/go.mod h1:I/4vKTgFclIsXRVucpH25X0mpFSczM7aHeaz0ZBLWjY= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.starlark.net v0.0.0-20201006213952-227f4aabceb5 h1:ApvY/1gw+Yiqb/FKeks3KnVPWpkR3xzij82XPKLjJVw= @@ -792,10 +803,8 @@ go.starlark.net v0.0.0-20201006213952-227f4aabceb5/go.mod h1:f0znQkUKRrkk36XxWbG go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -803,8 +812,8 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= +go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -822,8 +831,9 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= -golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -834,8 +844,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f h1:Al51T6tzvuh3oiwX11vex3QgJ2XTedFPGmbEVh8cdoc= -golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -863,8 +873,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -903,7 +913,6 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -923,8 +932,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -940,8 +949,8 @@ golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -954,8 +963,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1023,6 +1032,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1032,20 +1042,21 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1055,9 +1066,10 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1161,8 +1173,8 @@ google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjR google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= -google.golang.org/api v0.120.0 h1:TTmhTei0mkR+kiBSW2UzZmAbkTaBfUUzfchyXnzG9Hs= -google.golang.org/api v0.120.0/go.mod h1:CrSvlNEFCFLae9ZUtL1z+61+rEBD7J/aCYwVYKZoWFU= +google.golang.org/api v0.138.0 h1:K/tVp05MxNVbHShRw9m7e9VJGdagNeTdMzqPH7AUqr0= +google.golang.org/api v0.138.0/go.mod h1:4xyob8CxC+0GChNBvEUAk8VBKNvYOTWM9T3v3UfRxuY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1218,8 +1230,12 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= +google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 h1:nIgk/EEq3/YlnmVVXVnm14rC2oxgs1o0ong4sD/rd44= +google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 h1:wukfNtZmZUurLN/atp2hiIeTKn7QJWIQdHzqmsOnAOk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1245,8 +1261,8 @@ google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1260,8 +1276,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/hack/docker-push.sh b/hack/docker-push.sh index d675bffd34..e503358c9f 100755 --- a/hack/docker-push.sh +++ b/hack/docker-push.sh @@ -89,7 +89,7 @@ else fi if [[ -z "$BUILDX_PLATFORMS" ]]; then - BUILDX_PLATFORMS="linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le" + BUILDX_PLATFORMS="linux/amd64,linux/arm64" fi # Debugging info diff --git a/internal/resourcemodifiers/resource_modifiers.go b/internal/resourcemodifiers/resource_modifiers.go index ef81945223..dbcd8e7ba4 100644 --- a/internal/resourcemodifiers/resource_modifiers.go +++ b/internal/resourcemodifiers/resource_modifiers.go @@ -52,7 +52,7 @@ func GetResourceModifiersFromConfig(cm *v1.ConfigMap) (*ResourceModifiers, error return nil, fmt.Errorf("could not parse config from nil configmap") } if len(cm.Data) != 1 { - return nil, fmt.Errorf("illegal resource modifiers %s/%s configmap", cm.Name, cm.Namespace) + return nil, fmt.Errorf("illegal resource modifiers %s/%s configmap", cm.Namespace, cm.Name) } var yamlData string diff --git a/internal/resourcepolicies/resource_policies.go b/internal/resourcepolicies/resource_policies.go index 5da2da8fc6..956a06753c 100644 --- a/internal/resourcepolicies/resource_policies.go +++ b/internal/resourcepolicies/resource_policies.go @@ -132,7 +132,7 @@ func GetResourcePoliciesFromConfig(cm *v1.ConfigMap) (*Policies, error) { return nil, fmt.Errorf("could not parse config from nil configmap") } if len(cm.Data) != 1 { - return nil, fmt.Errorf("illegal resource policies %s/%s configmap", cm.Name, cm.Namespace) + return nil, fmt.Errorf("illegal resource policies %s/%s configmap", cm.Namespace, cm.Name) } var yamlData string diff --git a/internal/util/managercontroller/managercontroller.go b/internal/util/managercontroller/managercontroller.go deleted file mode 100644 index bb80826060..0000000000 --- a/internal/util/managercontroller/managercontroller.go +++ /dev/null @@ -1,37 +0,0 @@ -/* -Copyright 2020 the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// TODO(2.0) After converting all controllers to runtime-controller, -// the functions in this file will no longer be needed and should be removed. -package managercontroller - -import ( - "context" - - "sigs.k8s.io/controller-runtime/pkg/manager" - - "github.com/vmware-tanzu/velero/pkg/controller" -) - -// Runnable will turn a "regular" runnable component (such as a controller) -// into a controller-runtime Runnable -func Runnable(p controller.Interface, numWorkers int) manager.Runnable { - // Pass the provided Context down to the run function. - f := func(ctx context.Context) error { - return p.Run(ctx, numWorkers) - } - return manager.RunnableFunc(f) -} diff --git a/pkg/apis/velero/v1/pod_volume_backup_types.go b/pkg/apis/velero/v1/pod_volume_backup_types.go index a9a2ad4e80..8596178075 100644 --- a/pkg/apis/velero/v1/pod_volume_backup_types.go +++ b/pkg/apis/velero/v1/pod_volume_backup_types.go @@ -114,7 +114,6 @@ type PodVolumeBackupStatus struct { // +kubebuilder:printcolumn:name="Namespace",type="string",JSONPath=".spec.pod.namespace",description="Namespace of the pod containing the volume to be backed up" // +kubebuilder:printcolumn:name="Pod",type="string",JSONPath=".spec.pod.name",description="Name of the pod containing the volume to be backed up" // +kubebuilder:printcolumn:name="Volume",type="string",JSONPath=".spec.volume",description="Name of the volume to be backed up" -// +kubebuilder:printcolumn:name="Repository ID",type="string",JSONPath=".spec.repoIdentifier",description="Backup repository identifier for this backup" // +kubebuilder:printcolumn:name="Uploader Type",type="string",JSONPath=".spec.uploaderType",description="The type of the uploader to handle data transfer" // +kubebuilder:printcolumn:name="Storage Location",type="string",JSONPath=".spec.backupStorageLocation",description="Name of the Backup Storage Location where this backup should be stored" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" diff --git a/pkg/apis/velero/v2alpha1/zz_generated.deepcopy.go b/pkg/apis/velero/v2alpha1/zz_generated.deepcopy.go index 6f2f0441f5..9a9afaa6d9 100644 --- a/pkg/apis/velero/v2alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/velero/v2alpha1/zz_generated.deepcopy.go @@ -1,34 +1,17 @@ //go:build !ignore_autogenerated // +build !ignore_autogenerated -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by deepcopy-gen. DO NOT EDIT. +// Code generated by controller-gen. DO NOT EDIT. package v2alpha1 import ( - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CSISnapshotSpec) DeepCopyInto(out *CSISnapshotSpec) { *out = *in - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSISnapshotSpec. @@ -48,7 +31,6 @@ func (in *DataDownload) DeepCopyInto(out *DataDownload) { in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataDownload. @@ -81,7 +63,6 @@ func (in *DataDownloadList) DeepCopyInto(out *DataDownloadList) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataDownloadList. @@ -114,7 +95,6 @@ func (in *DataDownloadSpec) DeepCopyInto(out *DataDownloadSpec) { } } out.OperationTimeout = in.OperationTimeout - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataDownloadSpec. @@ -139,7 +119,6 @@ func (in *DataDownloadStatus) DeepCopyInto(out *DataDownloadStatus) { *out = (*in).DeepCopy() } out.Progress = in.Progress - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataDownloadStatus. @@ -159,7 +138,6 @@ func (in *DataUpload) DeepCopyInto(out *DataUpload) { in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataUpload. @@ -192,7 +170,6 @@ func (in *DataUploadList) DeepCopyInto(out *DataUploadList) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataUploadList. @@ -227,7 +204,6 @@ func (in *DataUploadResult) DeepCopyInto(out *DataUploadResult) { } } } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataUploadResult. @@ -260,7 +236,6 @@ func (in *DataUploadSpec) DeepCopyInto(out *DataUploadSpec) { } } out.OperationTimeout = in.OperationTimeout - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataUploadSpec. @@ -296,7 +271,6 @@ func (in *DataUploadStatus) DeepCopyInto(out *DataUploadStatus) { *out = (*in).DeepCopy() } out.Progress = in.Progress - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataUploadStatus. @@ -312,7 +286,6 @@ func (in *DataUploadStatus) DeepCopy() *DataUploadStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TargetVolumeSpec) DeepCopyInto(out *TargetVolumeSpec) { *out = *in - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetVolumeSpec. diff --git a/pkg/backup/item_collector.go b/pkg/backup/item_collector.go index 328125a278..c94f364c74 100644 --- a/pkg/backup/item_collector.go +++ b/pkg/backup/item_collector.go @@ -293,6 +293,9 @@ func (r *itemCollector) getResourceItems(log logrus.FieldLogger, gv schema.Group clusterScoped := !resource.Namespaced namespacesToList := getNamespacesToList(r.backupRequest.NamespaceIncludesExcludes) + clusterScoped := !resource.Namespaced + namespacesToList := getNamespacesToList(r.backupRequest.NamespaceIncludesExcludes) + // If we get here, we're backing up something other than namespaces if clusterScoped { namespacesToList = []string{""} diff --git a/pkg/builder/persistent_volume_builder.go b/pkg/builder/persistent_volume_builder.go index 5fee88c196..4cf2e47f20 100644 --- a/pkg/builder/persistent_volume_builder.go +++ b/pkg/builder/persistent_volume_builder.go @@ -95,6 +95,12 @@ func (b *PersistentVolumeBuilder) StorageClass(name string) *PersistentVolumeBui return b } +// VolumeMode sets the PersistentVolume's volume mode. +func (b *PersistentVolumeBuilder) VolumeMode(volMode corev1api.PersistentVolumeMode) *PersistentVolumeBuilder { + b.object.Spec.VolumeMode = &volMode + return b +} + // NodeAffinityRequired sets the PersistentVolume's NodeAffinity Requirement. func (b *PersistentVolumeBuilder) NodeAffinityRequired(req *corev1api.NodeSelector) *PersistentVolumeBuilder { b.object.Spec.NodeAffinity = &corev1api.VolumeNodeAffinity{ diff --git a/pkg/cmd/cli/backup/create.go b/pkg/cmd/cli/backup/create.go index f69a9a7fae..4925179dc0 100644 --- a/pkg/cmd/cli/backup/create.go +++ b/pkg/cmd/cli/backup/create.go @@ -95,6 +95,7 @@ type CreateOptions struct { ExcludeNamespaceScopedResources flag.StringArray Labels flag.Map Selector flag.LabelSelector + OrSelector flag.OrLabelSelector IncludeClusterResources flag.OptionalBool Wait bool StorageLocation string @@ -130,6 +131,7 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.StorageLocation, "storage-location", "", "Location in which to store the backup.") flags.StringSliceVar(&o.SnapshotLocations, "volume-snapshot-locations", o.SnapshotLocations, "List of locations (at most one per provider) where volume snapshots should be stored.") flags.VarP(&o.Selector, "selector", "l", "Only back up resources matching this label selector.") + flags.Var(&o.OrSelector, "or-selector", "Backup resources matching at least one of the label selector from the list. Label selectors should be separated by ' or '. For example, foo=bar or app=nginx") flags.StringVar(&o.OrderedResources, "ordered-resources", "", "Mapping Kinds to an ordered list of specific resources of that Kind. Resource names are separated by commas and their names are in format 'namespace/resourcename'. For cluster scope resource, simply use resource name. Key-value pairs in the mapping are separated by semi-colon. Example: 'pods=ns1/pod1,ns1/pod2;persistentvolumeclaims=ns1/pvc4,ns1/pvc8'. Optional.") flags.DurationVar(&o.CSISnapshotTimeout, "csi-snapshot-timeout", o.CSISnapshotTimeout, "How long to wait for CSI snapshot creation before timeout.") flags.DurationVar(&o.ItemOperationTimeout, "item-operation-timeout", o.ItemOperationTimeout, "How long to wait for async plugin operations before timeout.") @@ -168,9 +170,8 @@ func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Facto return err } - client, err := f.KubebuilderWatchClient() - if err != nil { - return err + if o.Selector.LabelSelector != nil && o.OrSelector.OrLabelSelectors != nil { + return fmt.Errorf("either a 'selector' or an 'or-selector' can be specified, but not both") } // Ensure that unless FromSchedule is set, args contains a backup name @@ -191,7 +192,7 @@ func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Facto if o.StorageLocation != "" { location := &velerov1api.BackupStorageLocation{} - if err := client.Get(context.Background(), kbclient.ObjectKey{ + if err := o.client.Get(context.Background(), kbclient.ObjectKey{ Namespace: f.Namespace(), Name: o.StorageLocation, }, location); err != nil { @@ -201,7 +202,7 @@ func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Facto for _, loc := range o.SnapshotLocations { snapshotLocation := new(velerov1api.VolumeSnapshotLocation) - if err := o.client.Get(context.TODO(), kbclient.ObjectKey{Namespace: f.Namespace(), Name: loc}, snapshotLocation); err != nil { + if err := o.client.Get(context.Background(), kbclient.ObjectKey{Namespace: f.Namespace(), Name: loc}, snapshotLocation); err != nil { return err } } @@ -365,6 +366,7 @@ func (o *CreateOptions) BuildBackup(namespace string) (*velerov1api.Backup, erro IncludedNamespaceScopedResources(o.IncludeNamespaceScopedResources...). ExcludedNamespaceScopedResources(o.ExcludeNamespaceScopedResources...). LabelSelector(o.Selector.LabelSelector). + OrLabelSelector(o.OrSelector.OrLabelSelectors). TTL(o.TTL). StorageLocation(o.StorageLocation). VolumeSnapshotLocations(o.SnapshotLocations...). diff --git a/pkg/cmd/cli/backup/create_test.go b/pkg/cmd/cli/backup/create_test.go index 34652c9596..4b88998d72 100644 --- a/pkg/cmd/cli/backup/create_test.go +++ b/pkg/cmd/cli/backup/create_test.go @@ -47,6 +47,15 @@ func TestCreateOptions_BuildBackup(t *testing.T) { orders, err := ParseOrderedResources(o.OrderedResources) o.CSISnapshotTimeout = 20 * time.Minute o.ItemOperationTimeout = 20 * time.Minute + orLabelSelectors := []*metav1.LabelSelector{ + { + MatchLabels: map[string]string{"k1": "v1", "k2": "v2"}, + }, + { + MatchLabels: map[string]string{"a1": "b1", "a2": "b2"}, + }, + } + o.OrSelector.OrLabelSelectors = orLabelSelectors assert.NoError(t, err) backup, err := o.BuildBackup(cmdtest.VeleroNameSpace) @@ -58,6 +67,7 @@ func TestCreateOptions_BuildBackup(t *testing.T) { SnapshotVolumes: o.SnapshotVolumes.Value, IncludeClusterResources: o.IncludeClusterResources.Value, OrderedResources: orders, + OrLabelSelectors: orLabelSelectors, CSISnapshotTimeout: metav1.Duration{Duration: o.CSISnapshotTimeout}, ItemOperationTimeout: metav1.Duration{Duration: o.ItemOperationTimeout}, }, backup.Spec) diff --git a/pkg/cmd/cli/backup/describe.go b/pkg/cmd/cli/backup/describe.go index 7d26dd963e..47132c27a3 100644 --- a/pkg/cmd/cli/backup/describe.go +++ b/pkg/cmd/cli/backup/describe.go @@ -57,6 +57,14 @@ func NewDescribeCommand(f client.Factory, use string) *cobra.Command { kbClient, err := f.KubebuilderClient() cmd.CheckError(err) + var csiClient *snapshotv1client.Clientset + if features.IsEnabled(velerov1api.CSIFeatureFlag) { + clientConfig, err := f.ClientConfig() + cmd.CheckError(err) + csiClient, err = snapshotv1client.NewForConfig(clientConfig) + cmd.CheckError(err) + } + if outputFormat != "plaintext" && outputFormat != "json" { cmd.CheckError(fmt.Errorf("invalid output format '%s'. valid value are 'plaintext, json'", outputFormat)) } @@ -96,16 +104,9 @@ func NewDescribeCommand(f client.Factory, use string) *cobra.Command { fmt.Fprintf(os.Stderr, "error getting PodVolumeBackups for backup %s: %v\n", backup.Name, err) } - var csiClient *snapshotv1client.Clientset // declare vscList up here since it may be empty and we'll pass the empty Items field into DescribeBackup vscList := new(snapshotv1api.VolumeSnapshotContentList) if features.IsEnabled(velerov1api.CSIFeatureFlag) { - clientConfig, err := f.ClientConfig() - cmd.CheckError(err) - - csiClient, err = snapshotv1client.NewForConfig(clientConfig) - cmd.CheckError(err) - opts := label.NewListOptionsForBackup(backup.Name) vscList, err = csiClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), opts) if err != nil { diff --git a/pkg/cmd/cli/backup/describe_test.go b/pkg/cmd/cli/backup/describe_test.go index 3272c68009..51f9476cf6 100644 --- a/pkg/cmd/cli/backup/describe_test.go +++ b/pkg/cmd/cli/backup/describe_test.go @@ -69,6 +69,7 @@ func TestNewDescribeCommand(t *testing.T) { if err == nil { assert.Contains(t, stdout, "Velero-Native Snapshots: ") + assert.Contains(t, stdout, "Or label selector: ") assert.Contains(t, stdout, fmt.Sprintf("Name: %s", backupName)) return } diff --git a/pkg/cmd/cli/backuplocation/delete.go b/pkg/cmd/cli/backuplocation/delete.go index d34bfa8e5a..922c32df99 100644 --- a/pkg/cmd/cli/backuplocation/delete.go +++ b/pkg/cmd/cli/backuplocation/delete.go @@ -33,8 +33,6 @@ import ( "github.com/vmware-tanzu/velero/pkg/cmd/cli" ) -const bslLabelKey = "velero.io/storage-location" - // NewDeleteCommand creates and returns a new cobra command for deleting backup-locations. func NewDeleteCommand(f client.Factory, use string) *cobra.Command { o := cli.NewDeleteOptions("backup-location") @@ -146,7 +144,7 @@ func findAssociatedBackups(client kbclient.Client, bslName, ns string) (velerov1 var backups velerov1api.BackupList err := client.List(context.Background(), &backups, &kbclient.ListOptions{ Namespace: ns, - Raw: &metav1.ListOptions{LabelSelector: bslLabelKey + "=" + bslName}, + Raw: &metav1.ListOptions{LabelSelector: velerov1api.StorageLocationLabel + "=" + bslName}, }) return backups, err } @@ -155,7 +153,7 @@ func findAssociatedBackupRepos(client kbclient.Client, bslName, ns string) (vele var repos velerov1api.BackupRepositoryList err := client.List(context.Background(), &repos, &kbclient.ListOptions{ Namespace: ns, - Raw: &metav1.ListOptions{LabelSelector: bslLabelKey + "=" + bslName}, + Raw: &metav1.ListOptions{LabelSelector: velerov1api.StorageLocationLabel + "=" + bslName}, }) return repos, err } diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 95f3297936..72fde9ef40 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -66,6 +66,7 @@ type Options struct { BackupStorageConfig flag.Map VolumeSnapshotConfig flag.Map UseNodeAgent bool + PrivilegedNodeAgent bool //TODO remove UseRestic when migration test out of using it UseRestic bool Wait bool @@ -79,6 +80,7 @@ type Options struct { Features string DefaultVolumesToFsBackup bool UploaderType string + DefaultSnapshotMoveData bool } // BindFlags adds command line values to the options struct. @@ -109,6 +111,7 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) { flags.BoolVar(&o.RestoreOnly, "restore-only", o.RestoreOnly, "Run the server in restore-only mode. Optional.") flags.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Generate resources, but don't send them to the cluster. Use with -o. Optional.") flags.BoolVar(&o.UseNodeAgent, "use-node-agent", o.UseNodeAgent, "Create Velero node-agent daemonset. Optional. Velero node-agent hosts Velero modules that need to run in one or more nodes(i.e. Restic, Kopia).") + flags.BoolVar(&o.PrivilegedNodeAgent, "privileged-node-agent", o.PrivilegedNodeAgent, "Use privileged mode for the node agent. Optional. Required to backup block devices.") flags.BoolVar(&o.Wait, "wait", o.Wait, "Wait for Velero deployment to be ready. Optional.") flags.DurationVar(&o.DefaultRepoMaintenanceFrequency, "default-repo-maintain-frequency", o.DefaultRepoMaintenanceFrequency, "How often 'maintain' is run for backup repositories by default. Optional.") flags.DurationVar(&o.GarbageCollectionFrequency, "garbage-collection-frequency", o.GarbageCollectionFrequency, "How often the garbage collection runs for expired backups.(default 1h)") @@ -118,6 +121,7 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.Features, "features", o.Features, "Comma separated list of Velero feature flags to be set on the Velero deployment and the node-agent daemonset, if node-agent is enabled") flags.BoolVar(&o.DefaultVolumesToFsBackup, "default-volumes-to-fs-backup", o.DefaultVolumesToFsBackup, "Bool flag to configure Velero server to use pod volume file system backup by default for all volumes on all backups. Optional.") flags.StringVar(&o.UploaderType, "uploader-type", o.UploaderType, fmt.Sprintf("The type of uploader to transfer the data of pod volumes, the supported values are '%s', '%s'", uploader.ResticType, uploader.KopiaType)) + flags.BoolVar(&o.DefaultSnapshotMoveData, "default-snapshot-move-data", o.DefaultSnapshotMoveData, "Bool flag to configure Velero server to move data by default for all snapshots supporting data movement. Optional.") } // NewInstallOptions instantiates a new, default InstallOptions struct. @@ -144,6 +148,7 @@ func NewInstallOptions() *Options { CRDsOnly: false, DefaultVolumesToFsBackup: false, UploaderType: uploader.KopiaType, + DefaultSnapshotMoveData: false, } } @@ -195,6 +200,7 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) { SecretData: secretData, RestoreOnly: o.RestoreOnly, UseNodeAgent: o.UseNodeAgent, + PrivilegedNodeAgent: o.PrivilegedNodeAgent, UseVolumeSnapshots: o.UseVolumeSnapshots, BSLConfig: o.BackupStorageConfig.Data(), VSLConfig: o.VolumeSnapshotConfig.Data(), @@ -206,6 +212,7 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) { Features: strings.Split(o.Features, ","), DefaultVolumesToFsBackup: o.DefaultVolumesToFsBackup, UploaderType: o.UploaderType, + DefaultSnapshotMoveData: o.DefaultSnapshotMoveData, }, nil } @@ -231,7 +238,7 @@ The '--namespace' flag can be used to specify a different namespace to install i Use '--wait' to wait for the Velero Deployment to be ready before proceeding. -Use '-o yaml' or '-o json' with '--dry-run' to output all generated resources as text instead of sending the resources to the server. +Use '-o yaml' or '-o json' with '--dry-run' to output all generated resources as text instead of sending the resources to the server. This is useful as a starting point for more customized installations. `, Example: ` # velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket mybucket --secret-file ./gcp-service-account.json diff --git a/pkg/cmd/cli/nodeagent/server.go b/pkg/cmd/cli/nodeagent/server.go index 2fa52f8917..10df06f89e 100644 --- a/pkg/cmd/cli/nodeagent/server.go +++ b/pkg/cmd/cli/nodeagent/server.go @@ -361,7 +361,7 @@ func (s *nodeAgentServer) markDataUploadsCancel(r *controller.DataUploadReconcil return } if dataUploads, err := r.FindDataUploads(s.ctx, client, s.namespace); err != nil { - s.logger.WithError(errors.WithStack(err)).Error("failed to find data downloads") + s.logger.WithError(errors.WithStack(err)).Error("failed to find data uploads") } else { for i := range dataUploads { du := dataUploads[i] @@ -463,7 +463,7 @@ func (s *nodeAgentServer) markInProgressPVRsFailed(client ctrlclient.Client) { continue } if pod.Spec.NodeName != s.nodeName { - s.logger.Debugf("the node of pod referenced by podvolumebackup %q is %q, not %q, skip", pvr.GetName(), pod.Spec.NodeName, s.nodeName) + s.logger.Debugf("the node of pod referenced by podvolumerestore %q is %q, not %q, skip", pvr.GetName(), pod.Spec.NodeName, s.nodeName) continue } diff --git a/pkg/cmd/cli/restore/create.go b/pkg/cmd/cli/restore/create.go index 410da5a346..45e5876236 100644 --- a/pkg/cmd/cli/restore/create.go +++ b/pkg/cmd/cli/restore/create.go @@ -92,6 +92,7 @@ type CreateOptions struct { StatusExcludeResources flag.StringArray NamespaceMappings flag.Map Selector flag.LabelSelector + OrSelector flag.OrLabelSelector IncludeClusterResources flag.OptionalBool Wait bool AllowPartiallyFailed flag.OptionalBool @@ -124,6 +125,7 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { flags.Var(&o.StatusIncludeResources, "status-include-resources", "Resources to include in the restore status, formatted as resource.group, such as storageclasses.storage.k8s.io.") flags.Var(&o.StatusExcludeResources, "status-exclude-resources", "Resources to exclude from the restore status, formatted as resource.group, such as storageclasses.storage.k8s.io.") flags.VarP(&o.Selector, "selector", "l", "Only restore resources matching this label selector.") + flags.Var(&o.OrSelector, "or-selector", "Restore resources matching at least one of the label selector from the list. Label selectors should be separated by ' or '. For example, foo=bar or app=nginx") flags.DurationVar(&o.ItemOperationTimeout, "item-operation-timeout", o.ItemOperationTimeout, "How long to wait for async plugin operations before timeout.") f := flags.VarPF(&o.RestoreVolumes, "restore-volumes", "", "Whether to restore volumes from snapshots.") // this allows the user to just specify "--restore-volumes" as shorthand for "--restore-volumes=true" @@ -185,6 +187,10 @@ func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Facto return errors.New("Velero client is not set; unable to proceed") } + if o.Selector.LabelSelector != nil && o.OrSelector.OrLabelSelectors != nil { + return errors.New("either a 'selector' or an 'or-selector' can be specified, but not both") + } + if len(o.ExistingResourcePolicy) > 0 && !isResourcePolicyValid(o.ExistingResourcePolicy) { return errors.New("existing-resource-policy has invalid value, it accepts only none, update as value") } @@ -304,6 +310,7 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { ExistingResourcePolicy: api.PolicyType(o.ExistingResourcePolicy), NamespaceMapping: o.NamespaceMappings.Data(), LabelSelector: o.Selector.LabelSelector, + OrLabelSelectors: o.OrSelector.OrLabelSelectors, RestorePVs: o.RestoreVolumes.Value, PreserveNodePorts: o.PreserveNodePorts.Value, IncludeClusterResources: o.IncludeClusterResources.Value, diff --git a/pkg/cmd/cli/schedule/create.go b/pkg/cmd/cli/schedule/create.go index c24ebd6383..4fc40e814e 100644 --- a/pkg/cmd/cli/schedule/create.go +++ b/pkg/cmd/cli/schedule/create.go @@ -145,6 +145,7 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { ExcludedNamespaceScopedResources: o.BackupOptions.ExcludeNamespaceScopedResources, IncludeClusterResources: o.BackupOptions.IncludeClusterResources.Value, LabelSelector: o.BackupOptions.Selector.LabelSelector, + OrLabelSelectors: o.BackupOptions.OrSelector.OrLabelSelectors, SnapshotVolumes: o.BackupOptions.SnapshotVolumes.Value, TTL: metav1.Duration{Duration: o.BackupOptions.TTL}, StorageLocation: o.BackupOptions.StorageLocation, diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 20e14e3686..d7ca453361 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -135,6 +135,7 @@ type serverConfig struct { defaultVolumesToFsBackup bool uploaderType string maxConcurrentK8SConnections int + defaultSnapshotMoveData bool } func NewCommand(f client.Factory) *cobra.Command { @@ -163,6 +164,7 @@ func NewCommand(f client.Factory) *cobra.Command { defaultVolumesToFsBackup: podvolume.DefaultVolumesToFsBackup, uploaderType: uploader.ResticType, maxConcurrentK8SConnections: defaultMaxConcurrentK8SConnections, + defaultSnapshotMoveData: false, } ) @@ -233,6 +235,7 @@ func NewCommand(f client.Factory) *cobra.Command { command.Flags().DurationVar(&config.defaultItemOperationTimeout, "default-item-operation-timeout", config.defaultItemOperationTimeout, "How long to wait on asynchronous BackupItemActions and RestoreItemActions to complete before timing out. Default is 4 hours") command.Flags().DurationVar(&config.resourceTimeout, "resource-timeout", config.resourceTimeout, "How long to wait for resource processes which are not covered by other specific timeout parameters. Default is 10 minutes.") command.Flags().IntVar(&config.maxConcurrentK8SConnections, "max-concurrent-k8s-connections", config.maxConcurrentK8SConnections, "Max concurrent connections number that Velero can create with kube-apiserver. Default is 30.") + command.Flags().BoolVar(&config.defaultSnapshotMoveData, "default-snapshot-move-data", config.defaultSnapshotMoveData, "Move data by default for all snapshots supporting data movement.") return command } @@ -700,7 +703,6 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string bslr := controller.NewBackupStorageLocationReconciler( s.ctx, s.mgr.GetClient(), - s.mgr.GetScheme(), storage.DefaultBackupLocationInfo{ StorageLocation: s.config.defaultBackupLocation, ServerValidationFrequency: s.config.storeValidationFrequency, @@ -757,6 +759,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.csiSnapshotClient, s.credentialFileStore, s.config.maxConcurrentK8SConnections, + s.config.defaultSnapshotMoveData, ).SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", controller.Backup) } diff --git a/pkg/cmd/util/flag/orlabelselector.go b/pkg/cmd/util/flag/orlabelselector.go new file mode 100644 index 0000000000..0ef08166f7 --- /dev/null +++ b/pkg/cmd/util/flag/orlabelselector.go @@ -0,0 +1,61 @@ +/* +Copyright 2017 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// OrLabelSelector is a Cobra-compatible wrapper for defining +// a Kubernetes or-label-selector flag. +type OrLabelSelector struct { + OrLabelSelectors []*metav1.LabelSelector +} + +// String returns a string representation of the or-label +// selector flag. +func (ls *OrLabelSelector) String() string { + orLabels := []string{} + for _, v := range ls.OrLabelSelectors { + orLabels = append(orLabels, metav1.FormatLabelSelector(v)) + } + return strings.Join(orLabels, " or ") +} + +// Set parses the provided string and assigns the result +// to the or-label-selector receiver. It returns an error if +// the string is not parseable. +func (ls *OrLabelSelector) Set(s string) error { + orItems := strings.Split(s, " or ") + ls.OrLabelSelectors = make([]*metav1.LabelSelector, 0) + for _, orItem := range orItems { + parsed, err := metav1.ParseToLabelSelector(orItem) + if err != nil { + return err + } + ls.OrLabelSelectors = append(ls.OrLabelSelectors, parsed) + } + return nil +} + +// Type returns a string representation of the +// OrLabelSelector type. +func (ls *OrLabelSelector) Type() string { + return "orLabelSelector" +} diff --git a/pkg/cmd/util/flag/orlabelselector_test.go b/pkg/cmd/util/flag/orlabelselector_test.go new file mode 100644 index 0000000000..09e2ec0ef5 --- /dev/null +++ b/pkg/cmd/util/flag/orlabelselector_test.go @@ -0,0 +1,102 @@ +package flag + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestStringOfOrLabelSelector(t *testing.T) { + tests := []struct { + name string + orLabelSelector *OrLabelSelector + expectedStr string + }{ + { + name: "or between two labels", + orLabelSelector: &OrLabelSelector{ + OrLabelSelectors: []*metav1.LabelSelector{ + { + MatchLabels: map[string]string{"k1": "v1"}, + }, + { + MatchLabels: map[string]string{"k2": "v2"}, + }, + }, + }, + expectedStr: "k1=v1 or k2=v2", + }, + { + name: "or between two label groups", + orLabelSelector: &OrLabelSelector{ + OrLabelSelectors: []*metav1.LabelSelector{ + { + MatchLabels: map[string]string{"k1": "v1", "k2": "v2"}, + }, + { + MatchLabels: map[string]string{"a1": "b1", "a2": "b2"}, + }, + }, + }, + expectedStr: "k1=v1,k2=v2 or a1=b1,a2=b2", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.expectedStr, test.orLabelSelector.String()) + }) + } +} + +func TestSetOfOrLabelSelector(t *testing.T) { + tests := []struct { + name string + inputStr string + expectedSelector *OrLabelSelector + }{ + { + name: "or between two labels", + inputStr: "k1=v1 or k2=v2", + expectedSelector: &OrLabelSelector{ + OrLabelSelectors: []*metav1.LabelSelector{ + { + MatchLabels: map[string]string{"k1": "v1"}, + }, + { + MatchLabels: map[string]string{"k2": "v2"}, + }, + }, + }, + }, + { + name: "or between two label groups", + inputStr: "k1=v1,k2=v2 or a1=b1,a2=b2", + expectedSelector: &OrLabelSelector{ + OrLabelSelectors: []*metav1.LabelSelector{ + { + MatchLabels: map[string]string{"k1": "v1", "k2": "v2"}, + }, + { + MatchLabels: map[string]string{"a1": "b1", "a2": "b2"}, + }, + }, + }, + }, + } + selector := &OrLabelSelector{} + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require.Nil(t, selector.Set(test.inputStr)) + assert.Equal(t, len(test.expectedSelector.OrLabelSelectors), len(selector.OrLabelSelectors)) + assert.Equal(t, test.expectedSelector.String(), selector.String()) + }) + } +} + +func TestTypeOfOrLabelSelector(t *testing.T) { + selector := &OrLabelSelector{} + assert.Equal(t, "orLabelSelector", selector.Type()) +} diff --git a/pkg/cmd/util/output/backup_describer.go b/pkg/cmd/util/output/backup_describer.go index 688cd7b6e9..039f9d6531 100644 --- a/pkg/cmd/util/output/backup_describer.go +++ b/pkg/cmd/util/output/backup_describer.go @@ -199,6 +199,18 @@ func DescribeBackupSpec(d *Describer, spec velerov1api.BackupSpec) { } d.Printf("Label selector:\t%s\n", s) + d.Println() + if len(spec.OrLabelSelectors) == 0 { + s = emptyDisplay + } else { + orLabelSelectors := []string{} + for _, v := range spec.OrLabelSelectors { + orLabelSelectors = append(orLabelSelectors, metav1.FormatLabelSelector(v)) + } + s = strings.Join(orLabelSelectors, " or ") + } + d.Printf("Or label selector:\t%s\n", s) + d.Println() d.Printf("Storage Location:\t%s\n", spec.StorageLocation) diff --git a/pkg/cmd/util/output/backup_describer_test.go b/pkg/cmd/util/output/backup_describer_test.go index cefb4034ab..b0f78a917e 100644 --- a/pkg/cmd/util/output/backup_describer_test.go +++ b/pkg/cmd/util/output/backup_describer_test.go @@ -91,6 +91,8 @@ Resources: Label selector: +Or label selector: + Storage Location: backup-location Velero-Native Snapshot PVs: auto @@ -153,6 +155,8 @@ Resources: Label selector: +Or label selector: + Storage Location: backup-location Velero-Native Snapshot PVs: auto @@ -208,6 +212,8 @@ Resources: Label selector: +Or label selector: + Storage Location: backup-location Velero-Native Snapshot PVs: auto diff --git a/pkg/cmd/util/output/restore_describer.go b/pkg/cmd/util/output/restore_describer.go index 00d1304dcf..c64c1a88a5 100644 --- a/pkg/cmd/util/output/restore_describer.go +++ b/pkg/cmd/util/output/restore_describer.go @@ -146,6 +146,18 @@ func DescribeRestore(ctx context.Context, kbClient kbclient.Client, restore *vel } d.Printf("Label selector:\t%s\n", s) + d.Println() + if len(restore.Spec.OrLabelSelectors) == 0 { + s = emptyDisplay + } else { + orLabelSelectors := []string{} + for _, v := range restore.Spec.OrLabelSelectors { + orLabelSelectors = append(orLabelSelectors, metav1.FormatLabelSelector(v)) + } + s = strings.Join(orLabelSelectors, " or ") + } + d.Printf("Or label selector:\t%s\n", s) + d.Println() d.Printf("Restore PVs:\t%s\n", BoolPointerString(restore.Spec.RestorePVs, "false", "true", "auto")) diff --git a/pkg/cmd/util/output/schedule_describe_test.go b/pkg/cmd/util/output/schedule_describe_test.go index ffd2f21698..7123199ec4 100644 --- a/pkg/cmd/util/output/schedule_describe_test.go +++ b/pkg/cmd/util/output/schedule_describe_test.go @@ -38,6 +38,8 @@ Backup Template: Label selector: + Or label selector: + Storage Location: Velero-Native Snapshot PVs: auto @@ -82,6 +84,8 @@ Backup Template: Label selector: + Or label selector: + Storage Location: Velero-Native Snapshot PVs: auto diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 44a1715fb5..c02ce7e483 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -88,6 +88,7 @@ type backupReconciler struct { volumeSnapshotClient snapshotterClientSet.Interface credentialFileStore credentials.FileStore maxConcurrentK8SConnections int + defaultSnapshotMoveData bool } func NewBackupReconciler( @@ -113,6 +114,7 @@ func NewBackupReconciler( volumeSnapshotClient snapshotterClientSet.Interface, credentialStore credentials.FileStore, maxConcurrentK8SConnections int, + defaultSnapshotMoveData bool, ) *backupReconciler { b := &backupReconciler{ ctx: ctx, @@ -138,6 +140,7 @@ func NewBackupReconciler( volumeSnapshotClient: volumeSnapshotClient, credentialFileStore: credentialStore, maxConcurrentK8SConnections: maxConcurrentK8SConnections, + defaultSnapshotMoveData: defaultSnapshotMoveData, } b.updateTotalBackupMetric() return b @@ -353,6 +356,10 @@ func (b *backupReconciler) prepareBackupRequest(backup *velerov1api.Backup, logg request.Spec.DefaultVolumesToFsBackup = &b.defaultVolumesToFsBackup } + if request.Spec.SnapshotMoveData == nil { + request.Spec.SnapshotMoveData = &b.defaultSnapshotMoveData + } + // find which storage location to use var serverSpecified bool if request.Spec.StorageLocation == "" { diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index 30296b7e5a..f187877330 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -583,6 +583,7 @@ func TestProcessBackupCompletions(t *testing.T) { backup *velerov1api.Backup backupLocation *velerov1api.BackupStorageLocation defaultVolumesToFsBackup bool + defaultSnapshotMoveData bool expectedResult *velerov1api.Backup backupExists bool existenceCheckError error @@ -615,6 +616,7 @@ func TestProcessBackupCompletions(t *testing.T) { Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.True(), + SnapshotMoveData: boolptr.False(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -651,6 +653,7 @@ func TestProcessBackupCompletions(t *testing.T) { Spec: velerov1api.BackupSpec{ StorageLocation: "alt-loc", DefaultVolumesToFsBackup: boolptr.False(), + SnapshotMoveData: boolptr.False(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -690,6 +693,7 @@ func TestProcessBackupCompletions(t *testing.T) { Spec: velerov1api.BackupSpec{ StorageLocation: "read-write", DefaultVolumesToFsBackup: boolptr.True(), + SnapshotMoveData: boolptr.False(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -727,6 +731,7 @@ func TestProcessBackupCompletions(t *testing.T) { TTL: metav1.Duration{Duration: 10 * time.Minute}, StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.False(), + SnapshotMoveData: boolptr.False(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -764,6 +769,7 @@ func TestProcessBackupCompletions(t *testing.T) { Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.True(), + SnapshotMoveData: boolptr.False(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -802,6 +808,7 @@ func TestProcessBackupCompletions(t *testing.T) { Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.False(), + SnapshotMoveData: boolptr.False(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -840,6 +847,7 @@ func TestProcessBackupCompletions(t *testing.T) { Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.True(), + SnapshotMoveData: boolptr.False(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -878,6 +886,7 @@ func TestProcessBackupCompletions(t *testing.T) { Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.True(), + SnapshotMoveData: boolptr.False(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -916,6 +925,7 @@ func TestProcessBackupCompletions(t *testing.T) { Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.False(), + SnapshotMoveData: boolptr.False(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -955,6 +965,7 @@ func TestProcessBackupCompletions(t *testing.T) { Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.True(), + SnapshotMoveData: boolptr.False(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFailed, @@ -994,6 +1005,7 @@ func TestProcessBackupCompletions(t *testing.T) { Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.True(), + SnapshotMoveData: boolptr.False(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFailed, @@ -1113,6 +1125,89 @@ func TestProcessBackupCompletions(t *testing.T) { Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.False(), + SnapshotMoveData: boolptr.False(), + }, + Status: velerov1api.BackupStatus{ + Phase: velerov1api.BackupPhaseFinalizing, + Version: 1, + FormatVersion: "1.1.0", + StartTimestamp: ×tamp, + Expiration: ×tamp, + CSIVolumeSnapshotsAttempted: 1, + CSIVolumeSnapshotsCompleted: 0, + }, + }, + volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + }, + { + name: "backup with snapshot data movement set to true and defaultSnapshotMoveData set to false", + backup: defaultBackup().SnapshotMoveData(true).Result(), + backupLocation: defaultBackupLocation, + defaultVolumesToFsBackup: false, + defaultSnapshotMoveData: false, + expectedResult: &velerov1api.Backup{ + TypeMeta: metav1.TypeMeta{ + Kind: "Backup", + APIVersion: "velero.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1api.DefaultNamespace, + Name: "backup-1", + Annotations: map[string]string{ + "velero.io/source-cluster-k8s-major-version": "1", + "velero.io/source-cluster-k8s-minor-version": "16", + "velero.io/source-cluster-k8s-gitversion": "v1.16.4", + "velero.io/resource-timeout": "0s", + }, + Labels: map[string]string{ + "velero.io/storage-location": "loc-1", + }, + }, + Spec: velerov1api.BackupSpec{ + StorageLocation: defaultBackupLocation.Name, + DefaultVolumesToFsBackup: boolptr.False(), + SnapshotMoveData: boolptr.True(), + }, + Status: velerov1api.BackupStatus{ + Phase: velerov1api.BackupPhaseFinalizing, + Version: 1, + FormatVersion: "1.1.0", + StartTimestamp: ×tamp, + Expiration: ×tamp, + CSIVolumeSnapshotsAttempted: 0, + CSIVolumeSnapshotsCompleted: 0, + }, + }, + volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + }, + { + name: "backup with snapshot data movement set to false and defaultSnapshotMoveData set to true", + backup: defaultBackup().SnapshotMoveData(false).Result(), + backupLocation: defaultBackupLocation, + defaultVolumesToFsBackup: false, + defaultSnapshotMoveData: true, + expectedResult: &velerov1api.Backup{ + TypeMeta: metav1.TypeMeta{ + Kind: "Backup", + APIVersion: "velero.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1api.DefaultNamespace, + Name: "backup-1", + Annotations: map[string]string{ + "velero.io/source-cluster-k8s-major-version": "1", + "velero.io/source-cluster-k8s-minor-version": "16", + "velero.io/source-cluster-k8s-gitversion": "v1.16.4", + "velero.io/resource-timeout": "0s", + }, + Labels: map[string]string{ + "velero.io/storage-location": "loc-1", + }, + }, + Spec: velerov1api.BackupSpec{ + StorageLocation: defaultBackupLocation.Name, + DefaultVolumesToFsBackup: boolptr.False(), + SnapshotMoveData: boolptr.False(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1126,6 +1221,47 @@ func TestProcessBackupCompletions(t *testing.T) { }, volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, + { + name: "backup with snapshot data movement not set and defaultSnapshotMoveData set to true", + backup: defaultBackup().Result(), + backupLocation: defaultBackupLocation, + defaultVolumesToFsBackup: false, + defaultSnapshotMoveData: true, + expectedResult: &velerov1api.Backup{ + TypeMeta: metav1.TypeMeta{ + Kind: "Backup", + APIVersion: "velero.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1api.DefaultNamespace, + Name: "backup-1", + Annotations: map[string]string{ + "velero.io/source-cluster-k8s-major-version": "1", + "velero.io/source-cluster-k8s-minor-version": "16", + "velero.io/source-cluster-k8s-gitversion": "v1.16.4", + "velero.io/resource-timeout": "0s", + }, + Labels: map[string]string{ + "velero.io/storage-location": "loc-1", + }, + }, + Spec: velerov1api.BackupSpec{ + StorageLocation: defaultBackupLocation.Name, + DefaultVolumesToFsBackup: boolptr.False(), + SnapshotMoveData: boolptr.True(), + }, + Status: velerov1api.BackupStatus{ + Phase: velerov1api.BackupPhaseFinalizing, + Version: 1, + FormatVersion: "1.1.0", + StartTimestamp: ×tamp, + Expiration: ×tamp, + CSIVolumeSnapshotsAttempted: 0, + CSIVolumeSnapshotsCompleted: 0, + }, + }, + volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + }, } for _, test := range tests { @@ -1178,6 +1314,7 @@ func TestProcessBackupCompletions(t *testing.T) { kbClient: fakeClient, defaultBackupLocation: defaultBackupLocation.Name, defaultVolumesToFsBackup: test.defaultVolumesToFsBackup, + defaultSnapshotMoveData: test.defaultSnapshotMoveData, backupTracker: NewBackupTracker(), metrics: metrics.NewServerMetrics(), clock: testclocks.NewFakeClock(now), diff --git a/pkg/controller/backup_storage_location_controller.go b/pkg/controller/backup_storage_location_controller.go index 26cc1e7341..051dcb6fe6 100644 --- a/pkg/controller/backup_storage_location_controller.go +++ b/pkg/controller/backup_storage_location_controller.go @@ -24,7 +24,6 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -47,7 +46,6 @@ const ( type backupStorageLocationReconciler struct { ctx context.Context client client.Client - scheme *runtime.Scheme defaultBackupLocationInfo storage.DefaultBackupLocationInfo // use variables to refer to these functions so they can be // replaced with fakes for testing. @@ -61,7 +59,6 @@ type backupStorageLocationReconciler struct { func NewBackupStorageLocationReconciler( ctx context.Context, client client.Client, - scheme *runtime.Scheme, defaultBackupLocationInfo storage.DefaultBackupLocationInfo, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupStoreGetter persistence.ObjectBackupStoreGetter, @@ -69,7 +66,6 @@ func NewBackupStorageLocationReconciler( return &backupStorageLocationReconciler{ ctx: ctx, client: client, - scheme: scheme, defaultBackupLocationInfo: defaultBackupLocationInfo, newPluginManager: newPluginManager, backupStoreGetter: backupStoreGetter, diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index 18f7e09f95..8bc650f5f9 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -177,7 +177,10 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } - exposeParam := r.setupExposeParam(du) + exposeParam, err := r.setupExposeParam(du) + if err != nil { + return r.errorOut(ctx, du, err, "failed to set exposer parameters", log) + } // Expose() will trigger to create one pod whose volume is restored by a given volume snapshot, // but the pod maybe is not in the same node of the current controller, so we need to return it here. @@ -735,18 +738,33 @@ func (r *DataUploadReconciler) closeDataPath(ctx context.Context, duName string) r.dataPathMgr.RemoveAsyncBR(duName) } -func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload) interface{} { +func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload) (interface{}, error) { if du.Spec.SnapshotType == velerov2alpha1api.SnapshotTypeCSI { + pvc := &corev1.PersistentVolumeClaim{} + err := r.client.Get(context.Background(), types.NamespacedName{ + Namespace: du.Spec.SourceNamespace, + Name: du.Spec.SourcePVC, + }, pvc) + + if err != nil { + return nil, errors.Wrapf(err, "failed to get PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC) + } + + accessMode := exposer.AccessModeFileSystem + if pvc.Spec.VolumeMode != nil && *pvc.Spec.VolumeMode == corev1.PersistentVolumeBlock { + accessMode = exposer.AccessModeBlock + } + return &exposer.CSISnapshotExposeParam{ SnapshotName: du.Spec.CSISnapshot.VolumeSnapshot, SourceNamespace: du.Spec.SourceNamespace, StorageClass: du.Spec.CSISnapshot.StorageClass, HostingPodLabels: map[string]string{velerov1api.DataUploadLabel: du.Name}, - AccessMode: exposer.AccessModeFileSystem, + AccessMode: accessMode, Timeout: du.Spec.OperationTimeout.Duration, - } + }, nil } - return nil + return nil, nil } func (r *DataUploadReconciler) setupWaitExposePara(du *velerov2alpha1api.DataUpload) interface{} { diff --git a/pkg/controller/data_upload_controller_test.go b/pkg/controller/data_upload_controller_test.go index 270a084ad9..34bc4a6aae 100644 --- a/pkg/controller/data_upload_controller_test.go +++ b/pkg/controller/data_upload_controller_test.go @@ -306,6 +306,7 @@ func TestReconcile(t *testing.T) { name string du *velerov2alpha1api.DataUpload pod *corev1.Pod + pvc *corev1.PersistentVolumeClaim snapshotExposerList map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer dataMgr *datapath.Manager expectedProcessed bool @@ -345,11 +346,21 @@ func TestReconcile(t *testing.T) { }, { name: "Dataupload should be accepted", du: dataUploadBuilder().Result(), - pod: builder.ForPod(velerov1api.DefaultNamespace, dataUploadName).Volumes(&corev1.Volume{Name: "dataupload-1"}).Result(), + pod: builder.ForPod("fake-ns", dataUploadName).Volumes(&corev1.Volume{Name: "test-pvc"}).Result(), + pvc: builder.ForPersistentVolumeClaim("fake-ns", "test-pvc").Result(), expectedProcessed: false, expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseAccepted).Result(), expectedRequeue: ctrl.Result{}, }, + { + name: "Dataupload should fail to get PVC information", + du: dataUploadBuilder().Result(), + pod: builder.ForPod("fake-ns", dataUploadName).Volumes(&corev1.Volume{Name: "wrong-pvc"}).Result(), + expectedProcessed: true, + expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Result(), + expectedRequeue: ctrl.Result{}, + expectedErrMsg: "failed to get PVC", + }, { name: "Dataupload should be prepared", du: dataUploadBuilder().SnapshotType(fakeSnapshotType).Result(), @@ -448,6 +459,11 @@ func TestReconcile(t *testing.T) { require.NoError(t, err) } + if test.pvc != nil { + err = r.client.Create(ctx, test.pvc) + require.NoError(t, err) + } + if test.dataMgr != nil { r.dataPathMgr = test.dataMgr } else { diff --git a/pkg/controller/pod_volume_backup_controller.go b/pkg/controller/pod_volume_backup_controller.go index 3507a49eaa..f074e0cc25 100644 --- a/pkg/controller/pod_volume_backup_controller.go +++ b/pkg/controller/pod_volume_backup_controller.go @@ -101,8 +101,6 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ ) } - log.Info("PodVolumeBackup starting") - // Only process items for this node. if pvb.Spec.Node != r.nodeName { return ctrl.Result{}, nil @@ -116,6 +114,8 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, nil } + log.Info("PodVolumeBackup starting") + callbacks := datapath.Callbacks{ OnCompleted: r.OnDataPathCompleted, OnFailed: r.OnDataPathFailed, diff --git a/pkg/controller/schedule_controller.go b/pkg/controller/schedule_controller.go index 86386752b5..ed36896eee 100644 --- a/pkg/controller/schedule_controller.go +++ b/pkg/controller/schedule_controller.go @@ -94,6 +94,7 @@ func (c *scheduleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c if err := c.Get(ctx, req.NamespacedName, schedule); err != nil { if apierrors.IsNotFound(err) { log.WithError(err).Error("schedule not found") + c.metrics.RemoveSchedule(req.Name) return ctrl.Result{}, nil } return ctrl.Result{}, errors.Wrapf(err, "error getting schedule %s", req.String()) diff --git a/pkg/datapath/file_system.go b/pkg/datapath/file_system.go index 741f6ae086..fba9eac7b1 100644 --- a/pkg/datapath/file_system.go +++ b/pkg/datapath/file_system.go @@ -133,10 +133,10 @@ func (fs *fileSystemBR) StartBackup(source AccessPoint, realSource string, paren if !fs.initialized { return errors.New("file system data path is not initialized") } - volMode := getPersistentVolumeMode(source) go func() { - snapshotID, emptySnapshot, err := fs.uploaderProv.RunBackup(fs.ctx, source.ByPath, realSource, tags, forceFull, parentSnapshot, volMode, fs) + snapshotID, emptySnapshot, err := fs.uploaderProv.RunBackup(fs.ctx, source.ByPath, realSource, tags, forceFull, + parentSnapshot, source.VolMode, fs) if err == provider.ErrorCanceled { fs.callbacks.OnCancelled(context.Background(), fs.namespace, fs.jobName) @@ -155,10 +155,8 @@ func (fs *fileSystemBR) StartRestore(snapshotID string, target AccessPoint) erro return errors.New("file system data path is not initialized") } - volMode := getPersistentVolumeMode(target) - go func() { - err := fs.uploaderProv.RunRestore(fs.ctx, snapshotID, target.ByPath, volMode, fs) + err := fs.uploaderProv.RunRestore(fs.ctx, snapshotID, target.ByPath, target.VolMode, fs) if err == provider.ErrorCanceled { fs.callbacks.OnCancelled(context.Background(), fs.namespace, fs.jobName) @@ -172,13 +170,6 @@ func (fs *fileSystemBR) StartRestore(snapshotID string, target AccessPoint) erro return nil } -func getPersistentVolumeMode(source AccessPoint) uploader.PersistentVolumeMode { - if source.ByBlock != "" { - return uploader.PersistentVolumeBlock - } - return uploader.PersistentVolumeFilesystem -} - // UpdateProgress which implement ProgressUpdater interface to update progress status func (fs *fileSystemBR) UpdateProgress(p *uploader.Progress) { if fs.callbacks.OnProgress != nil { diff --git a/pkg/datapath/types.go b/pkg/datapath/types.go index 431fccc5dc..e26cf94823 100644 --- a/pkg/datapath/types.go +++ b/pkg/datapath/types.go @@ -53,7 +53,7 @@ type Callbacks struct { // AccessPoint represents an access point that has been exposed to a data path instance type AccessPoint struct { ByPath string - ByBlock string + VolMode uploader.PersistentVolumeMode } // AsyncBR is the interface for asynchronous data path methods diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index e0842f5f13..a0492d5233 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -233,9 +233,12 @@ func (e *csiSnapshotExposer) CleanUp(ctx context.Context, ownerObject corev1.Obj } func getVolumeModeByAccessMode(accessMode string) (corev1.PersistentVolumeMode, error) { - if accessMode == AccessModeFileSystem { + switch accessMode { + case AccessModeFileSystem: return corev1.PersistentVolumeFilesystem, nil - } else { + case AccessModeBlock: + return corev1.PersistentVolumeBlock, nil + default: return "", errors.Errorf("unsupported access mode %s", accessMode) } } @@ -356,6 +359,7 @@ func (e *csiSnapshotExposer) createBackupPod(ctx context.Context, ownerObject co } var gracePeriod int64 = 0 + volumeMounts, volumeDevices := kube.MakePodPVCAttachment(volumeName, backupPVC.Spec.VolumeMode) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -379,10 +383,8 @@ func (e *csiSnapshotExposer) createBackupPod(ctx context.Context, ownerObject co Image: podInfo.image, ImagePullPolicy: corev1.PullNever, Command: []string{"/velero-helper", "pause"}, - VolumeMounts: []corev1.VolumeMount{{ - Name: volumeName, - MountPath: "/" + volumeName, - }}, + VolumeMounts: volumeMounts, + VolumeDevices: volumeDevices, }, }, ServiceAccountName: podInfo.serviceAccount, diff --git a/pkg/exposer/generic_restore.go b/pkg/exposer/generic_restore.go index ca5cd68a3f..0868aba475 100644 --- a/pkg/exposer/generic_restore.go +++ b/pkg/exposer/generic_restore.go @@ -82,7 +82,7 @@ func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1.O return errors.Errorf("Target PVC %s/%s has already been bound, abort", sourceNamespace, targetPVCName) } - restorePod, err := e.createRestorePod(ctx, ownerObject, hostingPodLabels, selectedNode) + restorePod, err := e.createRestorePod(ctx, ownerObject, targetPVC, hostingPodLabels, selectedNode) if err != nil { return errors.Wrapf(err, "error to create restore pod") } @@ -247,7 +247,8 @@ func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject co return nil } -func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObject corev1.ObjectReference, label map[string]string, selectedNode string) (*corev1.Pod, error) { +func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObject corev1.ObjectReference, targetPVC *corev1.PersistentVolumeClaim, + label map[string]string, selectedNode string) (*corev1.Pod, error) { restorePodName := ownerObject.Name restorePVCName := ownerObject.Name @@ -260,6 +261,7 @@ func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObjec } var gracePeriod int64 = 0 + volumeMounts, volumeDevices := kube.MakePodPVCAttachment(volumeName, targetPVC.Spec.VolumeMode) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -283,10 +285,8 @@ func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObjec Image: podInfo.image, ImagePullPolicy: corev1.PullNever, Command: []string{"/velero-helper", "pause"}, - VolumeMounts: []corev1.VolumeMount{{ - Name: volumeName, - MountPath: "/" + volumeName, - }}, + VolumeMounts: volumeMounts, + VolumeDevices: volumeDevices, }, }, ServiceAccountName: podInfo.serviceAccount, diff --git a/pkg/exposer/host_path.go b/pkg/exposer/host_path.go index 458667d923..94dc4503c3 100644 --- a/pkg/exposer/host_path.go +++ b/pkg/exposer/host_path.go @@ -26,11 +26,13 @@ import ( ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/datapath" + "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" ) var getVolumeDirectory = kube.GetVolumeDirectory +var getVolumeMode = kube.GetVolumeMode var singlePathMatch = kube.SinglePathMatch // GetPodVolumeHostPath returns a path that can be accessed from the host for a given volume of a pod @@ -45,7 +47,17 @@ func GetPodVolumeHostPath(ctx context.Context, pod *corev1.Pod, volumeName strin logger.WithField("volDir", volDir).Info("Got volume dir") - pathGlob := fmt.Sprintf("/host_pods/%s/volumes/*/%s", string(pod.GetUID()), volDir) + volMode, err := getVolumeMode(ctx, logger, pod, volumeName, cli) + if err != nil { + return datapath.AccessPoint{}, errors.Wrapf(err, "error getting volume mode for volume %s in pod %s", volumeName, pod.Name) + } + + volSubDir := "volumes" + if volMode == uploader.PersistentVolumeBlock { + volSubDir = "volumeDevices" + } + + pathGlob := fmt.Sprintf("/host_pods/%s/%s/*/%s", string(pod.GetUID()), volSubDir, volDir) logger.WithField("pathGlob", pathGlob).Debug("Looking for path matching glob") path, err := singlePathMatch(pathGlob, fs, logger) @@ -56,6 +68,7 @@ func GetPodVolumeHostPath(ctx context.Context, pod *corev1.Pod, volumeName strin logger.WithField("path", path).Info("Found path matching glob") return datapath.AccessPoint{ - ByPath: path, + ByPath: path, + VolMode: volMode, }, nil } diff --git a/pkg/exposer/host_path_test.go b/pkg/exposer/host_path_test.go index f71518d2d7..1022dffd30 100644 --- a/pkg/exposer/host_path_test.go +++ b/pkg/exposer/host_path_test.go @@ -29,17 +29,19 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) func TestGetPodVolumeHostPath(t *testing.T) { tests := []struct { - name string - getVolumeDirFunc func(context.Context, logrus.FieldLogger, *corev1.Pod, string, ctrlclient.Client) (string, error) - pathMatchFunc func(string, filesystem.Interface, logrus.FieldLogger) (string, error) - pod *corev1.Pod - pvc string - err string + name string + getVolumeDirFunc func(context.Context, logrus.FieldLogger, *corev1.Pod, string, ctrlclient.Client) (string, error) + getVolumeModeFunc func(context.Context, logrus.FieldLogger, *corev1.Pod, string, ctrlclient.Client) (uploader.PersistentVolumeMode, error) + pathMatchFunc func(string, filesystem.Interface, logrus.FieldLogger) (string, error) + pod *corev1.Pod + pvc string + err string }{ { name: "get volume dir fail", @@ -55,6 +57,9 @@ func TestGetPodVolumeHostPath(t *testing.T) { getVolumeDirFunc: func(context.Context, logrus.FieldLogger, *corev1.Pod, string, ctrlclient.Client) (string, error) { return "", nil }, + getVolumeModeFunc: func(context.Context, logrus.FieldLogger, *corev1.Pod, string, ctrlclient.Client) (uploader.PersistentVolumeMode, error) { + return uploader.PersistentVolumeFilesystem, nil + }, pathMatchFunc: func(string, filesystem.Interface, logrus.FieldLogger) (string, error) { return "", errors.New("fake-error-2") }, @@ -62,6 +67,18 @@ func TestGetPodVolumeHostPath(t *testing.T) { pvc: "fake-pvc-1", err: "error identifying unique volume path on host for volume fake-pvc-1 in pod fake-pod-2: fake-error-2", }, + { + name: "get block volume dir success", + getVolumeDirFunc: func(context.Context, logrus.FieldLogger, *corev1.Pod, string, ctrlclient.Client) ( + string, error) { + return "fake-pvc-1", nil + }, + pathMatchFunc: func(string, filesystem.Interface, logrus.FieldLogger) (string, error) { + return "/host_pods/fake-pod-1-id/volumeDevices/kubernetes.io~csi/fake-pvc-1-id", nil + }, + pod: builder.ForPod(velerov1api.DefaultNamespace, "fake-pod-1").Result(), + pvc: "fake-pvc-1", + }, } for _, test := range tests { @@ -70,12 +87,18 @@ func TestGetPodVolumeHostPath(t *testing.T) { getVolumeDirectory = test.getVolumeDirFunc } + if test.getVolumeModeFunc != nil { + getVolumeMode = test.getVolumeModeFunc + } + if test.pathMatchFunc != nil { singlePathMatch = test.pathMatchFunc } _, err := GetPodVolumeHostPath(context.Background(), test.pod, test.pvc, nil, nil, velerotest.NewLogger()) - assert.EqualError(t, err, test.err) + if test.err != "" || err != nil { + assert.EqualError(t, err, test.err) + } }) } } diff --git a/pkg/exposer/types.go b/pkg/exposer/types.go index 253256eb97..21c473366d 100644 --- a/pkg/exposer/types.go +++ b/pkg/exposer/types.go @@ -22,6 +22,7 @@ import ( const ( AccessModeFileSystem = "by-file-system" + AccessModeBlock = "by-block-device" ) // ExposeResult defines the result of expose. diff --git a/pkg/install/daemonset.go b/pkg/install/daemonset.go index b139f81242..8e74e16da1 100644 --- a/pkg/install/daemonset.go +++ b/pkg/install/daemonset.go @@ -86,6 +86,14 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1.DaemonSet { }, }, }, + { + Name: "host-plugins", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/lib/kubelet/plugins", + }, + }, + }, { Name: "scratch", VolumeSource: corev1.VolumeSource{ @@ -102,13 +110,20 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1.DaemonSet { "/velero", }, Args: daemonSetArgs, - + SecurityContext: &corev1.SecurityContext{ + Privileged: &c.privilegedNodeAgent, + }, VolumeMounts: []corev1.VolumeMount{ { Name: "host-pods", MountPath: "/host_pods", MountPropagation: &mountPropagationMode, }, + { + Name: "host-plugins", + MountPath: "/var/lib/kubelet/plugins", + MountPropagation: &mountPropagationMode, + }, { Name: "scratch", MountPath: "/scratch", diff --git a/pkg/install/daemonset_test.go b/pkg/install/daemonset_test.go index 762e95b16b..017d5004d2 100644 --- a/pkg/install/daemonset_test.go +++ b/pkg/install/daemonset_test.go @@ -35,7 +35,7 @@ func TestDaemonSet(t *testing.T) { ds = DaemonSet("velero", WithSecret(true)) assert.Equal(t, 7, len(ds.Spec.Template.Spec.Containers[0].Env)) - assert.Equal(t, 3, len(ds.Spec.Template.Spec.Volumes)) + assert.Equal(t, 4, len(ds.Spec.Template.Spec.Volumes)) ds = DaemonSet("velero", WithFeatures([]string{"foo,bar,baz"})) assert.Len(t, ds.Spec.Template.Spec.Containers[0].Args, 3) diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index 22e2e5a4dd..917f7d9f11 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -46,6 +46,8 @@ type podTemplateConfig struct { defaultVolumesToFsBackup bool serviceAccountName string uploaderType string + defaultSnapshotMoveData bool + privilegedNodeAgent bool } func WithImage(image string) podTemplateOption { @@ -136,12 +138,24 @@ func WithDefaultVolumesToFsBackup() podTemplateOption { } } +func WithDefaultSnapshotMoveData() podTemplateOption { + return func(c *podTemplateConfig) { + c.defaultSnapshotMoveData = true + } +} + func WithServiceAccountName(sa string) podTemplateOption { return func(c *podTemplateConfig) { c.serviceAccountName = sa } } +func WithPrivilegedNodeAgent() podTemplateOption { + return func(c *podTemplateConfig) { + c.privilegedNodeAgent = true + } +} + func Deployment(namespace string, opts ...podTemplateOption) *appsv1.Deployment { // TODO: Add support for server args c := &podTemplateConfig{ @@ -167,6 +181,10 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1.Deployment args = append(args, "--default-volumes-to-fs-backup=true") } + if c.defaultSnapshotMoveData { + args = append(args, "--default-snapshot-move-data=true") + } + if len(c.uploaderType) > 0 { args = append(args, fmt.Sprintf("--uploader-type=%s", c.uploaderType)) } diff --git a/pkg/install/resources.go b/pkg/install/resources.go index b7b3407eb3..9e2e8e4dab 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -240,6 +240,7 @@ type VeleroOptions struct { SecretData []byte RestoreOnly bool UseNodeAgent bool + PrivilegedNodeAgent bool UseVolumeSnapshots bool BSLConfig map[string]string VSLConfig map[string]string @@ -251,6 +252,7 @@ type VeleroOptions struct { Features []string DefaultVolumesToFsBackup bool UploaderType string + DefaultSnapshotMoveData bool } func AllCRDs() *unstructured.UnstructuredList { @@ -351,6 +353,10 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { deployOpts = append(deployOpts, WithDefaultVolumesToFsBackup()) } + if o.DefaultSnapshotMoveData { + deployOpts = append(deployOpts, WithDefaultSnapshotMoveData()) + } + deploy := Deployment(o.Namespace, deployOpts...) if err := appendUnstructured(resources, deploy); err != nil { @@ -369,6 +375,9 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { if len(o.Features) > 0 { dsOpts = append(dsOpts, WithFeatures(o.Features)) } + if o.PrivilegedNodeAgent { + dsOpts = append(dsOpts, WithPrivilegedNodeAgent()) + } ds := DaemonSet(o.Namespace, dsOpts...) if err := appendUnstructured(resources, ds); err != nil { fmt.Printf("error appending DaemonSet %s: %s\n", ds.GetName(), err.Error()) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index e6879f363c..593d642fd0 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -468,7 +468,7 @@ func (m *ServerMetrics) InitSchedule(scheduleName string) { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[backupLastStatus].(*prometheus.GaugeVec); ok { - c.WithLabelValues(scheduleName).Add(0) + c.WithLabelValues(scheduleName).Add(1) } if c, ok := m.metrics[restoreAttemptTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) @@ -505,6 +505,88 @@ func (m *ServerMetrics) InitSchedule(scheduleName string) { } } +// RemoveSchedule removes metrics associated with a specified schedule. +func (m *ServerMetrics) RemoveSchedule(scheduleName string) { + if g, ok := m.metrics[backupTarballSizeBytesGauge].(*prometheus.GaugeVec); ok { + g.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[backupAttemptTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[backupSuccessTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[backupPartialFailureTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[backupFailureTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[backupValidationFailureTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if h, ok := m.metrics[backupDurationSeconds].(*prometheus.HistogramVec); ok { + h.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[backupDeletionAttemptTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[backupDeletionSuccessTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[backupDeletionFailureTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if g, ok := m.metrics[backupLastSuccessfulTimestamp].(*prometheus.GaugeVec); ok { + g.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[backupItemsTotalGauge].(*prometheus.GaugeVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[backupItemsErrorsGauge].(*prometheus.GaugeVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[backupWarningTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[backupLastStatus].(*prometheus.GaugeVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[restoreAttemptTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[restorePartialFailureTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[restoreFailedTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[restoreSuccessTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[restoreValidationFailedTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[volumeSnapshotSuccessTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[volumeSnapshotAttemptTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[volumeSnapshotFailureTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName) + } + if c, ok := m.metrics[csiSnapshotAttemptTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName, "") + } + if c, ok := m.metrics[csiSnapshotSuccessTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName, "") + } + if c, ok := m.metrics[csiSnapshotFailureTotal].(*prometheus.CounterVec); ok { + c.DeleteLabelValues(scheduleName, "") + } +} + // InitSchedule initializes counter metrics for a node. func (m *ServerMetrics) InitMetricsForNode(node string) { if c, ok := m.metrics[podVolumeBackupEnqueueTotal].(*prometheus.CounterVec); ok { diff --git a/pkg/podexec/pod_command_executor.go b/pkg/podexec/pod_command_executor.go index 569fca4e58..b268d77a39 100644 --- a/pkg/podexec/pod_command_executor.go +++ b/pkg/podexec/pod_command_executor.go @@ -123,6 +123,12 @@ func (e *defaultPodCommandExecutor) ExecutePodCommand(log logrus.FieldLogger, it "hookTimeout": localHook.Timeout, }, ) + + if pod.Status.Phase == corev1api.PodSucceeded || pod.Status.Phase == corev1api.PodFailed { + hookLog.Infof("Pod entered phase %s before some post-backup exec hooks ran", pod.Status.Phase) + return nil + } + hookLog.Info("running exec hook") req := e.restClient.Post(). diff --git a/pkg/podexec/pod_command_executor_test.go b/pkg/podexec/pod_command_executor_test.go index 3f6937192e..bdd960a592 100644 --- a/pkg/podexec/pod_command_executor_test.go +++ b/pkg/podexec/pod_command_executor_test.go @@ -262,6 +262,37 @@ func TestEnsureContainerExists(t *testing.T) { assert.NoError(t, err) } +func TestPodCompeted(t *testing.T) { + pod := &corev1api.Pod{ + Spec: corev1api.PodSpec{ + Containers: []corev1api.Container{ + { + Name: "foo", + }, + }, + }, + Status: corev1api.PodStatus{ + Phase: corev1api.PodSucceeded, + }, + } + + obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) + require.NoError(t, err) + + clientConfig := &rest.Config{} + poster := &mockPoster{} + defer poster.AssertExpectations(t) + podCommandExecutor := NewPodCommandExecutor(clientConfig, poster).(*defaultPodCommandExecutor) + + hook := v1.ExecHook{ + Container: "foo", + Command: []string{"some", "command"}, + } + + err = podCommandExecutor.ExecutePodCommand(velerotest.NewLogger(), obj, "namespace", "name", "hookName", &hook) + require.NoError(t, err) +} + type mockStreamExecutorFactory struct { mock.Mock } diff --git a/pkg/podvolume/backupper.go b/pkg/podvolume/backupper.go index 57ab0c030f..78c2b6d651 100644 --- a/pkg/podvolume/backupper.go +++ b/pkg/podvolume/backupper.go @@ -200,10 +200,11 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api. b.resultsLock.Unlock() var ( - errs []error - podVolumeBackups []*velerov1api.PodVolumeBackup - podVolumes = make(map[string]corev1api.Volume) - mountedPodVolumes = sets.String{} + errs []error + podVolumeBackups []*velerov1api.PodVolumeBackup + podVolumes = make(map[string]corev1api.Volume) + mountedPodVolumes = sets.String{} + attachedPodDevices = sets.String{} ) pvcSummary := NewPVCBackupSummary() @@ -233,6 +234,14 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api. for _, volumeMount := range container.VolumeMounts { mountedPodVolumes.Insert(volumeMount.Name) } + for _, volumeDevice := range container.VolumeDevices { + attachedPodDevices.Insert(volumeDevice.Name) + } + } + + repoIdentifier := "" + if repositoryType == velerov1api.BackupRepositoryTypeRestic { + repoIdentifier = repo.Spec.ResticIdentifier } var numVolumeSnapshots int @@ -263,6 +272,15 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api. continue } + // check if volume is a block volume + if attachedPodDevices.Has(volumeName) { + msg := fmt.Sprintf("volume %s declared in pod %s/%s is a block volume. Block volumes are not supported for fs backup, skipping", + volumeName, pod.Namespace, pod.Name) + log.Warn(msg) + pvcSummary.addSkipped(volumeName, msg) + continue + } + // volumes that are not mounted by any container should not be backed up, because // its directory is not created if !mountedPodVolumes.Has(volumeName) { @@ -283,7 +301,7 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api. } } - volumeBackup := newPodVolumeBackup(backup, pod, volume, repo.Spec.ResticIdentifier, b.uploaderType, pvc) + volumeBackup := newPodVolumeBackup(backup, pod, volume, repoIdentifier, b.uploaderType, pvc) if _, err = b.veleroClient.VeleroV1().PodVolumeBackups(volumeBackup.Namespace).Create(context.TODO(), volumeBackup, metav1.CreateOptions{}); err != nil { errs = append(errs, err) continue diff --git a/pkg/podvolume/restorer.go b/pkg/podvolume/restorer.go index 0011f3d473..2db6d31996 100644 --- a/pkg/podvolume/restorer.go +++ b/pkg/podvolume/restorer.go @@ -157,6 +157,12 @@ func (r *restorer) RestorePodVolumes(data RestoreData) []error { for _, podVolume := range data.Pod.Spec.Volumes { podVolumes[podVolume.Name] = podVolume } + + repoIdentifier := "" + if repositoryType == velerov1api.BackupRepositoryTypeRestic { + repoIdentifier = repo.Spec.ResticIdentifier + } + for volume, backupInfo := range volumesToRestore { volumeObj, ok := podVolumes[volume] var pvc *corev1api.PersistentVolumeClaim @@ -170,7 +176,7 @@ func (r *restorer) RestorePodVolumes(data RestoreData) []error { } } - volumeRestore := newPodVolumeRestore(data.Restore, data.Pod, data.BackupLocation, volume, backupInfo.snapshotID, repo.Spec.ResticIdentifier, backupInfo.uploaderType, data.SourceNamespace, pvc) + volumeRestore := newPodVolumeRestore(data.Restore, data.Pod, data.BackupLocation, volume, backupInfo.snapshotID, repoIdentifier, backupInfo.uploaderType, data.SourceNamespace, pvc) if err := errorOnly(r.veleroClient.VeleroV1().PodVolumeRestores(volumeRestore.Namespace).Create(context.TODO(), volumeRestore, metav1.CreateOptions{})); err != nil { errs = append(errs, errors.WithStack(err)) diff --git a/pkg/repository/backup_repo_op.go b/pkg/repository/backup_repo_op.go index 7025349a97..ff253e6038 100644 --- a/pkg/repository/backup_repo_op.go +++ b/pkg/repository/backup_repo_op.go @@ -107,9 +107,9 @@ func NewBackupRepository(namespace string, key BackupRepositoryKey) *velerov1api } func isBackupRepositoryNotFoundError(err error) bool { - return (err == errBackupRepoNotFound) + return err == errBackupRepoNotFound } func isBackupRepositoryNotProvisionedError(err error) bool { - return (err == errBackupRepoNotProvisioned) + return err == errBackupRepoNotProvisioned } diff --git a/pkg/repository/config/aws.go b/pkg/repository/config/aws.go index c3a3f4dd8b..cc4e14f598 100644 --- a/pkg/repository/config/aws.go +++ b/pkg/repository/config/aws.go @@ -81,7 +81,7 @@ func GetS3Credentials(config map[string]string) (*credentials.Value, error) { opts := session.Options{} credentialsFile := config[CredentialsFileKey] if credentialsFile == "" { - credentialsFile = os.Getenv("AWS_SHARED_CREDENTIALS_FILE") + credentialsFile = os.Getenv(awsCredentialsFileEnvVar) } if credentialsFile != "" { opts.SharedConfigFiles = append(opts.SharedConfigFiles, credentialsFile) diff --git a/pkg/repository/config/azure.go b/pkg/repository/config/azure.go index 1c203330e1..6662d13c6d 100644 --- a/pkg/repository/config/azure.go +++ b/pkg/repository/config/azure.go @@ -17,225 +17,34 @@ limitations under the License. package config import ( - "context" - "fmt" - "os" - "strings" - - storagemgmt "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage" - "github.com/Azure/go-autorest/autorest/azure" - "github.com/Azure/go-autorest/autorest/azure/auth" - "github.com/joho/godotenv" "github.com/pkg/errors" -) - -const ( - subscriptionIDEnvVar = "AZURE_SUBSCRIPTION_ID" - cloudNameEnvVar = "AZURE_CLOUD_NAME" - resourceGroupConfigKey = "resourceGroup" - - storageAccountConfigKey = "storageAccount" - storageAccountKeyEnvVarConfigKey = "storageAccountKeyEnvVar" - subscriptionIDConfigKey = "subscriptionId" - storageDomainConfigKey = "storageDomain" + "github.com/vmware-tanzu/velero/pkg/util/azure" ) -// getSubscriptionID gets the subscription ID from the 'config' map if it contains -// it, else from the AZURE_SUBSCRIPTION_ID environment variable. -func getSubscriptionID(config map[string]string) string { - if subscriptionID := config[subscriptionIDConfigKey]; subscriptionID != "" { - return subscriptionID - } - - return os.Getenv(subscriptionIDEnvVar) -} - -func getStorageAccountKey(config map[string]string) (string, error) { - credentialsFile := selectCredentialsFile(config) - - if err := loadCredentialsIntoEnv(credentialsFile); err != nil { - return "", err - } - - // Get Azure cloud from AZURE_CLOUD_NAME, if it exists. If the env var does not - // exist, parseAzureEnvironment will return azure.PublicCloud. - env, err := parseAzureEnvironment(os.Getenv(cloudNameEnvVar)) - if err != nil { - return "", errors.Wrap(err, "unable to parse azure cloud name environment variable") - } - - // Get storage key from secret using key config[storageAccountKeyEnvVarConfigKey]. If the config does not - // exist, continue obtaining it using API - if secretKeyEnvVar := config[storageAccountKeyEnvVarConfigKey]; secretKeyEnvVar != "" { - storageKey := os.Getenv(secretKeyEnvVar) - if storageKey == "" { - return "", errors.Errorf("no storage key secret with key %s found", secretKeyEnvVar) - } - - return storageKey, nil - } - - // get subscription ID from object store config or AZURE_SUBSCRIPTION_ID environment variable - subscriptionID := getSubscriptionID(config) - if subscriptionID == "" { - return "", errors.New("azure subscription ID not found in object store's config or in environment variable") - } - - // we need config["resourceGroup"], config["storageAccount"] - if err := getRequiredValues(mapLookup(config), resourceGroupConfigKey, storageAccountConfigKey); err != nil { - return "", errors.Wrap(err, "unable to get all required config values") - } - - // get authorizer from environment in the following order: - // 1. client credentials (AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET) - // 2. client certificate (AZURE_CERTIFICATE_PATH, AZURE_CERTIFICATE_PASSWORD) - // 3. username and password (AZURE_USERNAME, AZURE_PASSWORD) - // 4. MSI (managed service identity) - authorizer, err := auth.NewAuthorizerFromEnvironment() - if err != nil { - return "", errors.Wrap(err, "error getting authorizer from environment") - } - - // get storageAccountsClient - storageAccountsClient := storagemgmt.NewAccountsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID) - storageAccountsClient.Authorizer = authorizer - - // get storage key - res, err := storageAccountsClient.ListKeys(context.TODO(), config[resourceGroupConfigKey], config[storageAccountConfigKey], storagemgmt.Kerb) - if err != nil { - return "", errors.WithStack(err) - } - if res.Keys == nil || len(*res.Keys) == 0 { - return "", errors.New("No storage keys found") - } - - var storageKey string - for _, key := range *res.Keys { - // The ListKeys call returns e.g. "FULL" but the storagemgmt.Full constant in the SDK is defined as "Full". - if strings.EqualFold(string(key.Permissions), string(storagemgmt.Full)) { - storageKey = *key.Value - break - } - } - - if storageKey == "" { - return "", errors.New("No storage key with Full permissions found") - } - - return storageKey, nil -} - -func mapLookup(data map[string]string) func(string) string { - return func(key string) string { - return data[key] - } -} - // GetAzureResticEnvVars gets the environment variables that restic // relies on (AZURE_ACCOUNT_NAME and AZURE_ACCOUNT_KEY) based // on info in the provided object storage location config map. func GetAzureResticEnvVars(config map[string]string) (map[string]string, error) { - storageAccountKey, err := getStorageAccountKey(config) - if err != nil { - return nil, err + storageAccount := config[azure.BSLConfigStorageAccount] + if storageAccount == "" { + return nil, errors.New("storageAccount is required in the BSL") } - if err := getRequiredValues(mapLookup(config), storageAccountConfigKey); err != nil { - return nil, errors.Wrap(err, "unable to get all required config values") - } - - return map[string]string{ - "AZURE_ACCOUNT_NAME": config[storageAccountConfigKey], - "AZURE_ACCOUNT_KEY": storageAccountKey, - }, nil -} - -// credentialsFileFromEnv retrieves the Azure credentials file from the environment. -func credentialsFileFromEnv() string { - return os.Getenv("AZURE_CREDENTIALS_FILE") -} - -// selectCredentialsFile selects the Azure credentials file to use, retrieving it -// from the given config or falling back to retrieving it from the environment. -func selectCredentialsFile(config map[string]string) string { - if credentialsFile, ok := config[CredentialsFileKey]; ok { - return credentialsFile - } - - return credentialsFileFromEnv() -} - -// loadCredentialsIntoEnv loads the variables in the given credentials -// file into the current environment. -func loadCredentialsIntoEnv(credentialsFile string) error { - if credentialsFile == "" { - return nil - } - - if err := godotenv.Overload(credentialsFile); err != nil { - return errors.Wrapf(err, "error loading environment from credentials file (%s)", credentialsFile) - } - - return nil -} - -// ParseAzureEnvironment returns an azure.Environment for the given cloud -// name, or azure.PublicCloud if cloudName is empty. -func parseAzureEnvironment(cloudName string) (*azure.Environment, error) { - if cloudName == "" { - return &azure.PublicCloud, nil - } - - env, err := azure.EnvironmentFromName(cloudName) - return &env, errors.WithStack(err) -} - -func getRequiredValues(getValue func(string) string, keys ...string) error { - missing := []string{} - results := map[string]string{} - - for _, key := range keys { - if val := getValue(key); val == "" { - missing = append(missing, key) - } else { - results[key] = val - } - } - - if len(missing) > 0 { - return errors.Errorf("the following keys do not have values: %s", strings.Join(missing, ", ")) - } - - return nil -} - -// GetAzureStorageDomain gets the Azure storage domain required by a Azure blob connection, -// if the provided credential file doesn't have the value, get it from system's environment variables -func GetAzureStorageDomain(config map[string]string) (string, error) { - credentialsFile := selectCredentialsFile(config) - - if err := loadCredentialsIntoEnv(credentialsFile); err != nil { - return "", err - } - - return getStorageDomainFromCloudName(os.Getenv(cloudNameEnvVar)) -} - -func GetAzureCredentials(config map[string]string) (string, string, error) { - storageAccountKey, err := getStorageAccountKey(config) + creds, err := azure.LoadCredentials(config) if err != nil { - return "", "", err + return nil, err } - return config[storageAccountConfigKey], storageAccountKey, nil -} - -func getStorageDomainFromCloudName(cloudName string) (string, error) { - env, err := parseAzureEnvironment(cloudName) + // restic doesn't support Azure AD, set it as false + config[azure.BSLConfigUseAAD] = "false" + credentials, err := azure.GetStorageAccountCredentials(config, creds) if err != nil { - return "", errors.Wrapf(err, "unable to parse azure env from cloud name %s", cloudName) + return nil, err } - return fmt.Sprintf("blob.%s", env.StorageEndpointSuffix), nil + return map[string]string{ + "AZURE_ACCOUNT_NAME": storageAccount, + "AZURE_ACCOUNT_KEY": credentials[azure.CredentialKeyStorageAccountAccessKey], + }, nil } diff --git a/pkg/repository/config/azure_test.go b/pkg/repository/config/azure_test.go index 87cd31bf15..c283197b3a 100644 --- a/pkg/repository/config/azure_test.go +++ b/pkg/repository/config/azure_test.go @@ -1,12 +1,9 @@ /* Copyright the Velero contributors. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,161 +15,37 @@ package config import ( "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" -) - -// setAzureEnvironment sets the Azure credentials environment variable to the -// given value and returns a function to restore it to its previous value -func setAzureEnvironment(t *testing.T, value string) func() { - envVar := "AZURE_CREDENTIALS_FILE" - var cleanup func() - - if original, exists := os.LookupEnv(envVar); exists { - cleanup = func() { - require.NoError(t, os.Setenv(envVar, original), "failed to reset %s environment variable", envVar) - } - } else { - cleanup = func() { - require.NoError(t, os.Unsetenv(envVar), "failed to reset %s environment variable", envVar) - } - } - - require.NoError(t, os.Setenv(envVar, value), "failed to set %s environment variable", envVar) - - return cleanup -} -func TestSelectCredentialsFile(t *testing.T) { - testCases := []struct { - name string - config map[string]string - environment string - expected string - }{ - { - name: "when config is empty and environment variable is not set, no file is selected", - expected: "", - }, - { - name: "when config contains credentials file and environment variable is not set, file from config is selected", - config: map[string]string{ - "credentialsFile": "/tmp/credentials/path/to/secret", - }, - expected: "/tmp/credentials/path/to/secret", - }, - { - name: "when config is empty and environment variable is set, file from environment is selected", - environment: "/credentials/file/from/env", - expected: "/credentials/file/from/env", - }, - { - name: "when config contains credentials file and environment variable is set, file from config is selected", - config: map[string]string{ - "credentialsFile": "/tmp/credentials/path/to/secret", - }, - environment: "/credentials/file/from/env", - expected: "/tmp/credentials/path/to/secret", - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - cleanup := setAzureEnvironment(t, tc.environment) - defer cleanup() - - selectedFile := selectCredentialsFile(tc.config) - require.Equal(t, tc.expected, selectedFile) - }) - } -} - -func TestGetStorageDomainFromCloudName(t *testing.T) { - testCases := []struct { - name string - cloudName string - expected string - expectedErr string - }{ - { - name: "get azure env fail", - cloudName: "fake-cloud", - expectedErr: "unable to parse azure env from cloud name fake-cloud: autorest/azure: There is no cloud environment matching the name \"FAKE-CLOUD\"", - }, - { - name: "cloud name is empty", - cloudName: "", - expected: "blob.core.windows.net", - }, - { - name: "azure public cloud", - cloudName: "AzurePublicCloud", - expected: "blob.core.windows.net", - }, - { - - name: "azure China cloud", - cloudName: "AzureChinaCloud", - expected: "blob.core.chinacloudapi.cn", - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - domain, err := getStorageDomainFromCloudName(tc.cloudName) - - require.Equal(t, tc.expected, domain) - - if tc.expectedErr == "" { - assert.NoError(t, err) - } else { - assert.EqualError(t, err, tc.expectedErr) - assert.Empty(t, domain) - } - }) - } -} - -func TestGetRequiredValues(t *testing.T) { - testCases := []struct { - name string - mp map[string]string - keys []string - err string - }{ - { - name: "with miss", - mp: map[string]string{ - "key1": "value1", - }, - keys: []string{"key1", "key2", "key3"}, - err: "the following keys do not have values: key2, key3", - }, - { - name: "without miss", - mp: map[string]string{ - "key1": "value1", - "key2": "value2", - "key3": "value3", - }, - keys: []string{"key1", "key2", "key3"}, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - err := getRequiredValues(func(key string) string { - if tc.mp == nil { - return "" - } else { - return tc.mp[key] - } - }, tc.keys...) + "github.com/vmware-tanzu/velero/pkg/util/azure" +) - if err == nil { - assert.NoError(t, err) - } else { - assert.EqualError(t, err, tc.err) - } - }) - } +func TestGetAzureResticEnvVars(t *testing.T) { + config := map[string]string{} + + // no storage account specified + _, err := GetAzureResticEnvVars(config) + require.NotNil(t, err) + + // specify storage account access key + name := filepath.Join(os.TempDir(), "credential") + file, err := os.Create(name) + require.Nil(t, err) + defer file.Close() + defer os.Remove(name) + _, err = file.WriteString("AccessKey: accesskey") + require.Nil(t, err) + + config[azure.BSLConfigStorageAccount] = "account01" + config[azure.BSLConfigStorageAccountAccessKeyName] = "AccessKey" + config["credentialsFile"] = name + envs, err := GetAzureResticEnvVars(config) + require.Nil(t, err) + + assert.Equal(t, "account01", envs["AZURE_ACCOUNT_NAME"]) + assert.Equal(t, "accesskey", envs["AZURE_ACCOUNT_KEY"]) } diff --git a/pkg/repository/config/gcp.go b/pkg/repository/config/gcp.go index 5be4670ec6..8beb8a016c 100644 --- a/pkg/repository/config/gcp.go +++ b/pkg/repository/config/gcp.go @@ -42,5 +42,5 @@ func GetGCPCredentials(config map[string]string) string { if credentialsFile, ok := config[CredentialsFileKey]; ok { return credentialsFile } - return os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + return os.Getenv(gcpCredentialsFileEnvVar) } diff --git a/pkg/repository/provider/unified_repo.go b/pkg/repository/provider/unified_repo.go index 8dc4701855..988174fa90 100644 --- a/pkg/repository/provider/unified_repo.go +++ b/pkg/repository/provider/unified_repo.go @@ -26,6 +26,7 @@ import ( "strings" "time" + "github.com/kopia/kopia/repo" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -47,11 +48,9 @@ type unifiedRepoProvider struct { // this func is assigned to a package-level variable so it can be // replaced when unit-testing -var getAzureCredentials = repoconfig.GetAzureCredentials var getS3Credentials = repoconfig.GetS3Credentials var getGCPCredentials = repoconfig.GetGCPCredentials var getS3BucketRegion = repoconfig.GetAWSBucketRegion -var getAzureStorageDomain = repoconfig.GetAzureStorageDomain type localFuncTable struct { getStorageVariables func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) @@ -190,10 +189,13 @@ func (urp *unifiedRepoProvider) PrepareRepo(ctx context.Context, param RepoParam log.Debug("Repo has already been initialized remotely") return nil } + if !errors.Is(err, repo.ErrRepositoryNotInitialized) { + return errors.Wrap(err, "error to connect to backup repo") + } err = urp.repoService.Init(ctx, *repoOption, true) if err != nil { - return errors.Wrap(err, "error to init backup repo") + return errors.Wrap(err, "error to create backup repo") } log.Debug("Prepare repo complete") @@ -436,13 +438,8 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr result[udmrepo.StoreOptionS3Token] = credValue.SessionToken } case repoconfig.AzureBackend: - storageAccount, accountKey, err := getAzureCredentials(config) - if err != nil { - return map[string]string{}, errors.Wrap(err, "error get azure credentials") - } - result[udmrepo.StoreOptionAzureStorageAccount] = storageAccount - result[udmrepo.StoreOptionAzureKey] = accountKey - + // do nothing here, will retrieve the credential in Azure Storage + return nil, nil case repoconfig.GCPBackend: result[udmrepo.StoreOptionCredentialFile] = getGCPCredentials(config) } @@ -504,21 +501,17 @@ func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repo result[udmrepo.StoreOptionS3Endpoint] = strings.Trim(s3URL, "/") result[udmrepo.StoreOptionS3DisableTLSVerify] = config["insecureSkipTLSVerify"] result[udmrepo.StoreOptionS3DisableTLS] = strconv.FormatBool(disableTLS) - - if backupLocation.Spec.ObjectStorage != nil && backupLocation.Spec.ObjectStorage.CACert != nil { - result[udmrepo.StoreOptionS3CustomCA] = base64.StdEncoding.EncodeToString(backupLocation.Spec.ObjectStorage.CACert) - } } else if backendType == repoconfig.AzureBackend { - domain, err := getAzureStorageDomain(config) - if err != nil { - return map[string]string{}, errors.Wrapf(err, "error to get azure storage domain") + for k, v := range config { + result[k] = v } - - result[udmrepo.StoreOptionAzureDomain] = domain } result[udmrepo.StoreOptionOssBucket] = bucket result[udmrepo.StoreOptionPrefix] = prefix + if backupLocation.Spec.ObjectStorage != nil && backupLocation.Spec.ObjectStorage.CACert != nil { + result[udmrepo.StoreOptionCACert] = base64.StdEncoding.EncodeToString(backupLocation.Spec.ObjectStorage.CACert) + } result[udmrepo.StoreOptionOssRegion] = strings.Trim(region, "/") result[udmrepo.StoreOptionFsPath] = config["fspath"] diff --git a/pkg/repository/provider/unified_repo_test.go b/pkg/repository/provider/unified_repo_test.go index 08f33eb575..74cdc74b22 100644 --- a/pkg/repository/provider/unified_repo_test.go +++ b/pkg/repository/provider/unified_repo_test.go @@ -23,6 +23,7 @@ import ( "testing" awscredentials "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/kopia/kopia/repo" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -39,16 +40,15 @@ import ( func TestGetStorageCredentials(t *testing.T) { testCases := []struct { - name string - backupLocation velerov1api.BackupStorageLocation - credFileStore *credmock.FileStore - credStoreError error - credStorePath string - getAzureCredentials func(map[string]string) (string, string, error) - getS3Credentials func(map[string]string) (*awscredentials.Value, error) - getGCPCredentials func(map[string]string) string - expected map[string]string - expectedErr string + name string + backupLocation velerov1api.BackupStorageLocation + credFileStore *credmock.FileStore + credStoreError error + credStorePath string + getS3Credentials func(map[string]string) (*awscredentials.Value, error) + getGCPCredentials func(map[string]string) string + expected map[string]string + expectedErr string }{ { name: "invalid credentials file store interface", @@ -160,43 +160,15 @@ func TestGetStorageCredentials(t *testing.T) { expected: map[string]string{}, }, { - name: "azure, Credential section exists in BSL", + name: "azure", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "velero.io/azure", - Config: map[string]string{ - "credentialsFile": "credentials-from-config-map", - }, + Provider: "velero.io/azure", Credential: &corev1api.SecretKeySelector{}, }, }, credFileStore: new(credmock.FileStore), - credStorePath: "credentials-from-credential-key", - getAzureCredentials: func(config map[string]string) (string, string, error) { - return "storage account from: " + config["credentialsFile"], "", nil - }, - - expected: map[string]string{ - "storageAccount": "storage account from: credentials-from-credential-key", - "storageKey": "", - }, - }, - { - name: "azure, get azure credentials fails", - backupLocation: velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "velero.io/azure", - Config: map[string]string{ - "credentialsFile": "credentials-from-config-map", - }, - }, - }, - getAzureCredentials: func(config map[string]string) (string, string, error) { - return "", "", errors.New("fake error") - }, - credFileStore: new(credmock.FileStore), - expected: map[string]string{}, - expectedErr: "error get azure credentials: fake error", + expected: nil, }, { name: "gcp, Credential section not exists in BSL", @@ -220,7 +192,6 @@ func TestGetStorageCredentials(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - getAzureCredentials = tc.getAzureCredentials getS3Credentials = tc.getS3Credentials getGCPCredentials = tc.getGCPCredentials @@ -245,14 +216,14 @@ func TestGetStorageCredentials(t *testing.T) { func TestGetStorageVariables(t *testing.T) { testCases := []struct { - name string - backupLocation velerov1api.BackupStorageLocation - repoName string - repoBackend string - getS3BucketRegion func(string) (string, error) - getAzureStorageDomain func(map[string]string) (string, error) - expected map[string]string - expectedErr string + name string + backupLocation velerov1api.BackupStorageLocation + credFileStore *credmock.FileStore + repoName string + repoBackend string + getS3BucketRegion func(string) (string, error) + expected map[string]string + expectedErr string }{ { name: "invalid provider", @@ -414,37 +385,11 @@ func TestGetStorageVariables(t *testing.T) { "endpoint": "fake-url", "doNotUseTLS": "false", "skipTLSVerify": "false", - "customCA": base64.StdEncoding.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04, 0x05}), + "caCert": base64.StdEncoding.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04, 0x05}), }, }, { - name: "azure, getAzureStorageDomain fail", - backupLocation: velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "velero.io/azure", - Config: map[string]string{ - "bucket": "fake-bucket-config", - "prefix": "fake-prefix-config", - "region": "fake-region", - "fspath": "", - }, - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{ - Bucket: "fake-bucket-object-store", - Prefix: "fake-prefix-object-store", - }, - }, - }, - }, - getAzureStorageDomain: func(config map[string]string) (string, error) { - return "", errors.New("fake error") - }, - repoBackend: "fake-repo-type", - expected: map[string]string{}, - expectedErr: "error to get azure storage domain: fake error", - }, - { - name: "azure, ObjectStorage section exists in BSL", + name: "azure", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/azure", @@ -462,42 +407,13 @@ func TestGetStorageVariables(t *testing.T) { }, }, }, - getAzureStorageDomain: func(config map[string]string) (string, error) { - return "fake-domain", nil - }, - repoBackend: "fake-repo-type", - expected: map[string]string{ - "bucket": "fake-bucket-object-store", - "prefix": "fake-prefix-object-store/fake-repo-type/", - "region": "fake-region", - "fspath": "", - "storageDomain": "fake-domain", - }, - }, - { - name: "azure, ObjectStorage section not exists in BSL, repo name exists", - backupLocation: velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "velero.io/azure", - Config: map[string]string{ - "bucket": "fake-bucket", - "prefix": "fake-prefix", - "region": "fake-region", - "fspath": "", - }, - }, - }, - repoName: "//fake-name//", - repoBackend: "fake-repo-type", - getAzureStorageDomain: func(config map[string]string) (string, error) { - return "fake-domain", nil - }, + credFileStore: new(credmock.FileStore), + repoBackend: "fake-repo-type", expected: map[string]string{ - "bucket": "fake-bucket", - "prefix": "fake-prefix/fake-repo-type/fake-name/", - "region": "fake-region", - "fspath": "", - "storageDomain": "fake-domain", + "bucket": "fake-bucket-object-store", + "prefix": "fake-prefix-object-store/fake-repo-type/", + "region": "fake-region", + "fspath": "", }, }, { @@ -524,7 +440,6 @@ func TestGetStorageVariables(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { getS3BucketRegion = tc.getS3BucketRegion - getAzureStorageDomain = tc.getAzureStorageDomain actual, err := getStorageVariables(&tc.backupLocation, tc.repoBackend, tc.repoName) @@ -734,7 +649,28 @@ func TestPrepareRepo(t *testing.T) { } return errors.New("fake-error-2") }, - expectedErr: "error to init backup repo: fake-error-2", + expectedErr: "error to connect to backup repo: fake-error-1", + }, + { + name: "not initialize", + getter: new(credmock.SecretStore), + credStoreReturn: "fake-password", + funcTable: localFuncTable{ + getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) { + return map[string]string{}, nil + }, + getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { + return map[string]string{}, nil + }, + }, + repoService: new(reposervicenmocks.BackupRepoService), + retFuncInit: func(ctx context.Context, repoOption udmrepo.RepoOptions, createNew bool) error { + if !createNew { + return repo.ErrRepositoryNotInitialized + } + return errors.New("fake-error-2") + }, + expectedErr: "error to create backup repo: fake-error-2", }, } diff --git a/pkg/repository/udmrepo/kopialib/backend/azure.go b/pkg/repository/udmrepo/kopialib/backend/azure.go index d5243820f4..ea7659f93d 100644 --- a/pkg/repository/udmrepo/kopialib/backend/azure.go +++ b/pkg/repository/udmrepo/kopialib/backend/azure.go @@ -20,41 +20,26 @@ import ( "context" "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/blob/azure" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/azure" ) type AzureBackend struct { - options azure.Options + option azure.Option } func (c *AzureBackend) Setup(ctx context.Context, flags map[string]string) error { - var err error - c.options.Container, err = mustHaveString(udmrepo.StoreOptionOssBucket, flags) - if err != nil { - return err + if flags[udmrepo.StoreOptionCACert] != "" { + flags["caCertEncoded"] = "true" } - - c.options.StorageAccount, err = mustHaveString(udmrepo.StoreOptionAzureStorageAccount, flags) - if err != nil { - return err - } - - c.options.StorageKey, err = mustHaveString(udmrepo.StoreOptionAzureKey, flags) - if err != nil { - return err + c.option = azure.Option{ + Config: flags, + Limits: setupLimits(ctx, flags), } - - c.options.Prefix = optionalHaveString(udmrepo.StoreOptionPrefix, flags) - c.options.SASToken = optionalHaveString(udmrepo.StoreOptionAzureToken, flags) - c.options.StorageDomain = optionalHaveString(udmrepo.StoreOptionAzureDomain, flags) - - c.options.Limits = setupLimits(ctx, flags) - return nil } func (c *AzureBackend) Connect(ctx context.Context, isCreate bool) (blob.Storage, error) { - return azure.New(ctx, &c.options, false) + return azure.NewStorage(ctx, &c.option, false) } diff --git a/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper.go b/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper.go new file mode 100644 index 0000000000..046b0459a3 --- /dev/null +++ b/pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper.go @@ -0,0 +1,78 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "context" + + "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/blob/azure" + "github.com/kopia/kopia/repo/blob/throttling" + "github.com/sirupsen/logrus" + + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" + azureutil "github.com/vmware-tanzu/velero/pkg/util/azure" +) + +const ( + storageType = "azure" +) + +func init() { + blob.AddSupportedStorage(storageType, Option{}, NewStorage) +} + +type Option struct { + Config map[string]string `json:"config" kopia:"sensitive"` + Limits throttling.Limits +} + +type Storage struct { + blob.Storage + Option *Option +} + +func (s *Storage) ConnectionInfo() blob.ConnectionInfo { + return blob.ConnectionInfo{ + Type: storageType, + Config: s.Option, + } +} + +func NewStorage(ctx context.Context, option *Option, isCreate bool) (blob.Storage, error) { + cfg := option.Config + + client, _, err := azureutil.NewStorageClient(logrus.New(), cfg) + if err != nil { + return nil, err + } + + opt := &azure.Options{ + Container: cfg[udmrepo.StoreOptionOssBucket], + Prefix: cfg[udmrepo.StoreOptionPrefix], + Limits: option.Limits, + } + azStorage, err := azure.NewWithClient(ctx, opt, client) + if err != nil { + return nil, err + } + + return &Storage{ + Option: option, + Storage: azStorage, + }, nil +} diff --git a/pkg/repository/udmrepo/kopialib/backend/azure_test.go b/pkg/repository/udmrepo/kopialib/backend/azure_test.go index bc4997fbe7..6814c635ab 100644 --- a/pkg/repository/udmrepo/kopialib/backend/azure_test.go +++ b/pkg/repository/udmrepo/kopialib/backend/azure_test.go @@ -20,83 +20,28 @@ import ( "context" "testing" + "github.com/kopia/kopia/repo/blob/throttling" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" - - "github.com/kopia/kopia/repo/blob/azure" - "github.com/kopia/kopia/repo/blob/throttling" ) func TestAzureSetup(t *testing.T) { - testCases := []struct { - name string - flags map[string]string - expected azure.Options - expectedErr string - }{ - { - name: "must have bucket name", - flags: map[string]string{}, - expectedErr: "key " + udmrepo.StoreOptionOssBucket + " not found", - }, - { - name: "must have storage account", - flags: map[string]string{ - udmrepo.StoreOptionOssBucket: "fake-bucket", - }, - expected: azure.Options{ - Container: "fake-bucket", - }, - expectedErr: "key " + udmrepo.StoreOptionAzureStorageAccount + " not found", - }, - { - name: "must have secret key", - flags: map[string]string{ - udmrepo.StoreOptionOssBucket: "fake-bucket", - udmrepo.StoreOptionAzureStorageAccount: "fake-account", - }, - expected: azure.Options{ - Container: "fake-bucket", - StorageAccount: "fake-account", - }, - expectedErr: "key " + udmrepo.StoreOptionAzureKey + " not found", - }, - { - name: "with limits", - flags: map[string]string{ - udmrepo.StoreOptionOssBucket: "fake-bucket", - udmrepo.StoreOptionAzureStorageAccount: "fake-account", - udmrepo.StoreOptionAzureKey: "fake-key", - udmrepo.ThrottleOptionReadOps: "100", - udmrepo.ThrottleOptionUploadBytes: "200", - }, - expected: azure.Options{ - Container: "fake-bucket", - StorageAccount: "fake-account", - StorageKey: "fake-key", - Limits: throttling.Limits{ - ReadsPerSecond: 100, - UploadBytesPerSecond: 200, - }, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - azFlags := AzureBackend{} + backend := AzureBackend{} - err := azFlags.Setup(context.Background(), tc.flags) - - require.Equal(t, tc.expected, azFlags.options) - - if tc.expectedErr == "" { - assert.NoError(t, err) - } else { - assert.EqualError(t, err, tc.expectedErr) - } - }) + flags := map[string]string{ + "key": "value", + udmrepo.ThrottleOptionReadOps: "100", + udmrepo.ThrottleOptionUploadBytes: "200", } + limits := throttling.Limits{ + ReadsPerSecond: 100, + UploadBytesPerSecond: 200, + } + + err := backend.Setup(context.Background(), flags) + require.Nil(t, err) + assert.Equal(t, flags, backend.option.Config) + assert.Equal(t, limits, backend.option.Limits) } diff --git a/pkg/repository/udmrepo/kopialib/backend/common.go b/pkg/repository/udmrepo/kopialib/backend/common.go index 90eb473e07..2896c068fb 100644 --- a/pkg/repository/udmrepo/kopialib/backend/common.go +++ b/pkg/repository/udmrepo/kopialib/backend/common.go @@ -69,9 +69,9 @@ func SetupNewRepositoryOptions(ctx context.Context, flags map[string]string) rep func SetupConnectOptions(ctx context.Context, repoOptions udmrepo.RepoOptions) repo.ConnectOptions { return repo.ConnectOptions{ CachingOptions: content.CachingOptions{ - MaxCacheSizeBytes: maxDataCacheMB << 20, - MaxMetadataCacheSizeBytes: maxMetadataCacheMB << 20, - MaxListCacheDuration: content.DurationSeconds(time.Duration(maxCacheDurationSecond) * time.Second), + ContentCacheSizeBytes: maxDataCacheMB << 20, + MetadataCacheSizeBytes: maxMetadataCacheMB << 20, + MaxListCacheDuration: content.DurationSeconds(time.Duration(maxCacheDurationSecond) * time.Second), }, ClientOptions: repo.ClientOptions{ Hostname: optionalHaveString(udmrepo.GenOptionOwnerDomain, repoOptions.GeneralOptions), diff --git a/pkg/repository/udmrepo/kopialib/backend/common_test.go b/pkg/repository/udmrepo/kopialib/backend/common_test.go index daf6e8479c..8ec90f069a 100644 --- a/pkg/repository/udmrepo/kopialib/backend/common_test.go +++ b/pkg/repository/udmrepo/kopialib/backend/common_test.go @@ -111,9 +111,9 @@ func TestSetupNewRepositoryOptions(t *testing.T) { func TestSetupConnectOptions(t *testing.T) { defaultCacheOption := content.CachingOptions{ - MaxCacheSizeBytes: 2000 << 20, - MaxMetadataCacheSizeBytes: 2000 << 20, - MaxListCacheDuration: content.DurationSeconds(time.Duration(30) * time.Second), + ContentCacheSizeBytes: 2000 << 20, + MetadataCacheSizeBytes: 2000 << 20, + MaxListCacheDuration: content.DurationSeconds(time.Duration(30) * time.Second), } testCases := []struct { diff --git a/pkg/repository/udmrepo/kopialib/backend/mocks/Storage.go b/pkg/repository/udmrepo/kopialib/backend/mocks/Storage.go index de49e75ff1..da816c48cc 100644 --- a/pkg/repository/udmrepo/kopialib/backend/mocks/Storage.go +++ b/pkg/repository/udmrepo/kopialib/backend/mocks/Storage.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. +// Code generated by mockery v2.22.1. DO NOT EDIT. package mocks @@ -71,6 +71,20 @@ func (_m *Storage) DisplayName() string { return r0 } +// ExtendBlobRetention provides a mock function with given fields: ctx, blobID, opts +func (_m *Storage) ExtendBlobRetention(ctx context.Context, blobID blob.ID, opts blob.ExtendOptions) error { + ret := _m.Called(ctx, blobID, opts) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, blob.ID, blob.ExtendOptions) error); ok { + r0 = rf(ctx, blobID, opts) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // FlushCaches provides a mock function with given fields: ctx func (_m *Storage) FlushCaches(ctx context.Context) error { ret := _m.Called(ctx) @@ -104,13 +118,16 @@ func (_m *Storage) GetCapacity(ctx context.Context) (blob.Capacity, error) { ret := _m.Called(ctx) var r0 blob.Capacity + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (blob.Capacity, error)); ok { + return rf(ctx) + } if rf, ok := ret.Get(0).(func(context.Context) blob.Capacity); ok { r0 = rf(ctx) } else { r0 = ret.Get(0).(blob.Capacity) } - var r1 error if rf, ok := ret.Get(1).(func(context.Context) error); ok { r1 = rf(ctx) } else { @@ -125,13 +142,16 @@ func (_m *Storage) GetMetadata(ctx context.Context, blobID blob.ID) (blob.Metada ret := _m.Called(ctx, blobID) var r0 blob.Metadata + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, blob.ID) (blob.Metadata, error)); ok { + return rf(ctx, blobID) + } if rf, ok := ret.Get(0).(func(context.Context, blob.ID) blob.Metadata); ok { r0 = rf(ctx, blobID) } else { r0 = ret.Get(0).(blob.Metadata) } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, blob.ID) error); ok { r1 = rf(ctx, blobID) } else { @@ -141,6 +161,20 @@ func (_m *Storage) GetMetadata(ctx context.Context, blobID blob.ID) (blob.Metada return r0, r1 } +// IsReadOnly provides a mock function with given fields: +func (_m *Storage) IsReadOnly() bool { + ret := _m.Called() + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + // ListBlobs provides a mock function with given fields: ctx, blobIDPrefix, cb func (_m *Storage) ListBlobs(ctx context.Context, blobIDPrefix blob.ID, cb func(blob.Metadata) error) error { ret := _m.Called(ctx, blobIDPrefix, cb) diff --git a/pkg/repository/udmrepo/kopialib/backend/s3.go b/pkg/repository/udmrepo/kopialib/backend/s3.go index 43a7fc821b..075be9af19 100644 --- a/pkg/repository/udmrepo/kopialib/backend/s3.go +++ b/pkg/repository/udmrepo/kopialib/backend/s3.go @@ -44,7 +44,7 @@ func (c *S3Backend) Setup(ctx context.Context, flags map[string]string) error { c.options.DoNotUseTLS = optionalHaveBool(ctx, udmrepo.StoreOptionS3DisableTLS, flags) c.options.DoNotVerifyTLS = optionalHaveBool(ctx, udmrepo.StoreOptionS3DisableTLSVerify, flags) c.options.SessionToken = optionalHaveString(udmrepo.StoreOptionS3Token, flags) - c.options.RootCA = optionalHaveBase64(ctx, udmrepo.StoreOptionS3CustomCA, flags) + c.options.RootCA = optionalHaveBase64(ctx, udmrepo.StoreOptionCACert, flags) c.options.Limits = setupLimits(ctx, flags) diff --git a/pkg/repository/udmrepo/kopialib/backend/s3_test.go b/pkg/repository/udmrepo/kopialib/backend/s3_test.go index 43a761688b..df96de36e9 100644 --- a/pkg/repository/udmrepo/kopialib/backend/s3_test.go +++ b/pkg/repository/udmrepo/kopialib/backend/s3_test.go @@ -95,8 +95,8 @@ func TestS3Setup(t *testing.T) { { name: "with wrong ca", flags: map[string]string{ - udmrepo.StoreOptionOssBucket: "fake-bucket", - udmrepo.StoreOptionS3CustomCA: "fake-base-64", + udmrepo.StoreOptionOssBucket: "fake-bucket", + udmrepo.StoreOptionCACert: "fake-base-64", }, expectedOptions: s3.Options{ BucketName: "fake-bucket", @@ -105,8 +105,8 @@ func TestS3Setup(t *testing.T) { { name: "with correct ca", flags: map[string]string{ - udmrepo.StoreOptionOssBucket: "fake-bucket", - udmrepo.StoreOptionS3CustomCA: "ZmFrZS1jYQ==", + udmrepo.StoreOptionOssBucket: "fake-bucket", + udmrepo.StoreOptionCACert: "ZmFrZS1jYQ==", }, expectedOptions: s3.Options{ BucketName: "fake-bucket", diff --git a/pkg/repository/udmrepo/repo_options.go b/pkg/repository/udmrepo/repo_options.go index 9337018f23..af54e09477 100644 --- a/pkg/repository/udmrepo/repo_options.go +++ b/pkg/repository/udmrepo/repo_options.go @@ -42,12 +42,6 @@ const ( StoreOptionS3Endpoint = "endpoint" StoreOptionS3DisableTLS = "doNotUseTLS" StoreOptionS3DisableTLSVerify = "skipTLSVerify" - StoreOptionS3CustomCA = "customCA" - - StoreOptionAzureKey = "storageKey" - StoreOptionAzureDomain = "storageDomain" - StoreOptionAzureStorageAccount = "storageAccount" - StoreOptionAzureToken = "sasToken" StoreOptionFsPath = "fspath" @@ -55,6 +49,7 @@ const ( StoreOptionOssBucket = "bucket" StoreOptionOssRegion = "region" + StoreOptionCACert = "caCert" StoreOptionCredentialFile = "credFile" StoreOptionPrefix = "prefix" diff --git a/pkg/restore/job_action.go b/pkg/restore/job_action.go index fbaf30b249..364ec79ad7 100644 --- a/pkg/restore/job_action.go +++ b/pkg/restore/job_action.go @@ -26,6 +26,11 @@ import ( "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) +const ( + legacyControllerUIDLabel = "controller-uid" // <=1.27 This still exists in 1.27 for backward compatibility, maybe remove in 1.28? + controllerUIDLabel = "batch.kubernetes.io/controller-uid" // >=1.27 https://github.com/kubernetes/kubernetes/pull/114930#issuecomment-1384667494 +) + type JobAction struct { logger logrus.FieldLogger } @@ -47,9 +52,11 @@ func (a *JobAction) Execute(input *velero.RestoreItemActionExecuteInput) (*veler } if job.Spec.Selector != nil { - delete(job.Spec.Selector.MatchLabels, "controller-uid") + delete(job.Spec.Selector.MatchLabels, controllerUIDLabel) + delete(job.Spec.Selector.MatchLabels, legacyControllerUIDLabel) } - delete(job.Spec.Template.ObjectMeta.Labels, "controller-uid") + delete(job.Spec.Template.ObjectMeta.Labels, controllerUIDLabel) + delete(job.Spec.Template.ObjectMeta.Labels, legacyControllerUIDLabel) res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(job) if err != nil { diff --git a/pkg/uploader/kopia/block_backup.go b/pkg/uploader/kopia/block_backup.go new file mode 100644 index 0000000000..a637925a49 --- /dev/null +++ b/pkg/uploader/kopia/block_backup.go @@ -0,0 +1,55 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kopia + +import ( + "os" + "syscall" + + "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/fs/virtualfs" + "github.com/pkg/errors" +) + +const ErrNotPermitted = "operation not permitted" + +func getLocalBlockEntry(sourcePath string) (fs.Entry, error) { + source, err := resolveSymlink(sourcePath) + if err != nil { + return nil, errors.Wrap(err, "resolveSymlink") + } + + fileInfo, err := os.Lstat(source) + if err != nil { + return nil, errors.Wrapf(err, "unable to get the source device information %s", source) + } + + if (fileInfo.Sys().(*syscall.Stat_t).Mode & syscall.S_IFMT) != syscall.S_IFBLK { + return nil, errors.Errorf("source path %s is not a block device", source) + } + + device, err := os.Open(source) + if err != nil { + if os.IsPermission(err) || err.Error() == ErrNotPermitted { + return nil, errors.Wrapf(err, "no permission to open the source device %s, make sure that node agent is running in privileged mode", source) + } + return nil, errors.Wrapf(err, "unable to open the source device %s", source) + } + + sf := virtualfs.StreamingFileFromReader(source, device) + return virtualfs.NewStaticDirectory(source, []fs.Entry{sf}), nil +} diff --git a/pkg/uploader/kopia/block_restore.go b/pkg/uploader/kopia/block_restore.go new file mode 100644 index 0000000000..25d11ee24e --- /dev/null +++ b/pkg/uploader/kopia/block_restore.go @@ -0,0 +1,99 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kopia + +import ( + "context" + "io" + "os" + "path/filepath" + "syscall" + + "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/snapshot/restore" + "github.com/pkg/errors" +) + +type BlockOutput struct { + *restore.FilesystemOutput + + targetFileName string +} + +var _ restore.Output = &BlockOutput{} + +const bufferSize = 128 * 1024 + +func (o *BlockOutput) WriteFile(ctx context.Context, relativePath string, remoteFile fs.File) error { + remoteReader, err := remoteFile.Open(ctx) + if err != nil { + return errors.Wrapf(err, "failed to open remote file %s", remoteFile.Name()) + } + defer remoteReader.Close() + + targetFile, err := os.Create(o.targetFileName) + if err != nil { + return errors.Wrapf(err, "failed to open file %s", o.targetFileName) + } + defer targetFile.Close() + + buffer := make([]byte, bufferSize) + + readData := true + for readData { + bytesToWrite, err := remoteReader.Read(buffer) + if err != nil { + if err != io.EOF { + return errors.Wrapf(err, "failed to read data from remote file %s", o.targetFileName) + } + readData = false + } + + if bytesToWrite > 0 { + offset := 0 + for bytesToWrite > 0 { + if bytesWritten, err := targetFile.Write(buffer[offset:bytesToWrite]); err == nil { + bytesToWrite -= bytesWritten + offset += bytesWritten + } else { + return errors.Wrapf(err, "failed to write data to file %s", o.targetFileName) + } + } + } + } + + return nil +} + +func (o *BlockOutput) BeginDirectory(ctx context.Context, relativePath string, e fs.Directory) error { + var err error + o.targetFileName, err = filepath.EvalSymlinks(o.TargetPath) + if err != nil { + return errors.Wrapf(err, "unable to evaluate symlinks for %s", o.targetFileName) + } + + fileInfo, err := os.Lstat(o.targetFileName) + if err != nil { + return errors.Wrapf(err, "unable to get the target device information for %s", o.TargetPath) + } + + if (fileInfo.Sys().(*syscall.Stat_t).Mode & syscall.S_IFMT) != syscall.S_IFBLK { + return errors.Errorf("target file %s is not a block device", o.TargetPath) + } + + return nil +} diff --git a/pkg/uploader/kopia/snapshot.go b/pkg/uploader/kopia/snapshot.go index b207713f63..27ac842532 100644 --- a/pkg/uploader/kopia/snapshot.go +++ b/pkg/uploader/kopia/snapshot.go @@ -68,6 +68,11 @@ func newOptionalInt(b int) *policy.OptionalInt { return &ob } +func newOptionalInt64(b int64) *policy.OptionalInt64 { + ob := policy.OptionalInt64(b) + return &ob +} + func newOptionalBool(b bool) *policy.OptionalBool { ob := policy.OptionalBool(b) return &ob @@ -88,7 +93,7 @@ func getDefaultPolicy() *policy.Policy { }, UploadPolicy: policy.UploadPolicy{ MaxParallelFileReads: newOptionalInt(runtime.NumCPU()), - ParallelUploadAboveSize: nil, + ParallelUploadAboveSize: newOptionalInt64(math.MaxInt64), }, SchedulingPolicy: policy.SchedulingPolicy{ Manual: true, @@ -106,6 +111,11 @@ func setupDefaultPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceIn return nil, errors.Wrap(err, "error to set policy") } + err = rep.Flush(ctx) + if err != nil { + return nil, errors.Wrap(err, "error to flush repo") + } + // retrieve policy from repo policyTree, err := treeForSourceFunc(ctx, rep, sourceInfo) if err != nil { @@ -121,25 +131,22 @@ func Backup(ctx context.Context, fsUploader SnapshotUploader, repoWriter repo.Re if fsUploader == nil { return nil, false, errors.New("get empty kopia uploader") } - - if volMode == uploader.PersistentVolumeBlock { - return nil, false, errors.New("unable to handle block storage") - } - - dir, err := filepath.Abs(sourcePath) + source, err := filepath.Abs(sourcePath) if err != nil { return nil, false, errors.Wrapf(err, "Invalid source path '%s'", sourcePath) } - // to be consistent with restic when backup empty dir returns one error for upper logic handle - dirs, err := os.ReadDir(dir) - if err != nil { - return nil, false, errors.Wrapf(err, "Unable to read dir in path %s", dir) - } else if len(dirs) == 0 { - return nil, true, nil + if volMode == uploader.PersistentVolumeFilesystem { + // to be consistent with restic when backup empty dir returns one error for upper logic handle + dirs, err := os.ReadDir(source) + if err != nil { + return nil, false, errors.Wrapf(err, "Unable to read dir in path %s", source) + } else if len(dirs) == 0 { + return nil, true, nil + } } - dir = filepath.Clean(dir) + source = filepath.Clean(source) sourceInfo := snapshot.SourceInfo{ UserName: udmrepo.GetRepoUser(), @@ -147,16 +154,25 @@ func Backup(ctx context.Context, fsUploader SnapshotUploader, repoWriter repo.Re Path: filepath.Clean(realSource), } if realSource == "" { - sourceInfo.Path = dir + sourceInfo.Path = source } - rootDir, err := getLocalFSEntry(dir) - if err != nil { - return nil, false, errors.Wrap(err, "Unable to get local filesystem entry") + var sourceEntry fs.Entry + + if volMode == uploader.PersistentVolumeBlock { + sourceEntry, err = getLocalBlockEntry(source) + if err != nil { + return nil, false, errors.Wrap(err, "unable to get local block device entry") + } + } else { + sourceEntry, err = getLocalFSEntry(source) + if err != nil { + return nil, false, errors.Wrap(err, "unable to get local filesystem entry") + } } kopiaCtx := kopia.SetupKopiaLog(ctx, log) - snapID, snapshotSize, err := SnapshotSource(kopiaCtx, repoWriter, fsUploader, sourceInfo, rootDir, forceFull, parentSnapshot, tags, log, "Kopia Uploader") + snapID, snapshotSize, err := SnapshotSource(kopiaCtx, repoWriter, fsUploader, sourceInfo, sourceEntry, forceFull, parentSnapshot, tags, log, "Kopia Uploader") if err != nil { return nil, false, err } @@ -338,7 +354,8 @@ func findPreviousSnapshotManifest(ctx context.Context, rep repo.Repository, sour } // Restore restore specific sourcePath with given snapshotID and update progress -func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { +func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, + log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { log.Info("Start to restore...") kopiaCtx := kopia.SetupKopiaLog(ctx, log) @@ -360,7 +377,7 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, return 0, 0, errors.Wrapf(err, "Unable to resolve path %v", dest) } - output := &restore.FilesystemOutput{ + fsOutput := &restore.FilesystemOutput{ TargetPath: path, OverwriteDirectories: true, OverwriteFiles: true, @@ -368,11 +385,18 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, IgnorePermissionErrors: true, } - err = output.Init(ctx) + err = fsOutput.Init(ctx) if err != nil { return 0, 0, errors.Wrap(err, "error to init output") } + var output restore.Output = fsOutput + if volMode == uploader.PersistentVolumeBlock { + output = &BlockOutput{ + FilesystemOutput: fsOutput, + } + } + stat, err := restoreEntryFunc(kopiaCtx, rep, output, rootEntry, restore.Options{ Parallel: runtime.NumCPU(), RestoreDirEntryAtDepth: math.MaxInt32, diff --git a/pkg/uploader/kopia/snapshot_test.go b/pkg/uploader/kopia/snapshot_test.go index 232ea92c4d..65ec22136a 100644 --- a/pkg/uploader/kopia/snapshot_test.go +++ b/pkg/uploader/kopia/snapshot_test.go @@ -23,6 +23,7 @@ import ( "time" "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/fs/virtualfs" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" @@ -594,11 +595,11 @@ func TestBackup(t *testing.T) { expectedError: errors.New("Unable to read dir"), }, { - name: "Unable to handle block mode", + name: "Source path is not a block device", sourcePath: "/", tags: nil, volMode: uploader.PersistentVolumeBlock, - expectedError: errors.New("unable to handle block storage"), + expectedError: errors.New("source path / is not a block device"), }, } @@ -660,6 +661,7 @@ func TestRestore(t *testing.T) { expectedBytes int64 expectedCount int32 expectedError error + volMode uploader.PersistentVolumeMode } // Define test cases @@ -697,6 +699,46 @@ func TestRestore(t *testing.T) { snapshotID: "snapshot-123", expectedError: nil, }, + { + name: "Expect block volume successful", + filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { + return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil + }, + restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { + return restore.Stats{}, nil + }, + snapshotID: "snapshot-123", + expectedError: nil, + volMode: uploader.PersistentVolumeBlock, + }, + { + name: "Unable to evaluate symlinks for block volume", + filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { + return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil + }, + restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { + err := output.BeginDirectory(ctx, "fake-dir", virtualfs.NewStaticDirectory("fake-dir", nil)) + return restore.Stats{}, err + }, + snapshotID: "snapshot-123", + expectedError: errors.New("unable to evaluate symlinks for"), + volMode: uploader.PersistentVolumeBlock, + dest: "/wrong-dest", + }, + { + name: "Target file is not a block device", + filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { + return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil + }, + restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { + err := output.BeginDirectory(ctx, "fake-dir", virtualfs.NewStaticDirectory("fake-dir", nil)) + return restore.Stats{}, err + }, + snapshotID: "snapshot-123", + expectedError: errors.New("target file /tmp is not a block device"), + volMode: uploader.PersistentVolumeBlock, + dest: "/tmp", + }, } em := &manifest.EntryMetadata{ @@ -706,6 +748,10 @@ func TestRestore(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + if tc.volMode == "" { + tc.volMode = uploader.PersistentVolumeFilesystem + } + if tc.invalidManifestType { em.Labels[manifest.TypeLabelKey] = "" } else { @@ -725,7 +771,7 @@ func TestRestore(t *testing.T) { repoWriterMock.On("OpenObject", mock.Anything, mock.Anything).Return(em, nil) progress := new(Progress) - bytesRestored, fileCount, err := Restore(context.Background(), repoWriterMock, progress, tc.snapshotID, tc.dest, logrus.New(), nil) + bytesRestored, fileCount, err := Restore(context.Background(), repoWriterMock, progress, tc.snapshotID, tc.dest, tc.volMode, logrus.New(), nil) // Check if the returned error matches the expected error if tc.expectedError != nil { diff --git a/pkg/uploader/provider/kopia.go b/pkg/uploader/provider/kopia.go index dd32173b82..706393362d 100644 --- a/pkg/uploader/provider/kopia.go +++ b/pkg/uploader/provider/kopia.go @@ -128,11 +128,6 @@ func (kp *kopiaProvider) RunBackup( return "", false, errors.New("path is empty") } - // For now, error on block mode - if volMode == uploader.PersistentVolumeBlock { - return "", false, errors.New("unable to currently support block mode") - } - log := kp.log.WithFields(logrus.Fields{ "path": path, "realSource": realSource, @@ -214,10 +209,6 @@ func (kp *kopiaProvider) RunRestore( "volumePath": volumePath, }) - if volMode == uploader.PersistentVolumeBlock { - return errors.New("unable to currently support block mode") - } - repoWriter := kopia.NewShimRepo(kp.bkRepo) progress := new(kopia.Progress) progress.InitThrottle(restoreProgressCheckInterval) @@ -235,7 +226,7 @@ func (kp *kopiaProvider) RunRestore( // We use the cancel channel to control the restore cancel, so don't pass a context with cancel to Kopia restore. // Otherwise, Kopia restore will not response to the cancel control but return an arbitrary error. // Kopia restore cancel is not designed as well as Kopia backup which uses the context to control backup cancel all the way. - size, fileCount, err := RestoreFunc(context.Background(), repoWriter, progress, snapshotID, volumePath, log, restoreCancel) + size, fileCount, err := RestoreFunc(context.Background(), repoWriter, progress, snapshotID, volumePath, volMode, log, restoreCancel) if err != nil { return errors.Wrapf(err, "Failed to run kopia restore") diff --git a/pkg/uploader/provider/kopia_test.go b/pkg/uploader/provider/kopia_test.go index 944cdbcceb..c38d370ce3 100644 --- a/pkg/uploader/provider/kopia_test.go +++ b/pkg/uploader/provider/kopia_test.go @@ -94,12 +94,12 @@ func TestRunBackup(t *testing.T) { notError: false, }, { - name: "error on vol mode", + name: "success to backup block mode volume", hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) { - return nil, true, nil + return &uploader.SnapshotInfo{}, false, nil }, volMode: uploader.PersistentVolumeBlock, - notError: false, + notError: true, }, } for _, tc := range testCases { @@ -125,31 +125,31 @@ func TestRunRestore(t *testing.T) { testCases := []struct { name string - hookRestoreFunc func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) + hookRestoreFunc func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) notError bool volMode uploader.PersistentVolumeMode }{ { name: "normal restore", - hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { + hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { return 0, 0, nil }, notError: true, }, { name: "failed to restore", - hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { + hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { return 0, 0, errors.New("failed to restore") }, notError: false, }, { - name: "failed to restore block mode", - hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { - return 0, 0, errors.New("failed to restore") + name: "normal block mode restore", + hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { + return 0, 0, nil }, volMode: uploader.PersistentVolumeBlock, - notError: false, + notError: true, }, } diff --git a/pkg/uploader/provider/restic_test.go b/pkg/uploader/provider/restic_test.go index f2203d7bdd..62f44d04f3 100644 --- a/pkg/uploader/provider/restic_test.go +++ b/pkg/uploader/provider/restic_test.go @@ -212,6 +212,9 @@ func TestResticRunRestore(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + if tc.volMode == "" { + tc.volMode = uploader.PersistentVolumeFilesystem + } resticRestoreCMDFunc = tc.hookResticRestoreFunc if tc.volMode == "" { tc.volMode = uploader.PersistentVolumeFilesystem diff --git a/pkg/util/azure/credential.go b/pkg/util/azure/credential.go new file mode 100644 index 0000000000..6d5e16ded3 --- /dev/null +++ b/pkg/util/azure/credential.go @@ -0,0 +1,133 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "os" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/pkg/errors" +) + +// NewCredential chains the config credential and workload identity credential +func NewCredential(creds map[string]string, options policy.ClientOptions) (azcore.TokenCredential, error) { + var ( + credential []azcore.TokenCredential + errMsgs []string + ) + + additionalTenants := []string{} + if tenants := creds[CredentialKeyAdditionallyAllowedTenants]; tenants != "" { + additionalTenants = strings.Split(tenants, ";") + } + + // config credential + cfgCred, err := newConfigCredential(creds, configCredentialOptions{ + ClientOptions: options, + AdditionallyAllowedTenants: additionalTenants, + }) + if err == nil { + credential = append(credential, cfgCred) + } else { + errMsgs = append(errMsgs, err.Error()) + } + + // workload identity credential + wic, err := azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{ + AdditionallyAllowedTenants: additionalTenants, + ClientOptions: options, + }) + if err == nil { + credential = append(credential, wic) + } else { + errMsgs = append(errMsgs, err.Error()) + } + + if len(credential) == 0 { + return nil, errors.Errorf("failed to create Azure credential: %s", strings.Join(errMsgs, "\n\t")) + } + + return azidentity.NewChainedTokenCredential(credential, nil) +} + +type configCredentialOptions struct { + azcore.ClientOptions + AdditionallyAllowedTenants []string +} + +// newConfigCredential works same as the azidentity.EnvironmentCredential but reads the credentials from a map +// rather than environment variables. This is required for Velero to run B/R concurrently +// https://github.com/Azure/azure-sdk-for-go/blob/sdk/azidentity/v1.3.0/sdk/azidentity/environment_credential.go#L80 +func newConfigCredential(creds map[string]string, options configCredentialOptions) (azcore.TokenCredential, error) { + tenantID := creds[CredentialKeyTenantID] + if tenantID == "" { + return nil, errors.Errorf("%s is required", CredentialKeyTenantID) + } + clientID := creds[CredentialKeyClientID] + if clientID == "" { + return nil, errors.Errorf("%s is required", CredentialKeyClientID) + } + + // client secret + if clientSecret := creds[CredentialKeyClientSecret]; clientSecret != "" { + return azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, &azidentity.ClientSecretCredentialOptions{ + AdditionallyAllowedTenants: options.AdditionallyAllowedTenants, + ClientOptions: options.ClientOptions, + }) + } + + // certificate + if certPath := creds[CredentialKeyClientCertificatePath]; certPath != "" { + certData, err := os.ReadFile(certPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to read certificate file %s", certPath) + } + var password []byte + if v := creds[CredentialKeyClientCertificatePassword]; v != "" { + password = []byte(v) + } + certs, key, err := azidentity.ParseCertificates(certData, password) + if err != nil { + return nil, errors.Wrapf(err, "failed to load certificate from %s", certPath) + } + o := &azidentity.ClientCertificateCredentialOptions{ + AdditionallyAllowedTenants: options.AdditionallyAllowedTenants, + ClientOptions: options.ClientOptions, + } + if v, ok := creds[CredentialKeySendCertChain]; ok { + o.SendCertificateChain = v == "1" || strings.ToLower(v) == "true" + } + return azidentity.NewClientCertificateCredential(tenantID, clientID, certs, key, o) + } + + // username/password + if username := creds[CredentialKeyUsername]; username != "" { + if password := creds[CredentialKeyPassword]; password != "" { + return azidentity.NewUsernamePasswordCredential(tenantID, clientID, username, password, + &azidentity.UsernamePasswordCredentialOptions{ + AdditionallyAllowedTenants: options.AdditionallyAllowedTenants, + ClientOptions: options.ClientOptions, + }) + } + return nil, errors.Errorf("%s is required", CredentialKeyPassword) + } + + return nil, errors.New("incomplete credential configuration. Only AZURE_TENANT_ID and AZURE_CLIENT_ID are set") +} diff --git a/pkg/util/azure/credential_test.go b/pkg/util/azure/credential_test.go new file mode 100644 index 0000000000..4d6d7f0d33 --- /dev/null +++ b/pkg/util/azure/credential_test.go @@ -0,0 +1,96 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/stretchr/testify/require" +) + +func TestNewCredential(t *testing.T) { + options := policy.ClientOptions{} + + // no credentials + creds := map[string]string{} + _, err := NewCredential(creds, options) + require.NotNil(t, err) + + // config credential + creds = map[string]string{ + CredentialKeyTenantID: "tenantid", + CredentialKeyClientID: "clientid", + CredentialKeyClientSecret: "secret", + } + _, err = NewCredential(creds, options) + require.Nil(t, err) +} + +func Test_newConfigCredential(t *testing.T) { + options := configCredentialOptions{} + + // tenantID not specified + creds := map[string]string{} + _, err := newConfigCredential(creds, options) + require.NotNil(t, err) + + // clientID not specified + creds = map[string]string{ + CredentialKeyTenantID: "clientid", + } + _, err = newConfigCredential(creds, options) + require.NotNil(t, err) + + // client secret + creds = map[string]string{ + CredentialKeyTenantID: "clientid", + CredentialKeyClientID: "clientid", + CredentialKeyClientSecret: "secret", + } + credential, err := newConfigCredential(creds, options) + require.Nil(t, err) + require.NotNil(t, credential) + _, ok := credential.(*azidentity.ClientSecretCredential) + require.True(t, ok) + + // client certificate + creds = map[string]string{ + CredentialKeyTenantID: "clientid", + CredentialKeyClientID: "clientid", + CredentialKeyClientCertificatePath: "testdata/certificate.pem", + } + credential, err = newConfigCredential(creds, options) + require.Nil(t, err) + require.NotNil(t, credential) + _, ok = credential.(*azidentity.ClientCertificateCredential) + require.True(t, ok) + + // username/password + creds = map[string]string{ + CredentialKeyTenantID: "clientid", + CredentialKeyClientID: "clientid", + CredentialKeyUsername: "username", + CredentialKeyPassword: "password", + } + credential, err = newConfigCredential(creds, options) + require.Nil(t, err) + require.NotNil(t, credential) + _, ok = credential.(*azidentity.UsernamePasswordCredential) + require.True(t, ok) +} diff --git a/pkg/util/azure/storage.go b/pkg/util/azure/storage.go new file mode 100644 index 0000000000..49943a3f92 --- /dev/null +++ b/pkg/util/azure/storage.go @@ -0,0 +1,276 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + _ "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const ( + // the keys of Azure BSL config: + // https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/blob/main/backupstoragelocation.md + BSLConfigResourceGroup = "resourceGroup" + BSLConfigStorageAccount = "storageAccount" + BSLConfigStorageAccountAccessKeyName = "storageAccountKeyEnvVar" + BSLConfigSubscriptionID = "subscriptionId" + BSLConfigStorageAccountURI = "storageAccountURI" + BSLConfigUseAAD = "useAAD" + BSLConfigActiveDirectoryAuthorityURI = "activeDirectoryAuthorityURI" + + serviceNameBlob cloud.ServiceName = "blob" +) + +func init() { + cloud.AzureChina.Services[serviceNameBlob] = cloud.ServiceConfiguration{ + Endpoint: "blob.core.chinacloudapi.cn", + } + cloud.AzureGovernment.Services[serviceNameBlob] = cloud.ServiceConfiguration{ + Endpoint: "blob.core.usgovcloudapi.net", + } + cloud.AzurePublic.Services[serviceNameBlob] = cloud.ServiceConfiguration{ + Endpoint: "blob.core.windows.net", + } +} + +// NewStorageClient creates a blob storage client(data plane) with the provided config which contains BSL config and the credential file name. +// The returned azblob.SharedKeyCredential is needed for Azure plugin to generate the SAS URL when auth with storage +// account access key +func NewStorageClient(log logrus.FieldLogger, config map[string]string) (*azblob.Client, *azblob.SharedKeyCredential, error) { + // rename to bslCfg for easy understanding + bslCfg := config + + // storage account is required + storageAccount := bslCfg[BSLConfigStorageAccount] + if storageAccount == "" { + return nil, nil, errors.Errorf("%s is required in BSL", BSLConfigStorageAccount) + } + + // read the credentials provided by users + creds, err := LoadCredentials(config) + if err != nil { + return nil, nil, err + } + // exchange the storage account access key if needed + creds, err = GetStorageAccountCredentials(bslCfg, creds) + if err != nil { + return nil, nil, err + } + + // get the storage account URI + uri, err := getStorageAccountURI(log, bslCfg, creds) + if err != nil { + return nil, nil, err + } + + clientOptions, err := GetClientOptions(bslCfg, creds) + if err != nil { + return nil, nil, err + } + blobClientOptions := &azblob.ClientOptions{ + ClientOptions: clientOptions, + } + + // auth with storage account access key + accessKey := creds[CredentialKeyStorageAccountAccessKey] + if accessKey != "" { + log.Info("auth with the storage account access key") + cred, err := azblob.NewSharedKeyCredential(storageAccount, accessKey) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to create storage account access key credential") + } + client, err := azblob.NewClientWithSharedKeyCredential(uri, cred, blobClientOptions) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to create blob client with the storage account access key") + } + return client, cred, nil + } + + // auth with Azure AD + log.Info("auth with Azure AD") + cred, err := NewCredential(creds, clientOptions) + if err != nil { + return nil, nil, err + } + client, err := azblob.NewClient(uri, cred, blobClientOptions) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to create blob client with the Azure AD credential") + } + return client, nil, nil +} + +// GetStorageAccountCredentials returns the credentials to interactive with storage account according to the config of BSL +// and credential file by the following order: +// 1. Return the storage account access key directly if it is provided +// 2. Return the content of the credential file directly if "userAAD" is set as true in BSL config +// 3. Call Azure API to exchange the storage account access key +func GetStorageAccountCredentials(bslCfg map[string]string, creds map[string]string) (map[string]string, error) { + // use storage account access key if specified + if name := bslCfg[BSLConfigStorageAccountAccessKeyName]; name != "" { + accessKey := creds[name] + if accessKey == "" { + return nil, errors.Errorf("no storage account access key with key %s found in credential", name) + } + creds[CredentialKeyStorageAccountAccessKey] = accessKey + return creds, nil + } + + // use AAD + if bslCfg[BSLConfigUseAAD] != "" { + useAAD, err := strconv.ParseBool(bslCfg[BSLConfigUseAAD]) + if err != nil { + return nil, errors.Errorf("failed to parse bool from useAAD string: %s", bslCfg[BSLConfigUseAAD]) + } + + if useAAD { + return creds, nil + } + } + + // exchange the storage account access key + accessKey, err := exchangeStorageAccountAccessKey(bslCfg, creds) + if err != nil { + return nil, errors.WithMessage(err, "failed to get storage account access key") + } + creds[CredentialKeyStorageAccountAccessKey] = accessKey + return creds, nil +} + +// getStorageAccountURI returns the storage account URI by the following order: +// 1. Return the storage account URI directly if it is specified in BSL config +// 2. Try to call Azure API to get the storage account URI if possible(Background: https://github.com/vmware-tanzu/velero/issues/6163) +// 3. Fall back to return the default URI +func getStorageAccountURI(log logrus.FieldLogger, bslCfg map[string]string, creds map[string]string) (string, error) { + // if the URI is specified in the BSL, return it directly + uri := bslCfg[BSLConfigStorageAccountURI] + if uri != "" { + log.Infof("the storage account URI %q is specified in the BSL, use it directly", uri) + return uri, nil + } + + storageAccount := bslCfg[BSLConfigStorageAccount] + + cloudCfg, err := getCloudConfiguration(bslCfg, creds) + if err != nil { + return "", err + } + + // the default URI + uri = fmt.Sprintf("https://%s.%s", storageAccount, cloudCfg.Services[serviceNameBlob].Endpoint) + + // the storage account access key cannot be used to get the storage account properties, + // so fallback to the default URI + if name := bslCfg[BSLConfigStorageAccountAccessKeyName]; name != "" && creds[name] != "" { + log.Infof("auth with the storage account access key, cannot retrieve the storage account properties, fallback to use the default URI %q", uri) + return uri, nil + } + + client, err := newStorageAccountManagemenClient(bslCfg, creds) + if err != nil { + log.Infof("failed to create the storage account management client: %v, fallback to use the default URI %q", err, uri) + return uri, nil + } + + resourceGroup := GetFromLocationConfigOrCredential(bslCfg, creds, BSLConfigResourceGroup, CredentialKeyResourceGroup) + // we cannot get the storage account properties without the resource group, so fallback to the default URI + if resourceGroup == "" { + log.Infof("resource group isn't set which is required to retrieve the storage account properties, fallback to use the default URI %q", uri) + return uri, nil + } + + properties, err := client.GetProperties(context.Background(), resourceGroup, storageAccount, nil) + // get error, fallback to the default URI + if err != nil { + log.Infof("failed to retrieve the storage account properties: %v, fallback to use the default URI %q", err, uri) + return uri, nil + } + + uri = *properties.Account.Properties.PrimaryEndpoints.Blob + log.Infof("use the storage account URI retrieved from the storage account properties %q", uri) + return uri, nil +} + +// try to exchange the storage account access key with the provided credentials +func exchangeStorageAccountAccessKey(bslCfg, creds map[string]string) (string, error) { + client, err := newStorageAccountManagemenClient(bslCfg, creds) + if err != nil { + return "", err + } + + resourceGroup := GetFromLocationConfigOrCredential(bslCfg, creds, BSLConfigResourceGroup, CredentialKeyResourceGroup) + if resourceGroup == "" { + return "", errors.New("resource group is required in BSL or credential to exchange the storage account access key") + } + storageAccount := bslCfg[BSLConfigStorageAccount] + if storageAccount == "" { + return "", errors.Errorf("%s is required in the BSL to exchange the storage account access key", BSLConfigStorageAccount) + } + + expand := "kerb" + resp, err := client.ListKeys(context.Background(), resourceGroup, storageAccount, &armstorage.AccountsClientListKeysOptions{ + Expand: &expand, + }) + if err != nil { + return "", errors.Wrap(err, "failed to list storage account access keys") + } + for _, key := range resp.Keys { + if key == nil || key.Permissions == nil { + continue + } + if strings.EqualFold(string(*key.Permissions), string(armstorage.KeyPermissionFull)) { + return *key.Value, nil + } + } + return "", errors.New("no storage key with Full permissions found") +} + +// new a management client for the storage account +func newStorageAccountManagemenClient(bslCfg map[string]string, creds map[string]string) (*armstorage.AccountsClient, error) { + clientOptions, err := GetClientOptions(bslCfg, creds) + if err != nil { + return nil, err + } + + cred, err := NewCredential(creds, clientOptions) + if err != nil { + return nil, errors.WithMessage(err, "failed to create Azure AD credential") + } + + subID := GetFromLocationConfigOrCredential(bslCfg, creds, BSLConfigSubscriptionID, CredentialKeySubscriptionID) + if subID == "" { + return nil, errors.New("subscription ID is required in BSL or credential to create the storage account client") + } + + client, err := armstorage.NewAccountsClient(subID, cred, &arm.ClientOptions{ + ClientOptions: clientOptions, + }) + if err != nil { + return nil, errors.Wrap(err, "failed to create the storage account client") + } + + return client, nil +} diff --git a/pkg/util/azure/storage_test.go b/pkg/util/azure/storage_test.go new file mode 100644 index 0000000000..e32b3e340f --- /dev/null +++ b/pkg/util/azure/storage_test.go @@ -0,0 +1,223 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "os" + "path/filepath" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewStorageClient(t *testing.T) { + log := logrus.New() + config := map[string]string{} + + name := filepath.Join(os.TempDir(), "credential") + file, err := os.Create(name) + require.Nil(t, err) + defer file.Close() + defer os.Remove(name) + _, err = file.WriteString("AccessKey: YWNjZXNza2V5\nAZURE_TENANT_ID: tenantid\nAZURE_CLIENT_ID: clientid\nAZURE_CLIENT_SECRET: secret") + require.Nil(t, err) + + // storage account isn't specified + _, _, err = NewStorageClient(log, config) + require.NotNil(t, err) + + // auth with storage account access key + config = map[string]string{ + BSLConfigStorageAccount: "storage-account", + "credentialsFile": name, + BSLConfigStorageAccountAccessKeyName: "AccessKey", + } + client, credential, err := NewStorageClient(log, config) + require.Nil(t, err) + assert.NotNil(t, client) + assert.NotNil(t, credential) + + // auth with Azure AD + config = map[string]string{ + BSLConfigStorageAccount: "storage-account", + "credentialsFile": name, + "useAAD": "true", + } + client, credential, err = NewStorageClient(log, config) + require.Nil(t, err) + assert.NotNil(t, client) + assert.Nil(t, credential) +} + +func TestGetStorageAccountCredentials(t *testing.T) { + // use access secret but no secret specified + cfg := map[string]string{ + BSLConfigStorageAccountAccessKeyName: "KEY", + } + creds := map[string]string{} + _, err := GetStorageAccountCredentials(cfg, creds) + require.NotNil(t, err) + + // use access secret + cfg = map[string]string{ + BSLConfigStorageAccountAccessKeyName: "KEY", + } + creds = map[string]string{ + "KEY": "key", + } + m, err := GetStorageAccountCredentials(cfg, creds) + require.Nil(t, err) + assert.Equal(t, "key", m[CredentialKeyStorageAccountAccessKey]) + + // use AAD, but useAAD invalid + cfg = map[string]string{ + "useAAD": "invalid", + } + creds = map[string]string{} + _, err = GetStorageAccountCredentials(cfg, creds) + require.NotNil(t, err) + + // use AAD + cfg = map[string]string{ + "useAAD": "true", + } + creds = map[string]string{ + "KEY": "key", + } + m, err = GetStorageAccountCredentials(cfg, creds) + require.Nil(t, err) + assert.Equal(t, creds, m) +} + +func Test_getStorageAccountURI(t *testing.T) { + log := logrus.New() + + // URI specified + bslCfg := map[string]string{ + BSLConfigStorageAccountURI: "uri", + } + creds := map[string]string{} + uri, err := getStorageAccountURI(log, bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, "uri", uri) + + // no URI specified, and auth with access key + bslCfg = map[string]string{ + BSLConfigStorageAccountAccessKeyName: "KEY", + } + creds = map[string]string{ + "KEY": "value", + } + uri, err = getStorageAccountURI(log, bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, "https://.blob.core.windows.net", uri) + + // no URI specified, auth with AAD, resource group isn't specified + bslCfg = map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + } + creds = map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + uri, err = getStorageAccountURI(log, bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, "https://.blob.core.windows.net", uri) + + // no URI specified, auth with AAD, resource group specified + bslCfg = map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + BSLConfigResourceGroup: "resourcegroup", + BSLConfigStorageAccount: "account", + } + creds = map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + uri, err = getStorageAccountURI(log, bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, "https://account.blob.core.windows.net", uri) +} + +func Test_exchangeStorageAccountAccessKey(t *testing.T) { + // resource group isn't specified + bslCfg := map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + } + creds := map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + _, err := exchangeStorageAccountAccessKey(bslCfg, creds) + require.NotNil(t, err) + + // storage account isn't specified + bslCfg = map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + BSLConfigResourceGroup: "resourcegroup", + } + creds = map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + _, err = exchangeStorageAccountAccessKey(bslCfg, creds) + require.NotNil(t, err) + + // storage account specified + bslCfg = map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + BSLConfigResourceGroup: "resourcegroup", + BSLConfigStorageAccount: "account", + } + creds = map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + _, err = exchangeStorageAccountAccessKey(bslCfg, creds) + require.NotNil(t, err) +} + +func Test_newStorageAccountManagemenClient(t *testing.T) { + // subscription ID isn't specified + bslCfg := map[string]string{} + creds := map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + _, err := newStorageAccountManagemenClient(bslCfg, creds) + require.NotNil(t, err) + + // subscription ID isn't specified + bslCfg = map[string]string{ + BSLConfigSubscriptionID: "subscriptionid", + } + creds = map[string]string{ + "AZURE_TENANT_ID": "tenantid", + "AZURE_CLIENT_ID": "clientid", + "AZURE_CLIENT_SECRET": "secret", + } + _, err = newStorageAccountManagemenClient(bslCfg, creds) + require.Nil(t, err) +} diff --git a/pkg/util/azure/testdata/certificate.pem b/pkg/util/azure/testdata/certificate.pem new file mode 100644 index 0000000000..4b66bfa021 --- /dev/null +++ b/pkg/util/azure/testdata/certificate.pem @@ -0,0 +1,49 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDL1hG+JYCfIPp3 +tlZ05J4pYIJ3Ckfs432bE3rYuWlR2w9KqdjWkKxuAxpjJ+T+uoqVaT3BFMfi4ZRY +OCI69s4+lP3DwR8uBCp9xyVkF8thXfS3iui0liGDviVBoBJJWvjDFU8a/Hseg+Qf +oxAb6tx0kEc7V3ozBLWoIDJjfwJ3NdsLZGVtAC34qCWeEIvS97CDA4g3Kc6hYJIr +Aa7pxHzo/Nd0U3e7z+DlBcJV7dY6TZUyjBVTpzppWe+XQEOfKsjkDNykHEC1C1bC +lG0u7unS7QOBMd6bOGkeL+Bc+n22slTzs5amsbDLNuobSaUsFt9vgD5jRD6FwhpX +wj/Ek0F7AgMBAAECggEAblU3UWdXUcs2CCqIbcl52wfEVs8X05/n01MeAcWKvqYG +hvGcz7eLvhir5dQoXcF3VhybMrIe6C4WcBIiZSxGwxU+rwEP8YaLwX1UPfOrQM7s +sZTdFTLWfUslO3p7q300fdRA92iG9COMDZvkElh0cBvQksxs9sSr149l9vk+ymtC +uBhZtHG6Ki0BIMBNC9jGUqDuOatXl/dkK4tNjXrNJT7tVwzPaqnNALIWl6B+k9oQ +m1oNhSH2rvs9tw2ITXfIoIk9KdOMjQVUD43wKOaz0hNZhUsb1OFuls7UtRzaFcZH +rMd/M8DtA104QTTlHK+XS7r+nqdv7+ZyB+suTdM+oQKBgQDxCrJZU3hJ0eJ4VYhK +xGDfVGNpYxNkQ4CDB9fwRNbFr/Ck3kgzfE9QxTx1pJOolVmfuFmk9B86in4UNy91 +KdaqT79AU5RdOBXNN6tuMbLC0AVqe8sZq+1vWVVwbCstffxEMmyW1Ju/FLYPl2Zp +e5P96dBh5B3mXrQtpDJ0RkxxaQKBgQDYfE6tQQnQSs2ewD6ae8Mu6j8ueDlVoZ37 +vze1QdBasR26xu2H8XBt3u41zc524BwQsB1GE1tnC8ZylrqwVEayK4FesSQRCO6o +yK8QSdb06I5J4TaN+TppCDPLzstOh0Dmxp+iFUGoErb7AEOLAJ/VebhF9kBZObL/ +HYy4Es+bQwKBgHW/4vYuB3IQXNCp/+V+X1BZ+iJOaves3gekekF+b2itFSKFD8JO +9LQhVfKmTheptdmHhgtF0keXxhV8C+vxX1Ndl7EF41FSh5vzmQRAtPHkCvFEviex +TFD70/gSb1lO1UA/Xbqk69yBcprVPAtFejss0EYx2MVj+CLftmIEwW0ZAoGBAIMG +EVQ45eikLXjkn78+Iq7VZbIJX6IdNBH29I+GqsUJJ5Yw6fh6P3KwF3qG+mvmTfYn +sUAFXS+r58rYwVsRVsxlGmKmUc7hmhibhaEVH72QtvWuEiexbRG+viKfIVuA7t39 +3wXpWZiQ4yBdU4Pgt9wrVEU7ukyGaHiReOa7s90jAoGAJc0K7smn98YutQQ+g2ur +ybfnsl0YdsksaP2S2zvZUmNevKPrgnaIDDabOlhYYga+AK1G3FQ7/nefUgiIg1Nd +kr+T6Q4osS3xHB6Az9p/jaF4R2KaWN2nNVCn7ecsmPxDdM7k1vLxaT26vwO9OP5f +YU/5CeIzrfA5nQyPZkOXZBk= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDazCCAlOgAwIBAgIUF2VIP4+AnEtb52KTCHbo4+fESfswDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0xOTEwMzAyMjQ2MjBaFw0yMjA4 +MTkyMjQ2MjBaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDL1hG+JYCfIPp3tlZ05J4pYIJ3Ckfs432bE3rYuWlR +2w9KqdjWkKxuAxpjJ+T+uoqVaT3BFMfi4ZRYOCI69s4+lP3DwR8uBCp9xyVkF8th +XfS3iui0liGDviVBoBJJWvjDFU8a/Hseg+QfoxAb6tx0kEc7V3ozBLWoIDJjfwJ3 +NdsLZGVtAC34qCWeEIvS97CDA4g3Kc6hYJIrAa7pxHzo/Nd0U3e7z+DlBcJV7dY6 +TZUyjBVTpzppWe+XQEOfKsjkDNykHEC1C1bClG0u7unS7QOBMd6bOGkeL+Bc+n22 +slTzs5amsbDLNuobSaUsFt9vgD5jRD6FwhpXwj/Ek0F7AgMBAAGjUzBRMB0GA1Ud +DgQWBBT6Mf9uXFB67bY2PeW3GCTKfkO7vDAfBgNVHSMEGDAWgBT6Mf9uXFB67bY2 +PeW3GCTKfkO7vDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCZ +1+kTISX85v9/ag7glavaPFUYsOSOOofl8gSzov7L01YL+srq7tXdvZmWrjQ/dnOY +h18rp9rb24vwIYxNioNG/M2cW1jBJwEGsDPOwdPV1VPcRmmUJW9kY130gRHBCd/N +qB7dIkcQnpNsxPIIWI+sRQp73U0ijhOByDnCNHLHon6vbfFTwkO1XggmV5BdZ3uQ +JNJyckILyNzlhmf6zhonMp4lVzkgxWsAm2vgdawd6dmBa+7Avb2QK9s+IdUSutFh +DgW2L12Obgh12Y4sf1iKQXA0RbZ2k+XQIz8EKZa7vJQY0ciYXSgB/BV3a96xX3cx +LIPL8Vam8Ytkopi3gsGA +-----END CERTIFICATE----- \ No newline at end of file diff --git a/pkg/util/azure/util.go b/pkg/util/azure/util.go new file mode 100644 index 0000000000..0b4cfa067a --- /dev/null +++ b/pkg/util/azure/util.go @@ -0,0 +1,159 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "crypto/tls" + "crypto/x509" + "encoding/base64" + "fmt" + "net" + "net/http" + "os" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/joho/godotenv" + "github.com/pkg/errors" +) + +const ( + // the keys of Azure variables in credential + CredentialKeySubscriptionID = "AZURE_SUBSCRIPTION_ID" // #nosec + CredentialKeyResourceGroup = "AZURE_RESOURCE_GROUP" // #nosec + CredentialKeyCloudName = "AZURE_CLOUD_NAME" // #nosec + CredentialKeyStorageAccountAccessKey = "AZURE_STORAGE_KEY" // #nosec + CredentialKeyAdditionallyAllowedTenants = "AZURE_ADDITIONALLY_ALLOWED_TENANTS" // #nosec + CredentialKeyTenantID = "AZURE_TENANT_ID" // #nosec + CredentialKeyClientID = "AZURE_CLIENT_ID" // #nosec + CredentialKeyClientSecret = "AZURE_CLIENT_SECRET" // #nosec + CredentialKeyClientCertificatePath = "AZURE_CLIENT_CERTIFICATE_PATH" // #nosec + CredentialKeyClientCertificatePassword = "AZURE_CLIENT_CERTIFICATE_PASSWORD" // #nosec + CredentialKeySendCertChain = "AZURE_CLIENT_SEND_CERTIFICATE_CHAIN" // #nosec + CredentialKeyUsername = "AZURE_USERNAME" // #nosec + CredentialKeyPassword = "AZURE_PASSWORD" // #nosec + + credentialFile = "credentialsFile" +) + +// LoadCredentials gets the credential file from config and loads it into a map +func LoadCredentials(config map[string]string) (map[string]string, error) { + // the default credential file + credFile := os.Getenv("AZURE_CREDENTIALS_FILE") + + // use the credential file specified in the BSL spec if provided + if config != nil && config[credentialFile] != "" { + credFile = config[credentialFile] + } + + // put the credential file content into a map + creds, err := godotenv.Read(credFile) + if err != nil { + return nil, errors.Wrapf(err, "failed to read credentials from file %s", credFile) + } + return creds, nil +} + +// GetClientOptions returns the client options based on the BSL/VSL config and credentials +func GetClientOptions(locationCfg, creds map[string]string) (policy.ClientOptions, error) { + options := policy.ClientOptions{} + + cloudCfg, err := getCloudConfiguration(locationCfg, creds) + if err != nil { + return options, err + } + options.Cloud = cloudCfg + + if locationCfg["caCert"] != "" { + certPool, _ := x509.SystemCertPool() + if certPool == nil { + certPool = x509.NewCertPool() + } + var caCert []byte + // As this function is used in both repository and plugin, the caCert isn't encoded + // when passing to the plugin while is encoded when works with repository, use one + // config item to distinguish these two cases + if locationCfg["caCertEncoded"] != "" { + caCert, err = base64.StdEncoding.DecodeString(locationCfg["caCert"]) + if err != nil { + return options, err + } + } else { + caCert = []byte(locationCfg["caCert"]) + } + + certPool.AppendCertsFromPEM(caCert) + + // https://github.com/Azure/azure-sdk-for-go/blob/sdk/azcore/v1.6.1/sdk/azcore/runtime/transport_default_http_client.go#L19 + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: certPool, + }, + } + options.Transport = &http.Client{ + Transport: transport, + } + } + + return options, nil +} + +// getCloudConfiguration based on the BSL/VSL config and credentials +func getCloudConfiguration(locationCfg, creds map[string]string) (cloud.Configuration, error) { + name := creds[CredentialKeyCloudName] + activeDirectoryAuthorityURI := locationCfg[BSLConfigActiveDirectoryAuthorityURI] + + var cfg cloud.Configuration + switch strings.ToUpper(name) { + case "", "AZURECLOUD", "AZUREPUBLICCLOUD": + cfg = cloud.AzurePublic + case "AZURECHINACLOUD": + cfg = cloud.AzureChina + case "AZUREUSGOVERNMENT", "AZUREUSGOVERNMENTCLOUD": + cfg = cloud.AzureGovernment + default: + return cloud.Configuration{}, errors.New(fmt.Sprintf("unknown cloud: %s", name)) + } + if activeDirectoryAuthorityURI != "" { + cfg.ActiveDirectoryAuthorityHost = activeDirectoryAuthorityURI + } + return cfg, nil +} + +// GetFromLocationConfigOrCredential returns the value of the specified key from BSL/VSL config or credentials +// as some common configuration items can be set in BSL/VSL config or credential file(such as the subscription ID or resource group) +// Reading from BSL/VSL config takes first. +func GetFromLocationConfigOrCredential(cfg, creds map[string]string, cfgKey, credKey string) string { + value := cfg[cfgKey] + if value != "" { + return value + } + return creds[credKey] +} diff --git a/pkg/util/azure/util_test.go b/pkg/util/azure/util_test.go new file mode 100644 index 0000000000..e5a92f78cf --- /dev/null +++ b/pkg/util/azure/util_test.go @@ -0,0 +1,211 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "os" + "path/filepath" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadCredentials(t *testing.T) { + // no credential file + _, err := LoadCredentials(nil) + require.NotNil(t, err) + + // specified credential file in the config + name := filepath.Join(os.TempDir(), "credential") + file, err := os.Create(name) + require.Nil(t, err) + defer file.Close() + defer os.Remove(name) + _, err = file.WriteString("key: value") + require.Nil(t, err) + + config := map[string]string{ + "credentialsFile": name, + } + credentials, err := LoadCredentials(config) + require.Nil(t, err) + assert.Equal(t, "value", credentials["key"]) + + // use the default path defined via env variable + config = nil + os.Setenv("AZURE_CREDENTIALS_FILE", name) + credentials, err = LoadCredentials(config) + require.Nil(t, err) + assert.Equal(t, "value", credentials["key"]) +} + +func TestGetClientOptions(t *testing.T) { + // invalid cloud name + bslCfg := map[string]string{} + creds := map[string]string{ + CredentialKeyCloudName: "invalid", + } + _, err := GetClientOptions(bslCfg, creds) + require.NotNil(t, err) + + // specify caCert + bslCfg = map[string]string{ + CredentialKeyCloudName: "", + "caCert": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZTakNDQXpLZ0F3SUJBZ0lVWmcxbzRpWld2bVh5ekJrQ0J6SGdiODZGemtFd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1VqRUxNQWtHQTFVRUJoTUNRMDR4RERBS0JnTlZCQWdNQTFCRlN6RVJNQThHQTFVRUJ3d0lRbVZwSUVwcApibWN4RHpBTkJnTlZCQW9NQmxaTmQyRnlaVEVSTUE4R0ExVUVBd3dJU0dGeVltOXlRMEV3SGhjTk1qTXdPVEEyCk1ESXpOakUyV2hjTk1qUXdPVEExTURJek5qRTJXakJYTVFzd0NRWURWUVFHRXdKRFRqRU1NQW9HQTFVRUNBd0QKVUVWTE1SRXdEd1lEVlFRSERBaENaV2tnU21sdVp6RVBNQTBHQTFVRUNnd0dWazEzWVhKbE1SWXdGQVlEVlFRRApEQTFJWVhKaWIzSk5ZVzVoWjJWeU1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBCnIrK1FHaHYvUnBDUTFIcncrMnYyQWNoaGhUVTVQL3hCd2RIWkZHWWJzMmxGbGtiL3oycEs2Y05ycFZmNUtmdjIKVUNpZEovMjFhZHc2SWNGZWxkSnFudU4rSlJaWXh5S0w0bzdRRGNVSk1sUTZJZk5kbEI0NUNwcGFBZVA4blVVTgo0YUwyV244b094L1pROTd2YmRXeERIR1FqZGR4N3p0Q09PaVZ0SEk4NS9Ka3kydTJnNmVhMklndmh1ZEVPZ3JtCjJzNU8zZlVtdHhSTEhwNnpDbURYZGZFUWg4ZFpndCs1d0RlazRWR2t4Zk81VG1tUHJ0LzBPTnVGYjJMWGVWV0sKeXkzVDFFTGNOSWhPSzQ1amhEejNnb2JhQzAwK0JTdzJMejVocXRxdUQ2RGxmME53TWtBQkt6d1dMZkROOXNrRApVazVYTmZNa2c0L3JhblRIWHlCNUNKSlk1akhORUtBQWlnM2NFSFNvejVjeGJqTE14VDhoMEk3MitEWldmUzFTCjU3Rm1SN2ZTNXk0QUcrU3Z1U3kvbktCUktJS2dQZ0t2Rjl6NktuL2ZPMTNsbk5LbHQwWU5mWFBFV2hZQytmMUoKTWpTOWc5eHBpYkZhM2Q0aHpOeWZhMWJHcUxtbkUwNVNpbWZNMVI1Z21Tamw1Z1FKQlltTHA3dWRLdjFDSUNRSwptQng2WG4vcnJEZHFiMndCRUNRSjBMbUo2SW5SaFZtT0s0WUdFeFRqZ1FRMldSWHYxMnhVK05GYWlZS3cxZkp0CkdaemFQeENxaG5JZXM5cGNPY0FjdmFHVngwSjlFYnRod0ltekdoTjBBREdCOVZaL1dFdHYvN0gwQ2xjOVlyT0gKNnRMb212b2pjQUZnN0xFbXZxeFFEOFFSTzlZZVdTTkgvV1REY1hVb3R5a0NBd0VBQWFNVE1CRXdEd1lEVlIwUgpCQWd3Qm9jRUNycFFxakFOQmdrcWhraUc5dzBCQVFzRkFBT0NBZ0VBZnRVdmR1UFMvajhSaWx4ME5aelhSeEY3Ck9HZW9qU2JaQ1ZvamNPdnRNTVlMaFkxdDE2Y2laY1VWMGF4Z2FUWkkvak9WMGJPTEl3dmEvcVB2Z1RmSWZqL0gKVzhiMlNTRVdIUzZPSFFaR1BYNy9zVFVwQzB6QVcva2haN1FWR1BoWEcyK0V1NjFaNE95ejZ5dTRPdi9MYjlMUQpmMU9zTXhwandkbmhxazFKaERxUkpZbGIvZ05TRGZnVlN0YmhHVzVhb3paUlBBMUtqVXVaT3QzR2xQR09Wd3ZLCnpUcFFMdGVTUHNibTJMcUl2ZEg4dlgzK1kwcHIzdEdtdnExbWtIWUhYQTlBZWtYRkVsRHc4dGtZVHdLaEFqblUKZEFjWTFkTis5ODNiMDI0L0JQUXZKQlRTVjd4blEyUnlrUmMrVGxIL3B5RlM1cEtVbUF0aU9qTElxL2ZEMmJVagorTzlxT1hjK0c1b0xEaXlXWDRXSG9XdkZZdTdva1gwT1dGcHFETXFOcHlLUkRzQ1FENXViMEVQaVlVS0hnWEhiCnV3UXVtK0pRRUREdzRXL1kzZktnMW9TWW1XOHJndFNPZmtRQlQ0UnlaTUg2SzN6cFp5dVVsbmJUV0NWeEcyYVoKWVo0T2JpbUFGbVlveGRYdktWdFU0YUdlTjRoaXBvb2dzaXVXKzZYQ3Bqa2pWZlZuUEY4elZVNlZ3anRQVkkzKwpxdWxRNWJLS3lKYng3bk9NNXFob2svSmk2N1pyZDhob3ZwclhhRUdvakNDTVI3MllPWGVuMlB3bVlZZWNkQ2pyCnErSDdHNUV3ZXBoRWxrN3RWRWY4RVV4OEc1Mk9SVEtZMkF1dlRGVlliUC8yaTROS1FlMWdEWWZrWnNzUk1MajEKK0JCQVVJcnFVMnRuUHhwZW4vMD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=", + } + creds = map[string]string{} + options, err := GetClientOptions(bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, options.Cloud, cloud.AzurePublic) + assert.NotNil(t, options.Transport) + + // doesn't specify caCert + bslCfg = map[string]string{ + CredentialKeyCloudName: "", + } + creds = map[string]string{} + options, err = GetClientOptions(bslCfg, creds) + require.Nil(t, err) + assert.Equal(t, options.Cloud, cloud.AzurePublic) + assert.Nil(t, options.Transport) +} + +func Test_getCloudConfiguration(t *testing.T) { + publicCloudWithADURI := cloud.AzurePublic + publicCloudWithADURI.ActiveDirectoryAuthorityHost = "https://example.com" + cases := []struct { + name string + bslCfg map[string]string + creds map[string]string + err bool + expected cloud.Configuration + }{ + { + name: "invalid cloud name", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "invalid", + }, + err: true, + }, + { + name: "null cloud name", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "", + }, + err: false, + expected: cloud.AzurePublic, + }, + { + name: "azure public cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "AZURECLOUD", + }, + err: false, + expected: cloud.AzurePublic, + }, + { + name: "azure public cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "AZUREPUBLICCLOUD", + }, + err: false, + expected: cloud.AzurePublic, + }, + { + name: "azure public cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "azurecloud", + }, + err: false, + expected: cloud.AzurePublic, + }, + { + name: "azure China cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "AZURECHINACLOUD", + }, + err: false, + expected: cloud.AzureChina, + }, + { + name: "azure US government cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "AZUREUSGOVERNMENT", + }, + err: false, + expected: cloud.AzureGovernment, + }, + { + name: "azure US government cloud", + bslCfg: map[string]string{}, + creds: map[string]string{ + CredentialKeyCloudName: "AZUREUSGOVERNMENTCLOUD", + }, + err: false, + expected: cloud.AzureGovernment, + }, + { + name: "AD authority URI provided", + bslCfg: map[string]string{ + BSLConfigActiveDirectoryAuthorityURI: "https://example.com", + }, + creds: map[string]string{ + CredentialKeyCloudName: "", + }, + err: false, + expected: publicCloudWithADURI, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cfg, err := getCloudConfiguration(c.bslCfg, c.creds) + require.Equal(t, c.err, err != nil) + if !c.err { + assert.Equal(t, c.expected, cfg) + } + }) + } +} + +func TestGetFromLocationConfigOrCredential(t *testing.T) { + // from cfg + cfg := map[string]string{ + "cfgkey": "value", + } + creds := map[string]string{} + cfgKey, credKey := "cfgkey", "credkey" + str := GetFromLocationConfigOrCredential(cfg, creds, cfgKey, credKey) + assert.Equal(t, "value", str) + + // from cred + cfg = map[string]string{} + creds = map[string]string{ + "credkey": "value", + } + str = GetFromLocationConfigOrCredential(cfg, creds, cfgKey, credKey) + assert.Equal(t, "value", str) +} diff --git a/pkg/util/kube/pvc_pv.go b/pkg/util/kube/pvc_pv.go index 2af818d909..1a393b3ab2 100644 --- a/pkg/util/kube/pvc_pv.go +++ b/pkg/util/kube/pvc_pv.go @@ -316,3 +316,23 @@ func WaitPVBound(ctx context.Context, pvGetter corev1client.CoreV1Interface, pvN func IsPVCBound(pvc *corev1api.PersistentVolumeClaim) bool { return pvc.Spec.VolumeName != "" } + +// MakePodPVCAttachment returns the volume mounts and devices for a pod needed to attach a PVC +func MakePodPVCAttachment(volumeName string, volumeMode *corev1api.PersistentVolumeMode) ([]corev1api.VolumeMount, []corev1api.VolumeDevice) { + var volumeMounts []corev1api.VolumeMount = nil + var volumeDevices []corev1api.VolumeDevice = nil + + if volumeMode != nil && *volumeMode == corev1api.PersistentVolumeBlock { + volumeDevices = []corev1api.VolumeDevice{{ + Name: volumeName, + DevicePath: "/" + volumeName, + }} + } else { + volumeMounts = []corev1api.VolumeMount{{ + Name: volumeName, + MountPath: "/" + volumeName, + }} + } + + return volumeMounts, volumeDevices +} diff --git a/pkg/util/kube/utils.go b/pkg/util/kube/utils.go index 3d8d4e3ef1..e1ed48dba2 100644 --- a/pkg/util/kube/utils.go +++ b/pkg/util/kube/utils.go @@ -35,6 +35,7 @@ import ( corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) @@ -50,6 +51,8 @@ const ( KubeAnnSelectedNode = "volume.kubernetes.io/selected-node" ) +var ErrorPodVolumeIsNotPVC = errors.New("pod volume is not a PVC") + // NamespaceAndName returns a string in the format / func NamespaceAndName(objMeta metav1.Object) string { if objMeta.GetNamespace() == "" { @@ -122,6 +125,57 @@ func EnsureNamespaceExistsAndIsReady(namespace *corev1api.Namespace, client core // where the specified volume lives. // For volumes with a CSIVolumeSource, append "/mount" to the directory name. func GetVolumeDirectory(ctx context.Context, log logrus.FieldLogger, pod *corev1api.Pod, volumeName string, cli client.Client) (string, error) { + pvc, pv, volume, err := GetPodPVCVolume(ctx, log, pod, volumeName, cli) + if err != nil { + // This case implies the administrator created the PV and attached it directly, without PVC. + // Note that only one VolumeSource can be populated per Volume on a pod + if err == ErrorPodVolumeIsNotPVC { + if volume.VolumeSource.CSI != nil { + return volume.Name + "/mount", nil + } + return volume.Name, nil + } + return "", errors.WithStack(err) + } + + // Most common case is that we have a PVC VolumeSource, and we need to check the PV it points to for a CSI source. + // PV's been created with a CSI source. + isProvisionedByCSI, err := isProvisionedByCSI(log, pv, cli) + if err != nil { + return "", errors.WithStack(err) + } + if isProvisionedByCSI { + if pv.Spec.VolumeMode != nil && *pv.Spec.VolumeMode == corev1api.PersistentVolumeBlock { + return pvc.Spec.VolumeName, nil + } + return pvc.Spec.VolumeName + "/mount", nil + } + + return pvc.Spec.VolumeName, nil +} + +// GetVolumeMode gets the uploader.PersistentVolumeMode of the volume. +func GetVolumeMode(ctx context.Context, log logrus.FieldLogger, pod *corev1api.Pod, volumeName string, cli client.Client) ( + uploader.PersistentVolumeMode, error) { + _, pv, _, err := GetPodPVCVolume(ctx, log, pod, volumeName, cli) + + if err != nil { + if err == ErrorPodVolumeIsNotPVC { + return uploader.PersistentVolumeFilesystem, nil + } + return "", errors.WithStack(err) + } + + if pv.Spec.VolumeMode != nil && *pv.Spec.VolumeMode == corev1api.PersistentVolumeBlock { + return uploader.PersistentVolumeBlock, nil + } + return uploader.PersistentVolumeFilesystem, nil +} + +// GetPodPVCVolume gets the PVC, PV and volume for a pod volume name. +// Returns pod volume in case of ErrorPodVolumeIsNotPVC error +func GetPodPVCVolume(ctx context.Context, log logrus.FieldLogger, pod *corev1api.Pod, volumeName string, cli client.Client) ( + *corev1api.PersistentVolumeClaim, *corev1api.PersistentVolume, *corev1api.Volume, error) { var volume *corev1api.Volume for i := range pod.Spec.Volumes { @@ -132,41 +186,26 @@ func GetVolumeDirectory(ctx context.Context, log logrus.FieldLogger, pod *corev1 } if volume == nil { - return "", errors.New("volume not found in pod") + return nil, nil, nil, errors.New("volume not found in pod") } - // This case implies the administrator created the PV and attached it directly, without PVC. - // Note that only one VolumeSource can be populated per Volume on a pod if volume.VolumeSource.PersistentVolumeClaim == nil { - if volume.VolumeSource.CSI != nil { - return volume.Name + "/mount", nil - } - return volume.Name, nil + return nil, nil, volume, ErrorPodVolumeIsNotPVC // There is a pod volume but it is not a PVC } - // Most common case is that we have a PVC VolumeSource, and we need to check the PV it points to for a CSI source. pvc := &corev1api.PersistentVolumeClaim{} err := cli.Get(ctx, client.ObjectKey{Namespace: pod.Namespace, Name: volume.VolumeSource.PersistentVolumeClaim.ClaimName}, pvc) if err != nil { - return "", errors.WithStack(err) + return nil, nil, nil, errors.WithStack(err) } pv := &corev1api.PersistentVolume{} err = cli.Get(ctx, client.ObjectKey{Name: pvc.Spec.VolumeName}, pv) if err != nil { - return "", errors.WithStack(err) + return nil, nil, nil, errors.WithStack(err) } - // PV's been created with a CSI source. - isProvisionedByCSI, err := isProvisionedByCSI(log, pv, cli) - if err != nil { - return "", errors.WithStack(err) - } - if isProvisionedByCSI { - return pvc.Spec.VolumeName + "/mount", nil - } - - return pvc.Spec.VolumeName, nil + return pvc, pv, volume, nil } // isProvisionedByCSI function checks whether this is a CSI PV by annotation. diff --git a/pkg/util/kube/utils_test.go b/pkg/util/kube/utils_test.go index 6edf084351..31110dc8c0 100644 --- a/pkg/util/kube/utils_test.go +++ b/pkg/util/kube/utils_test.go @@ -38,6 +38,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/builder" velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/uploader" ) func TestNamespaceAndName(t *testing.T) { @@ -164,6 +165,13 @@ func TestGetVolumeDirectorySuccess(t *testing.T) { pv: builder.ForPersistentVolume("a-pv").CSI("csi.test.com", "provider-volume-id").Result(), want: "a-pv/mount", }, + { + name: "Block CSI volume with a PVC/PV does not append '/mount' to the volume name", + pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").PersistentVolumeClaimSource("my-pvc").Result()).Result(), + pvc: builder.ForPersistentVolumeClaim("ns-1", "my-pvc").VolumeName("a-pv").Result(), + pv: builder.ForPersistentVolume("a-pv").CSI("csi.test.com", "provider-volume-id").VolumeMode(corev1.PersistentVolumeBlock).Result(), + want: "a-pv", + }, { name: "CSI volume mounted without a PVC appends '/mount' to the volume name", pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").CSISource("csi.test.com").Result()).Result(), @@ -211,6 +219,54 @@ func TestGetVolumeDirectorySuccess(t *testing.T) { } } +// TestGetVolumeModeSuccess tests the GetVolumeMode function +func TestGetVolumeModeSuccess(t *testing.T) { + tests := []struct { + name string + pod *corev1.Pod + pvc *corev1.PersistentVolumeClaim + pv *corev1.PersistentVolume + want uploader.PersistentVolumeMode + }{ + { + name: "Filesystem PVC volume", + pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").PersistentVolumeClaimSource("my-pvc").Result()).Result(), + pvc: builder.ForPersistentVolumeClaim("ns-1", "my-pvc").VolumeName("a-pv").Result(), + pv: builder.ForPersistentVolume("a-pv").VolumeMode(corev1.PersistentVolumeFilesystem).Result(), + want: uploader.PersistentVolumeFilesystem, + }, + { + name: "Block PVC volume", + pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").PersistentVolumeClaimSource("my-pvc").Result()).Result(), + pvc: builder.ForPersistentVolumeClaim("ns-1", "my-pvc").VolumeName("a-pv").Result(), + pv: builder.ForPersistentVolume("a-pv").VolumeMode(corev1.PersistentVolumeBlock).Result(), + want: uploader.PersistentVolumeBlock, + }, + { + name: "Pod volume without a PVC", + pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").Result()).Result(), + want: uploader.PersistentVolumeFilesystem, + }, + } + + for _, tc := range tests { + clientBuilder := fake.NewClientBuilder() + + if tc.pvc != nil { + clientBuilder = clientBuilder.WithObjects(tc.pvc) + } + if tc.pv != nil { + clientBuilder = clientBuilder.WithObjects(tc.pv) + } + + // Function under test + mode, err := GetVolumeMode(context.Background(), logrus.StandardLogger(), tc.pod, tc.pod.Spec.Volumes[0].Name, clientBuilder.Build()) + + require.NoError(t, err) + assert.Equal(t, tc.want, mode) + } +} + func TestIsV1Beta1CRDReady(t *testing.T) { tests := []struct { name string diff --git a/site/content/contributors/06-anshul-ahuja.md b/site/content/contributors/06-anshul-ahuja.md new file mode 100644 index 0000000000..39cf1f2fea --- /dev/null +++ b/site/content/contributors/06-anshul-ahuja.md @@ -0,0 +1,7 @@ +--- +first_name: Anshul +last_name: Ahuja +image: /img/contributors/anshul-ahuja.jpeg +github_handle: anshulahuja98 +--- +Engineer diff --git a/site/content/docs/main/csi-snapshot-data-movement.md b/site/content/docs/main/csi-snapshot-data-movement.md index 667be0a48c..7a68821ab7 100644 --- a/site/content/docs/main/csi-snapshot-data-movement.md +++ b/site/content/docs/main/csi-snapshot-data-movement.md @@ -41,7 +41,7 @@ velero install --use-node-agent ### Configure Node Agent DaemonSet spec After installation, some PaaS/CaaS platforms based on Kubernetes also require modifications the node-agent DaemonSet spec. -The steps in this section are only needed if you are installing on RancherOS, OpenShift, VMware Tanzu Kubernetes Grid +The steps in this section are only needed if you are installing on RancherOS, Nutanix, OpenShift, VMware Tanzu Kubernetes Grid Integrated Edition (formerly VMware Enterprise PKS), or Microsoft Azure. @@ -63,6 +63,22 @@ hostPath: path: /opt/rke/var/lib/kubelet/pods ``` +**Nutanix** + +Update the host path for volumes in the node-agent DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to +`/var/nutanix/var/lib/kubelet`. + +```yaml +hostPath: + path: /var/lib/kubelet/pods +``` + +to + +```yaml +hostPath: + path: /var/nutanix/var/lib/kubelet +``` **OpenShift** @@ -75,24 +91,10 @@ To mount the correct hostpath to pods volumes, run the node-agent pod in `privil oc adm policy add-scc-to-user privileged -z velero -n velero ``` -2. Modify the DaemonSet yaml to request a privileged mode: - - ```diff - @@ -67,3 +67,5 @@ spec: - value: /credentials/cloud - - name: VELERO_SCRATCH_DIR - value: /scratch - + securityContext: - + privileged: true +2. Install Velero with the '--privileged-node-agent' option to request a privileged mode: + ``` - - or - - ```shell - oc patch ds/node-agent \ - --namespace velero \ - --type json \ - -p '[{"op":"add","path":"/spec/template/spec/containers/0/securityContext","value": { "privileged": true}}]' + velero install --use-node-agent --privileged-node-agent ``` If node-agent is not running in a privileged mode, it will not be able to access snapshot volumes within the mounted diff --git a/site/content/docs/main/customize-installation.md b/site/content/docs/main/customize-installation.md index 0d8621685d..a4e2299c8a 100644 --- a/site/content/docs/main/customize-installation.md +++ b/site/content/docs/main/customize-installation.md @@ -23,6 +23,14 @@ By default, `velero install` does not install Velero's [File System Backup][3]. If you've already run `velero install` without the `--use-node-agent` flag, you can run the same command again, including the `--use-node-agent` flag, to add the file system backup to your existing install. +## CSI Snapshot Data Movement + +Velero node-agent is required by CSI snapshot data movement when Velero built-in data mover is used. By default, `velero install` does not install Velero's node-agent. To enable it, specify the `--use-node-agent` flag. + +For some use cases, Velero node-agent requires to run under privileged mode. For example, when backing up block volumes, it is required to allow the node-agent to access the block device. To enable it set velero install flags `--privileged-node-agent`. + +If you've already run `velero install` without the `--use-node-agent` or `--privileged-node-agent` flag, you can run the same command again, including the `--use-node-agent` or `--privileged-node-agent` flag, to add CSI snapshot data movement to your existing install. + ## Default Pod Volume backup to file system backup By default, `velero install` does not enable the use of File System Backup (FSB) to take backups of all pod volumes. You must apply an [annotation](file-system-backup.md/#using-opt-in-pod-volume-backup) to every pod which contains volumes for Velero to use FSB for the backup. diff --git a/site/content/docs/main/file-system-backup.md b/site/content/docs/main/file-system-backup.md index 108daede44..9fe6e97aee 100644 --- a/site/content/docs/main/file-system-backup.md +++ b/site/content/docs/main/file-system-backup.md @@ -77,7 +77,7 @@ backup which created the backup repository, then Velero will not be able to conn ### Configure Node Agent DaemonSet spec After installation, some PaaS/CaaS platforms based on Kubernetes also require modifications the node-agent DaemonSet spec. -The steps in this section are only needed if you are installing on RancherOS, OpenShift, VMware Tanzu Kubernetes Grid +The steps in this section are only needed if you are installing on RancherOS, Nutanix, OpenShift, VMware Tanzu Kubernetes Grid Integrated Edition (formerly VMware Enterprise PKS), or Microsoft Azure. @@ -99,6 +99,23 @@ hostPath: path: /opt/rke/var/lib/kubelet/pods ``` +**Nutanix** + + +Update the host path for volumes in the node-agent DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to +`/var/nutanix/var/lib/kubelet`. + +```yaml +hostPath: + path: /var/lib/kubelet/pods +``` + +to + +```yaml +hostPath: + path: /var/nutanix/var/lib/kubelet +``` **OpenShift** @@ -111,24 +128,10 @@ To mount the correct hostpath to pods volumes, run the node-agent pod in `privil oc adm policy add-scc-to-user privileged -z velero -n velero ``` -2. Modify the DaemonSet yaml to request a privileged mode: - - ```diff - @@ -67,3 +67,5 @@ spec: - value: /credentials/cloud - - name: VELERO_SCRATCH_DIR - value: /scratch - + securityContext: - + privileged: true +2. Install Velero with the '--privileged-node-agent' option to request a privileged mode: + ``` - - or - - ```shell - oc patch ds/node-agent \ - --namespace velero \ - --type json \ - -p '[{"op":"add","path":"/spec/template/spec/containers/0/securityContext","value": { "privileged": true}}]' + velero install --use-node-agent --privileged-node-agent ``` diff --git a/site/content/docs/main/resource-filtering.md b/site/content/docs/main/resource-filtering.md index 3edf69b12b..621560089e 100644 --- a/site/content/docs/main/resource-filtering.md +++ b/site/content/docs/main/resource-filtering.md @@ -101,6 +101,24 @@ Includes cluster-scoped resources. Cannot work with `--include-cluster-scoped-re For more information read the [Kubernetes label selector documentation](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors) +### --or-selector + +To include the resources that match at least one of the label selectors from the list. Separate the selectors with ` or `. The ` or ` is used as a separator to split label selectors, and it is not an operator. + +This option cannot be used together with `--selector`. + +* Include resources matching any one of the label selector, `foo=bar` or `baz=qux` + + ```bash + velero backup create backup1 --or-selector "foo=bar or baz=qux" + ``` + +* Include resources that are labeled `environment=production` or `env=prod` or `env=production` or `environment=prod`. + + ```bash + velero restore create restore-prod --from-backup=prod-backup --or-selector "env in (prod,production) or environment in (prod, production)" + ``` + ### --include-cluster-scoped-resources Kubernetes cluster-scoped resources to include in the backup, formatted as resource.group, such as `storageclasses.storage.k8s.io`(use '*' for all resources). Cannot work with `--include-resources`, `--exclude-resources` and `--include-cluster-resources`. This parameter only works for backup, not for restore. diff --git a/site/content/docs/main/restore-reference.md b/site/content/docs/main/restore-reference.md index 8d646407c6..d50a12a43f 100644 --- a/site/content/docs/main/restore-reference.md +++ b/site/content/docs/main/restore-reference.md @@ -269,7 +269,9 @@ are `none` (default) and `update`. If you choose to update existing resources du You can also configure the existing resource policy in a [Restore](api-types/restore.md) object. -**NOTE:** Update of a resource only applies to the Kubernetes resource data such as its spec. It may not work as expected for certain resource types such as PVCs and Pods. In case of PVCs for example, data in the PV is not restored or overwritten in any way. +**NOTE:** +* Update of a resource only applies to the Kubernetes resource data such as its spec. It may not work as expected for certain resource types such as PVCs and Pods. In case of PVCs for example, data in the PV is not restored or overwritten in any way. +* `update` existing resource policy works in a best-effort way, which means when restore's `--existing-resource-policy` is set to `update`, Velero will try to update the resource if the resource already exists, if the update fails, Velero will fall back to the default non-destructive way in the restore, and just logs a warning without failing the restore. ## Removing a Restore object diff --git a/site/content/docs/main/velero-install.md b/site/content/docs/main/velero-install.md index 1ac1395d52..9dba90d12d 100644 --- a/site/content/docs/main/velero-install.md +++ b/site/content/docs/main/velero-install.md @@ -39,7 +39,7 @@ This section provides examples that serve as a starting point for more customize ```bash velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket mybucket --secret-file ./gcp-service-account.json -velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --provider aws --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-node-agent +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-node-agent velero install --provider azure --plugins velero/velero-plugin-for-microsoft-azure:v1.0.0 --bucket $BLOB_CONTAINER --secret-file ./credentials-velero --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] ``` diff --git a/site/content/docs/v1.10/csi.md b/site/content/docs/v1.10/csi.md index 80198c4c97..c660776035 100644 --- a/site/content/docs/v1.10/csi.md +++ b/site/content/docs/v1.10/csi.md @@ -41,7 +41,7 @@ This section documents some of the choices made during implementation of the Vel ```yaml velero.io/csi-volumesnapshot-class: "true" ``` - 1. The VolumeSnapshot objects will be removed from the cluster after the backup is uploaded to the object storage, so that the namespace that is backed up can be deleted without removing the snapshot in the storage provider if the `DeletionPolicy` is `Delete. + 1. The VolumeSnapshot objects will be removed from the cluster after the backup is uploaded to the object storage, so that the namespace that is backed up can be deleted without removing the snapshot in the storage provider if the `DeletionPolicy` is `Delete`. ## How it Works - Overview diff --git a/site/content/docs/v1.10/restore-reference.md b/site/content/docs/v1.10/restore-reference.md index 31a3e79602..19eaf539cf 100644 --- a/site/content/docs/v1.10/restore-reference.md +++ b/site/content/docs/v1.10/restore-reference.md @@ -224,6 +224,10 @@ You can change this policy for a restore by using the `--existing-resource-polic You can also configure the existing resource policy in a [Restore](api-types/restore.md) object. +**NOTE:** +* Update of a resource only applies to the Kubernetes resource data such as its spec. It may not work as expected for certain resource types such as PVCs and Pods. In case of PVCs for example, data in the PV is not restored or overwritten in any way. +* `update` existing resource policy works in a best-effort way, which means when restore's `--existing-resource-policy` is set to `update`, Velero will try to update the resource if the resource already exists, if the update fails, Velero will fall back to the default non-destructive way in the restore, and just logs a warning without failing the restore. + ## Removing a Restore object There are two ways to delete a Restore object: diff --git a/site/content/docs/v1.10/velero-install.md b/site/content/docs/v1.10/velero-install.md index 7fdd0f9361..1d503bb0ae 100644 --- a/site/content/docs/v1.10/velero-install.md +++ b/site/content/docs/v1.10/velero-install.md @@ -39,7 +39,7 @@ This section provides examples that serve as a starting point for more customize ```bash velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket mybucket --secret-file ./gcp-service-account.json -velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --provider aws --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-node-agent +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-node-agent velero install --provider azure --plugins velero/velero-plugin-for-microsoft-azure:v1.0.0 --bucket $BLOB_CONTAINER --secret-file ./credentials-velero --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] ``` diff --git a/site/content/docs/v1.11/csi.md b/site/content/docs/v1.11/csi.md index 58899efc1c..ea0ae78fee 100644 --- a/site/content/docs/v1.11/csi.md +++ b/site/content/docs/v1.11/csi.md @@ -41,7 +41,7 @@ This section documents some of the choices made during implementation of the Vel ```yaml velero.io/csi-volumesnapshot-class: "true" ``` - 1. The VolumeSnapshot objects will be removed from the cluster after the backup is uploaded to the object storage, so that the namespace that is backed up can be deleted without removing the snapshot in the storage provider if the `DeletionPolicy` is `Delete. + 1. The VolumeSnapshot objects will be removed from the cluster after the backup is uploaded to the object storage, so that the namespace that is backed up can be deleted without removing the snapshot in the storage provider if the `DeletionPolicy` is `Delete`. ## How it Works - Overview diff --git a/site/content/docs/v1.11/restore-reference.md b/site/content/docs/v1.11/restore-reference.md index dce9a67d76..e0bc65a1aa 100644 --- a/site/content/docs/v1.11/restore-reference.md +++ b/site/content/docs/v1.11/restore-reference.md @@ -267,7 +267,9 @@ You can change this policy for a restore by using the `--existing-resource-polic You can also configure the existing resource policy in a [Restore](api-types/restore.md) object. -**NOTE:** Update of a resource only applies to the Kubernetes resource data such as its spec. It may not work as expected for certain resource types such as PVCs and Pods. In case of PVCs for example, data in the PV is not restored or overwritten in any way. +**NOTE:** +* Update of a resource only applies to the Kubernetes resource data such as its spec. It may not work as expected for certain resource types such as PVCs and Pods. In case of PVCs for example, data in the PV is not restored or overwritten in any way. +* `update` existing resource policy works in a best-effort way, which means when restore's `--existing-resource-policy` is set to `update`, Velero will try to update the resource if the resource already exists, if the update fails, Velero will fall back to the default non-destructive way in the restore, and just logs a warning without failing the restore. ## Removing a Restore object diff --git a/site/content/docs/v1.11/velero-install.md b/site/content/docs/v1.11/velero-install.md index 1ac1395d52..9dba90d12d 100644 --- a/site/content/docs/v1.11/velero-install.md +++ b/site/content/docs/v1.11/velero-install.md @@ -39,7 +39,7 @@ This section provides examples that serve as a starting point for more customize ```bash velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket mybucket --secret-file ./gcp-service-account.json -velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --provider aws --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-node-agent +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-node-agent velero install --provider azure --plugins velero/velero-plugin-for-microsoft-azure:v1.0.0 --bucket $BLOB_CONTAINER --secret-file ./credentials-velero --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] ``` diff --git a/site/content/docs/v1.12/csi-snapshot-data-movement.md b/site/content/docs/v1.12/csi-snapshot-data-movement.md index 667be0a48c..1fa4026380 100644 --- a/site/content/docs/v1.12/csi-snapshot-data-movement.md +++ b/site/content/docs/v1.12/csi-snapshot-data-movement.md @@ -41,7 +41,7 @@ velero install --use-node-agent ### Configure Node Agent DaemonSet spec After installation, some PaaS/CaaS platforms based on Kubernetes also require modifications the node-agent DaemonSet spec. -The steps in this section are only needed if you are installing on RancherOS, OpenShift, VMware Tanzu Kubernetes Grid +The steps in this section are only needed if you are installing on RancherOS, Nutanix, OpenShift, VMware Tanzu Kubernetes Grid Integrated Edition (formerly VMware Enterprise PKS), or Microsoft Azure. @@ -62,7 +62,22 @@ to hostPath: path: /opt/rke/var/lib/kubelet/pods ``` +**Nutanix** +Update the host path for volumes in the node-agent DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to +`/var/nutanix/var/lib/kubelet`. + +```yaml +hostPath: + path: /var/lib/kubelet/pods +``` + +to + +```yaml +hostPath: + path: /var/nutanix/var/lib/kubelet +``` **OpenShift** diff --git a/site/content/docs/v1.12/file-system-backup.md b/site/content/docs/v1.12/file-system-backup.md index 108daede44..ab87a3befc 100644 --- a/site/content/docs/v1.12/file-system-backup.md +++ b/site/content/docs/v1.12/file-system-backup.md @@ -77,7 +77,7 @@ backup which created the backup repository, then Velero will not be able to conn ### Configure Node Agent DaemonSet spec After installation, some PaaS/CaaS platforms based on Kubernetes also require modifications the node-agent DaemonSet spec. -The steps in this section are only needed if you are installing on RancherOS, OpenShift, VMware Tanzu Kubernetes Grid +The steps in this section are only needed if you are installing on RancherOS, Nutanix, OpenShift, VMware Tanzu Kubernetes Grid Integrated Edition (formerly VMware Enterprise PKS), or Microsoft Azure. @@ -99,6 +99,23 @@ hostPath: path: /opt/rke/var/lib/kubelet/pods ``` +**Nutanix** + + +Update the host path for volumes in the node-agent DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to +`/var/nutanix/var/lib/kubelet`. + +```yaml +hostPath: + path: /var/lib/kubelet/pods +``` + +to + +```yaml +hostPath: + path: /var/nutanix/var/lib/kubelet +``` **OpenShift** diff --git a/site/content/docs/v1.12/restore-reference.md b/site/content/docs/v1.12/restore-reference.md index 8d646407c6..2994d72fd6 100644 --- a/site/content/docs/v1.12/restore-reference.md +++ b/site/content/docs/v1.12/restore-reference.md @@ -269,7 +269,9 @@ are `none` (default) and `update`. If you choose to update existing resources du You can also configure the existing resource policy in a [Restore](api-types/restore.md) object. -**NOTE:** Update of a resource only applies to the Kubernetes resource data such as its spec. It may not work as expected for certain resource types such as PVCs and Pods. In case of PVCs for example, data in the PV is not restored or overwritten in any way. +**NOTE:** +* Update of a resource only applies to the Kubernetes resource data such as its spec. It may not work as expected for certain resource types such as PVCs and Pods. In case of PVCs for example, data in the PV is not restored or overwritten in any way. +* `update` existing resource policy works in a best-effort way, which means when restore's `--existing-resource-policy` is set to `update`, Velero will try to update the resource if the resource already exists, if the update fails, Velero will fall back to the default non-destructive way in the restore, and just logs a warning without failing the restore. ## Removing a Restore object diff --git a/site/content/docs/v1.12/velero-install.md b/site/content/docs/v1.12/velero-install.md index 1ac1395d52..9dba90d12d 100644 --- a/site/content/docs/v1.12/velero-install.md +++ b/site/content/docs/v1.12/velero-install.md @@ -39,7 +39,7 @@ This section provides examples that serve as a starting point for more customize ```bash velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket mybucket --secret-file ./gcp-service-account.json -velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --provider aws --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-node-agent +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-node-agent velero install --provider azure --plugins velero/velero-plugin-for-microsoft-azure:v1.0.0 --bucket $BLOB_CONTAINER --secret-file ./credentials-velero --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] ``` diff --git a/site/content/docs/v1.2.0/velero-install.md b/site/content/docs/v1.2.0/velero-install.md index 42be5d6aca..38813241ef 100644 --- a/site/content/docs/v1.2.0/velero-install.md +++ b/site/content/docs/v1.2.0/velero-install.md @@ -38,7 +38,7 @@ This section provides examples that serve as a starting point for more customize ```bash velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket mybucket --secret-file ./gcp-service-account.json -velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --provider aws --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-restic +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-restic velero install --provider azure --plugins velero/velero-plugin-for-microsoft-azure:v1.0.0 --bucket $BLOB_CONTAINER --secret-file ./credentials-velero --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] ``` diff --git a/site/content/docs/v1.3.0/velero-install.md b/site/content/docs/v1.3.0/velero-install.md index 42be5d6aca..38813241ef 100644 --- a/site/content/docs/v1.3.0/velero-install.md +++ b/site/content/docs/v1.3.0/velero-install.md @@ -38,7 +38,7 @@ This section provides examples that serve as a starting point for more customize ```bash velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket mybucket --secret-file ./gcp-service-account.json -velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --provider aws --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-restic +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-restic velero install --provider azure --plugins velero/velero-plugin-for-microsoft-azure:v1.0.0 --bucket $BLOB_CONTAINER --secret-file ./credentials-velero --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] ``` diff --git a/site/content/docs/v1.3.1/velero-install.md b/site/content/docs/v1.3.1/velero-install.md index 42be5d6aca..38813241ef 100644 --- a/site/content/docs/v1.3.1/velero-install.md +++ b/site/content/docs/v1.3.1/velero-install.md @@ -38,7 +38,7 @@ This section provides examples that serve as a starting point for more customize ```bash velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket mybucket --secret-file ./gcp-service-account.json -velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --provider aws --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-restic +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-restic velero install --provider azure --plugins velero/velero-plugin-for-microsoft-azure:v1.0.0 --bucket $BLOB_CONTAINER --secret-file ./credentials-velero --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] ``` diff --git a/site/content/docs/v1.3.2/velero-install.md b/site/content/docs/v1.3.2/velero-install.md index 42be5d6aca..38813241ef 100644 --- a/site/content/docs/v1.3.2/velero-install.md +++ b/site/content/docs/v1.3.2/velero-install.md @@ -38,7 +38,7 @@ This section provides examples that serve as a starting point for more customize ```bash velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket mybucket --secret-file ./gcp-service-account.json -velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --provider aws --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-restic +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-restic velero install --provider azure --plugins velero/velero-plugin-for-microsoft-azure:v1.0.0 --bucket $BLOB_CONTAINER --secret-file ./credentials-velero --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] ``` diff --git a/site/content/docs/v1.4/velero-install.md b/site/content/docs/v1.4/velero-install.md index 42be5d6aca..38813241ef 100644 --- a/site/content/docs/v1.4/velero-install.md +++ b/site/content/docs/v1.4/velero-install.md @@ -38,7 +38,7 @@ This section provides examples that serve as a starting point for more customize ```bash velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket mybucket --secret-file ./gcp-service-account.json -velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --provider aws --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-restic +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-restic velero install --provider azure --plugins velero/velero-plugin-for-microsoft-azure:v1.0.0 --bucket $BLOB_CONTAINER --secret-file ./credentials-velero --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] ``` diff --git a/site/content/docs/v1.7/velero-install.md b/site/content/docs/v1.7/velero-install.md index 8cc6d4d2e7..5f1138480d 100644 --- a/site/content/docs/v1.7/velero-install.md +++ b/site/content/docs/v1.7/velero-install.md @@ -39,7 +39,7 @@ This section provides examples that serve as a starting point for more customize ```bash velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket mybucket --secret-file ./gcp-service-account.json -velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --provider aws --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-restic +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-restic velero install --provider azure --plugins velero/velero-plugin-for-microsoft-azure:v1.0.0 --bucket $BLOB_CONTAINER --secret-file ./credentials-velero --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] ``` diff --git a/site/content/docs/v1.9/csi.md b/site/content/docs/v1.9/csi.md index 80198c4c97..c660776035 100644 --- a/site/content/docs/v1.9/csi.md +++ b/site/content/docs/v1.9/csi.md @@ -41,7 +41,7 @@ This section documents some of the choices made during implementation of the Vel ```yaml velero.io/csi-volumesnapshot-class: "true" ``` - 1. The VolumeSnapshot objects will be removed from the cluster after the backup is uploaded to the object storage, so that the namespace that is backed up can be deleted without removing the snapshot in the storage provider if the `DeletionPolicy` is `Delete. + 1. The VolumeSnapshot objects will be removed from the cluster after the backup is uploaded to the object storage, so that the namespace that is backed up can be deleted without removing the snapshot in the storage provider if the `DeletionPolicy` is `Delete`. ## How it Works - Overview diff --git a/site/content/docs/v1.9/restore-reference.md b/site/content/docs/v1.9/restore-reference.md index a91ef960db..d843369db3 100644 --- a/site/content/docs/v1.9/restore-reference.md +++ b/site/content/docs/v1.9/restore-reference.md @@ -224,6 +224,10 @@ You can change this policy for a restore by using the `--existing-resource-polic You can also configure the existing resource policy in a [Restore](api-types/restore.md) object. +**NOTE:** +* Update of a resource only applies to the Kubernetes resource data such as its spec. It may not work as expected for certain resource types such as PVCs and Pods. In case of PVCs for example, data in the PV is not restored or overwritten in any way. +* `update` existing resource policy works in a best-effort way, which means when restore's `--existing-resource-policy` is set to `update`, Velero will try to update the resource if the resource already exists, if the update fails, Velero will fall back to the default non-destructive way in the restore, and just logs a warning without failing the restore. + ## Removing a Restore object There are two ways to delete a Restore object: diff --git a/site/content/docs/v1.9/velero-install.md b/site/content/docs/v1.9/velero-install.md index 8cc6d4d2e7..5f1138480d 100644 --- a/site/content/docs/v1.9/velero-install.md +++ b/site/content/docs/v1.9/velero-install.md @@ -39,7 +39,7 @@ This section provides examples that serve as a starting point for more customize ```bash velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket mybucket --secret-file ./gcp-service-account.json -velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --provider aws --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-restic +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-restic velero install --provider azure --plugins velero/velero-plugin-for-microsoft-azure:v1.0.0 --bucket $BLOB_CONTAINER --secret-file ./credentials-velero --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] ``` diff --git a/site/static/img/contributors/anshul-ahuja.jpeg b/site/static/img/contributors/anshul-ahuja.jpeg new file mode 100644 index 0000000000..43b19ff4f0 Binary files /dev/null and b/site/static/img/contributors/anshul-ahuja.jpeg differ diff --git a/tilt-resources/examples/node-agent.yaml b/tilt-resources/examples/node-agent.yaml index d5c10fc47e..835ba297ff 100644 --- a/tilt-resources/examples/node-agent.yaml +++ b/tilt-resources/examples/node-agent.yaml @@ -49,6 +49,9 @@ spec: - mountPath: /host_pods mountPropagation: HostToContainer name: host-pods + - mountPath: /var/lib/kubelet/plugins + mountPropagation: HostToContainer + name: host-plugins - mountPath: /scratch name: scratch - mountPath: /credentials @@ -60,6 +63,9 @@ spec: - hostPath: path: /var/lib/kubelet/pods name: host-pods + - hostPath: + path: /var/lib/kubelet/plugins + name: host-plugins - emptyDir: {} name: scratch - name: cloud-credentials