diff --git a/changelogs/unreleased/7100-blackpiglet b/changelogs/unreleased/7100-blackpiglet new file mode 100644 index 0000000000..1084a29d1a --- /dev/null +++ b/changelogs/unreleased/7100-blackpiglet @@ -0,0 +1 @@ +Generate VolumeInfo for backup. \ No newline at end of file diff --git a/config/crd/v1/bases/velero.io_downloadrequests.yaml b/config/crd/v1/bases/velero.io_downloadrequests.yaml index 2d9e2a9819..bf5f5e74a0 100644 --- a/config/crd/v1/bases/velero.io_downloadrequests.yaml +++ b/config/crd/v1/bases/velero.io_downloadrequests.yaml @@ -53,6 +53,7 @@ spec: - RestoreItemOperations - CSIBackupVolumeSnapshots - CSIBackupVolumeSnapshotContents + - BackupVolumeInfos type: string name: description: Name is the name of the Kubernetes resource with diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index c86a0c5519..f0814a70ac 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -33,7 +33,7 @@ var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=M\x93\xdb:rw\xff\n\x94r\xf0\xee\xd6H^'\x97\xd4ܼc;Q\xbd\xf7\xfc\xa6<\xb3\xdeK.\x10ْ\xf0\x06\x04\xb8\x00\xa8\xb16\x95\xff\x9eB\x03\xe0'H\x82\xb2\xe6ś2/\xf6P@\xa3\xd1\xdd\xe8\x0ft\x03\\\xafׯhɾ\x80\xd2L\x8a[BK\x06_\r\b\xfb\x97\xde<\xfd\xbb\xde0\xf9\xe6\xf4\xf6\xd5\x13\x13\xf9-\xb9\xab\xb4\x91\xc5gвR\x19\xbc\x87=\x13\xcc0)^\x15`hN\r\xbd}E\b\x15B\x1aj_k\xfb'!\x99\x14FI\xceA\xad\x0f 6O\xd5\x0ev\x15\xe39(\x04\x1e\x86>\xfdy\xf3\xf6_7\x7f~E\x88\xa0\x05ܒ\x1d͞\xaaRoN\xc0A\xc9\r\x93\xaft\t\x99\x05yP\xb2*oI\xf3\x83\xeb\xe2\x87s\xa8\xfe\x05{\xe3\vδ\xf9\xa9\xf5\xf2g\xa6\r\xfeP\xf2JQ^\x8f\x84\xef4\x13\x87\x8aS\x15\u07be\"Dg\xb2\x84[\xf2\xc9\x0eQ\xd2\f\xf2W\x84x\xacqȵG\xf8\xf4\xd6AȎPP\x87\v!\xb2\x04\xf1\xee~\xfb\xe5\xdf\x1e:\xaf\t\xc9Ag\x8a\x95\x06\xe7\xee\x10#L\x13J\xbeഈ\xf2T&\xe6H\rQP*\xd0 \x8c&\xe6\b$\xa3\xa5\xa9\x14\x10\xb9'?U;P\x02\f\xe8\x1a4!\x19\xaf\xb4\x01E\xb4\xa1\x06\b5\x84\x92R2a\b\x13İ\x02\xc8\x1f\xde\xddo\x89\xdc\xfd\x06\x99ф\x8a\x9cP\xadeƨ\x81\x9c\x9c$\xaf\np}\xff\xb8\xa9\xa1\x96J\x96\xa0\f\vtvOKxZo{\xd3{m)\xe0Z\x91\xdcJ\r\xb8ix*B\xee\x89f\xe7c\x8eL7\xd3E9\xea\x00&\xb6\x11\x15\x1e\xf9\ry\x00e\xc1\x10}\x94\x15ϭ\xb0\x9d@Y\x82e\xf2 \xd8?jؚ\x18\x89\x83rj\xc0\v@\xf30a@\t\xcaɉ\xf2\nn\x90$\x05=\x13\x05v\x14R\x89\x16\x93\x96r\x88\xfe؛G\xc3H+\xa98\xf31ԭ#зh᱈Z\x0f\x17-g\xceN,\xaf(G#JE\xe6\xe6Ck\xbcb\xc6x\x82\xc9\x03\x9c\x9d\x89\x0e\x98[Nt\xc2\x11)\xc0\xba\xa0\x85\x8d\xc1\x86McF\xc6=c\xd3\xdeQ\xebgH'\xa2\xaa\xe2\xa0\xfdPαkt\xc0\xcd(\xe8\x9a#.~\xe7t\a\x9ch\xe0\x90\x19\xa9\xe2\xe4\x98c\xb2{R\xf4\xda\b\x15#\x1a\xae\x1b\x104\x13\x9b\x00I0\xa8:\xb2\xec\xe8\xdc4+A\b\x87\xe4\x124\xaerZ\x96\x055\xd9\xf1\xc3W\xeby\xe9&\x97\x93H\x97~g\xe7\xbf\x06\x7f\xbek\x98g\xe0\x12\xdc\x14f\n\n\xb7\xd9\xfc\x88\xd4lޠG\xf5\xee\xd3\xfb\xd8nV\xf7I\x90\xbc\xc1D\xde\xf5\x90m\x0f\xed\x9d\xf2\xd4ixק\x8eo\\\x1a\xe1\x86P\xf2\x04g\xe7\xb1PA,s\xa8\x1dh$\xd2\x19\x12\a\xf3\x19(dOpF0>A1\xdb;U\x14\xdc\xf3\x04\xe7\x94f=\x02Z\x9c\x98\xf6\x89\x17KI\xfb\x02\t\x81\xfb\xd9\xe9\xc4#\x98l\n\xbah~r$]\x91\x84'\xd0\xfe\x82i\xd6lk%ꐱ\xaf\xb5c\x91]\x05GV&NԚ9\xdcJ\x90\xfb:\xdd\xf4\x85r\x96\xd7\x039\xb9ߊqo\xb8\xfb|\x92f+nȇ\xafL\xfb\x8c\xdf{\t\xfa\x934\xf8\xe6E\xc8\xe9\x10\xbf\x80\x98\xae#./\xe1Զ\xa5C;o\x95 \xdc\xeeٺ\b\xaff\x0f\xd3d+l\xdc\xe2\xe9\x81YH7ܴ}\xe8>E\xa511%\xa4X\xbb\xad\x97\xd8H\x8e؉ \xa5\xeapd\x88Z=\xe8\xc8^O\xfcy\xb4\x96\xc4\xf5wyUN3\xc8C^\x05\xb3\x81\xd4\xc0\x81e\xa4\x00u\x982\x1c\xed\xa7\xb4\xfa=\r\x85D\xad랅\x12\x96f\xda\xc3\xe3Uwt\xf3\xbb\xfb\xac\xed\xcaMh\x15\x98=\xdbt$\t8\xdet~Fhb\xd1\xff\x98\xa5.\xcds,Ӡ\xfc~\x81\xc6_\xc0\x8b\xa1\xedw\x889\vYPLN\xfc\xb75s(\xd0\xffCJ\xcaT\xc2\x1a~\x87\xe5\x18\x1c:}\xfd.V{\x18;\x02\xd3\xc4\xf2\xf7D\xf90\xbd\x1c\x99\x9c\xb4\xba\x05\xb83\xe4r?\xf0Xn\xc8\xf3QjgS1)2\v\x92i\xb2z\x82\xb3OƵ\xf5\xc0j+V\xce\xc0/V7\xb5\xb7 \x05?\x93\x15\xf6]}\x8b\x13\x94(\x89\x89;\xae\x9f\xea\xf2\x93uA˵\x97^#\v\x96\x8d\xf6\xc3r\x99T\x17\xdbƠ\xc1\x83\xb0\x1d\xeb\x1a\x11\xeb\x1eO\xcd6I~K\xa9#\x99\xef\x11T\xee\xa56nG\xb2\xe3\xce.\xd9\xfd\"N\xf6\xfc\xae\x17\xa1{W\xa5#U\xa8\xbf\xb0겷Qk\xb9\xad\xa75\xb3\xcb\x19\xf8\x9d4\a\xd4\x06d\xabf\xe5;=\xbcr9\v\x1c\x84f\xe8\x94\xcc\xc2-\x95\xcc@G\xb3\xc5͓\xa0\xe5g6\x17\xeb\x8dE\xea\x02\x1fW\xdc0\xbd\x99\x19\x9etG\xd6\x12ia\b\xf0\xe1kk\xd7\xd3*\r\xfb\xf7\x9c\xf0-ŋ\xe0Z/\nگ\xe2IB\xf1\xce\xf5\f\xcb\xc4\x03r!\x85:T\xa8\"\xd2=O/H߃y/\x98\xd8\xe2\x00\xe4\xed\xd5݁Z\xb9\xc6j9bO\x8f\xe4\xbeoC\xf4\xfa\x85\x18)\xe6\x88=\xa5\xc4\x1d\x7f\x05\x1d\xce\r\xf7ǭ\x83\x99\bRH\xd3ކ\xb0pK\x99\xbf\xd6dϔ6mDS\x85\"^+\x12{\x96F\\\xe2\x83R\x17\x05\\\xbf\xba\x9e\xad\r\xb0\xa3|\x0e\xb5P\xa3\xc5\x13\xb1\a\x93I@؞0C@d\xb2\x12\xb8mc\x97:\x0e\xe1X\xe0\x14t2\xc9\xd2\x14\x84}@TE\x1a\x01\xd6(uLL\xeeﴛ\x7f\xa4,\x96\x81\x1e>\v\xd9f\xc6J\xc6bO\x87m\xa1v\xac]\xd4VЯ\xac\xa8\nB\vK\xfa\xd4pi\xef*\xce:\x1c\xaf\xeb\xce\x10.\x9a\x11#\xed\xa2*9\x98\xd4\x15\xe9*\xcc\xec2\xd1,\x87\xda0{)\x90\x82P\xb2\xa7\x8c\x8f\x94\xbb\f\x9fE\xb4]\x12\xa3xeq\xbd\xe0#m\xf05\x92\"a\x037\xd1ɜ\xd6֥Jw\x15\xef\x15\xa4\xb9gs\x9b\xd9\xc1=+\x15\x93X\xa7we\x0f͋\x18\x15\xe7\x1f.\xda\xe0\xf9\xe1\xa2\xcd3ś\xdfP/\x1f\x06\xbd\xa8^~;\xd9\xf9J\xf5\xf2\x1eþ\xd7}\xa5j\xf90\xffe\xd5\xf27\xbeT\xa3\x00\x1a\xb6\xe7].>\x1f\x1b\xb27\xda\x00\xf0\xef\\\x7f;\xa8\x0f\xbb\x8c\xf1/^m?\xc2\xfc\xc4\xc2\xf8՟V\xdf\x1f\xa5\x17\xd3v\x94\x9a\x032E&\x15\x8e\xf9ڸ\xb2]\xdc\xd5-\xa4\xfb>\x85s\xa94\xa6V\xccO\xd1k\xa8eZ\x04\xfb^\x17\xb3\x81\xe2\xd7\xd2ۊ\xb4\x13\x9d\xdbH\x97\xb93\x9d\x91\xf9\xe0&\x80>\x8b쨤\x90\x95\xf6\xfb\x06\x16\xfa;ܾ\xf0\xa9P,\x03KT\xb0o\xc9QV\x91\x8a\xed\t\xda\xcd\xd4\xef\x8dW\xed\xf9\x1c5\x18zz\xbb\xe9\xfeb\xa4\xaf\xe1#\xcf\xcc\x1c#x>\x1fA`v]\x1c\xda\x05\xf9a\xc1\xf9\x03\xe7}A\"R\x11\xc1\xf8\x98\xc1\xaaO\xe5wLӯ\xa5\xdb$Zl\xf9\xa778Ҫ\xfc.\xae\xed\xeb\xd6\xee\x8d8\x81K\x93\xd9\xe9G\x18ҫ\xf7\xa6\xcb\xed\x96\xd4\xec\xf5+\xf2F\x81\xceW\xea\xa5\xecM\xcdT\xe5]P\x8b\x97X\x87\xfdͩ\xf7\x94j\xbb\x8bj\xecfK\x95\x13+\xeb\xba5s\xd3 \x17\xd4\xd3%\x11g\xbevnqŜ\xafP\x9b\x9cGr\x9d\\\xa4\x02n\x12\xf0hu\xdcT\xdd\xdb̾\xf7\xb0&.\xbd\xdam\x124V\xc2\xcd\u05f8]\xaf\x92\xfd\x1aQ\xf6\xb8\xaa\x99\xadS\x9b\x8d§\xf1\x9b\xadD[R\x7f6K\xb1\vk\xcd\xeaZ\xb2\x91q\x97V\x98u+\xc8F\x80\xa6ԕ\x8dԍ\x8d@\x9c\xac&K\xad\x16\x1b\x81=cv'\xa5d\xf2\xc7%Ub\xf1KTȬ5俗\xfc]J\x06\xa9:\xce\xe5\\@\xf3k\xaf\xb9\xe5|𱦝\u0558\x9f\xca\xccq\xb9\xb3ZTܰ\x92cz\xf1\xc4\xf2h\xccn\x8ep\xae/\x86\xf8M\xe2qMw\x99\t\xf9\xf5s-̛\x9e\xcbM5y\x06\xce\t\x8d\x89\xe2`晻\a(\x93k\xb0\x16\xc2.O\x7f兿.\xe8\xc6\xc9;\x9eH\x8de`\xcc\x11\n\ve\xfcړQU>\xedN:\xcf\x17\xdf\xfd\xbd\x02u&x=K\xed_̜\x87r\xcbR\xdbX((\n\xafm\xdc\xedS=7\xbbY\x9e\xe4\x9dp\x06/\n\xb6\x87#±\x1a\x82\u05fc\xb6\xca\xd0F\r#M\xe3\x1b\xb1\xb2\xee\x1d\xf9}\xceSM=L\xf4\xb2\x81\xc6\xf2Pc\xd6ȿH\xb8qy\xc01\x012\xf5pPZB|\xf60\xd0K\x05\x1es\xa1G\xb2ϕv\xd8\xe7%\x0e\xf9,8ܳ \x04Y\x16\x84$\x93)\xe5\x10ϋ\x84\"/\x18\x8c\xbcD8rY@2\x03\xb2w8'\xe5\xd8MR\xb1Gr\xbe3\xa5Xc>%9}\x9c&\xe1\x18MB\xb2r\x0eӄ\xe32ˎ\xc9$\xd0\xf0\x85B\x95\x17\nV^\"\\yـe6d\x99\x95\x9c\x99\x9f\x97\x1do\xb9x\xf3^\xaa\x1c\xd4d\xae#U4'\x85\xb2\x17_t\xc7\xec\xed\xfc\x87;\xe5l\xab\x8e+\x1b۰\xaeO\xbdg\xe4'&|\x1e\xd5\na\xcb\xeew\x120\x8d#\x12\xdf\xffo\xbc<\x7fۨ\xcb\xdah(\xa9\xc2\f\xeb\xee\xecJ+\xf4\x86|\xa0ٱ\a\xfd\x18\x8d+\xf6R\x15ԐU\x9d\xf2z\xe3\x80ۿW\x1bB>\xca:i߾IF\xb3\xa2\xe4g\x1b7D`\xae\xda .\x13\x88\xa8\xf0\x85\xf1\xef%gY\xc4ӊ^.\xe4\x1a\x0f\xae\x84\xc0+\x8f\xb2v껴\r\xe3\x8e\x16:e\xdd\xeb\x15\xf7\x92s\xf9\xbc0\x1c\xa7%\xfb\x0f\xbc\xa5y~\x0f\xe7\xdd\xfd\x16\x9b\x06I\xc1\u06dd\xeb\n\xa1\x1a\xe9\x1dX\x8b\xd9Lgl\xc5o\xf7\x1d\x88\x91J\xbb\xfaO\x94\xd6\xdab\xb3\xb1[\x97\\՟\xd54\xf7[\x87\xdd\x06\x85\x85\x8a3\x91X\xeba\x8eL\xe5\xeb\x92*svu\x0575\x0e\xe3\xfb8\xc1nN\xed\xb6\x8c\x9a\x97\xe1u\xbfQچ[\x7f1\x99w.\xbb\xa9\xd0>E/\xc1c\xfc(\xdf\xec!\xbe+\xe21\ue0ac\x91R\x91\xd7Ѣ\xa4\xab\xedbi\x7f\xb5\xed/\xf2\x04\uf8fbY\x1d\xf2<\xf4\x9aGʉ\x02Dw\xb9\xeb\xd4\xed\xa0x9\xe7e\xba(^\x1f\x14\x86\xf6\xd7w&\xceŷ\x8eL%\xdc\\\x1a\xe0\xea\xf8\xae\x8d]^\xf7_0\xb4\xaaU\x98wv|\xf0\x14\xb6\xae\xfa\x17\xdc\xfd\xe5\xfa5R\xdaHE\x0f\xf0\xb3tW/\xcfѠۺs\xef\xb6wyB\xcdbX\r\xb1P\xc0_\x02\xdd\x03֔\"\x0f\xae\xc1\xb5X.\xbc\xd5\xd7\x18>3\x99\xc7ǟ\xdd\x04\f+`\xf3\xber\xb9|\xab\xed4Xj\x86\x89\xb9N;\xfb\xdfc\xc4^\x10\xbcO\xb6ş\x16\xde\n\xb0\xdc\x19\xcb\xde\x16a\x7f\xea\\$\x1dH4'\xa2_\xe2\xbdZ\xfbK-&9\xd7#*\xa1cpZw\xe9\xe3\xce+\x1eW\xbe\ueb4bc\xce\xe4\xd8m\xe3x\xc3\xf6\xfc}\xe3\xee\"n\xffu\x01_4_)\xbc2\xd1_ҍW\f^t\xe5\xf8\xae\xae\v\xa9\xabN\xf4;cl\xa0\x1c\xd3\xdc\x11\xf4F\xfa\xd6\x06N\x1aʉ\xa8\x8a\x1d\xfa\xac1\x95Rw\xc1\x8a\x95\xc9R\x15\xe7\x80L0Α\x9a\t\x03\x87\xc1\xa6{l\xaew\xbe\xc6\xf9\x92\xb9\xd6}\xd3窫,\x03\xad\xf7\x15\xe7纾z\xc9\xc4c\xd6\xe5J\xa4\xf8H\x19\xbf\x88\x0e\xae\xe3\b\x11\xdc\xdcF\xf5h\x12\x9b}Q'\x88<,ށ)\xb0\x0f\x9e:XF\a\xcf\x02_k\xa5\r-\xe6nN\xbf\x1b\xf6\xc0\xcfZ\xa8\xbcU\x9dU_\xff\xfdLu\xc3\xe6\x98Oـs=\xd1\x05\xb5\xd0 'p\x02A\xa4\xc0\xaay\xbc\x8b\xd3}z\xa5\xdf'\x02\xb5\rŗ\xe5W%\x974\x0f\x06.D\x92\xfes\x1d\x8fh\xbe\xd5\t\xd4k=\x01\xb3\xbe\xd0=B\x84\xa1d\xba\xd0\xee\xd6\xfaF\xb0\x8e\x02M2\xfdQ]\x9bi\xd6\xd5\xf3\xc9J\xeb\xeea;\xd6sT\x82C\x83\x18\xff\x06\x1fN\xf8F%5\x9cY\xaa\x8a\x1a\xcelNAu\xd4Qdr\x8d\x82\xba\xfa4q\xad\xce\u07b8\x8c\x8d\x9c\a\x80Nj\xc2u\xfa\xee|Q\x01Z\xd3C\xb8j\xf9\xd9:`\a\x10\x80\x9b\x11\x91\xd9\xf8\xed\xdd\xe6\x027\xaf,R\xa4D\r\xa1C\x99\x86\xa9\x94h\xa5r\xea\xf3b\r\xba4{\x1a\xc5ԧ\xa2\xc3G\x0f\xdf\xf8\x8b\xd3\xd7{%\x8b\xb5\xe7\x05VW\xdc\xf8\xf4\x8ab\xd2\x06\xec\xe6\x18%9q\xdfT\xf27\x14\xa3\x18\x94%\bB\xb5\xc7'\xe1b\x89i\xb6N\xec\xa6jC\x95I\x8d\x83\x1e:\x8dgB \x84\x1c\xc7\xf7\xc1\xa7\x8f\xdc\x05\x1bw\xfe\x93b5\xe0\x1b\xa2\x99\b\xdf[t\xc9)'\n\xdaFF\nps-Z83\x88i:\x11L\x17\xfd\xdf7x9\xd56\xf1C\x8a\x17\xfc\xa5\u05fcw\x88\n?\xaeU7\xf1\x9ek\x84\x1e\x7f`{W˓Y\xac\xff\xf8\x7f~8\xea\x94\xe4e\xbd\x9et\xb0\xd0w\xaa=\xa5\x99Oi\xdds\xb0\x9e\x8f\x06\xe8\xfan\xaf\x179\xe9\xa7\xcb\xc2\xcekƜ\xe1S\xa0\u05c9\xc4N\x97E\x9b/\x16j^wv\xcf\x14??8\xb7\xc6\xfe\xe6\x9bEbM\x0f!\x12mF\xa6Qǟ\xb3\xd1f+\xd8\f8\x8e|-\xa8\x17\x80^)܌ځ\xc1KT\xa0ykm\xfb\x91\xfc\x9b\xff\r\x00\x00\xff\xff\x9a\xfbL\xe1\xa9x\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YKs\xe3\xb8\x11\xbe\xebWt\xed\x1e|YQ3\x9bKJ\x97\x94FN\xaa\xa6\xe2\x89]#ǹ\xe4\xb0\x10Д\xb0\x06\x01\x06\x0fi\x94T\xfe{\xaa\x01\xf0!\x92\xb2\xe4\xa9$\x8b\x8bM\xb2\xd1\xe8\xfe\xfa\r\xcd\xe7\xf3\x19\xab\xe5\vZ'\x8d^\x02\xab%~\xf3\xa8\xe9\xc9\x15\xaf\xbfw\x854\x8b\xc3\xc7٫\xd4b\t\xeb༩\xbe\xa23\xc1r\xbc\xc7Rj\xe9\xa5ѳ\n=\x13̳\xe5\f\x80im<\xa3\u05ce\x1e\x01\xb8\xd1\xde\x1a\xa5\xd0\xcew\xa8\x8bװ\xc5m\x90J\xa0\x8d̛\xa3\x0f\x1f\x8a\x8f?\x17\x1ff\x00\x9aU\xb8\x84-㯡v\xdeX\xb6CexbY\x1cP\xa15\x8543W#\xa7\x13vքz\t݇\xc4!\x9f\x9e$\xff\x14\x99m\x12\xb3\x87\xcc,~W\xd2\xf9?_\xa6y\x90\xceG\xbaZ\x05\xcb\xd4%\xb1\"\x89\xdb\x1b\xeb\xff\xd2\x1d=\x87\xadS\xe9\x8bԻ\xa0\x98\xbd\xb0}\x06ษq\tqw\xcd8\x8a\x19@\x86&r\x9b\x03\x13\"\x82\xcdԓ\x95ڣ]\x1b\x15*ݞ%\xd0q+k\x1f\xc1L\xba@V\x06\x1am\xc0y\xe6\x83\x03\x17\xf8\x1e\x98\x83ՁIŶ\n\x17\x7fլ\xf9?\xf2\x03\xf8\xd5\x19\xfd\xc4\xfc~\tE\xdaU\xd4{暯\xc9FO\xbd7\xfeD\n8o\xa5\xdeM\x89\xf4\xc0\x9c\x7faJ\x8a(ɳ\xac\x10\xa4\x03\xbfGP\xccy\xf0\xf4\x82\x9e\x12B@\x10!4\b\xc1\x91\xb9|\x0e\xc0!q\x89\x18MK\xaaFg\x9d\x89M\xa2\xc0ˀK\x92\x9f\xded\xe9{l\x1b\xff.\xb8Ŗ\xa5\xf3\xac\xaa\xcf\xf8\xaevx\x89\xd9\x19\x14\xf7X\xb2\xa0|_U\xb2\x92\xea\xfb\xe5\xb9Z5\xf2B\xa4]g'ޟ\xbdK\xa7n\x8dQ\xc8\x12\x97Du\xf8\x98\xbc\x90\xef\xb1b\xcbLljԫ\xa7\xcf/\xbfۜ\xbd\x86)G\x1a\x04\x05\x19\x8e\xf5l\xb3G\x8b\xf0\x12\xe3/\xd9\xcde\xd5Z\x9e\x00f\xfb+r\xdf\x19\xb1\xb6\xa6F\xebe\x13,i\xf5rQ\xef\xed@\xa6;\x12;Q\x81\xa0$\x84ɏr\xbc\xa0Ț\x82)\xc1\xef\xa5\x03\x8b\xb5E\x87\xda\xf7\xe1m\x05+\x81\xe9,^\x01\x1b\xb4Ćb9(A\xb9\xeb\x80փEnvZ\xfe\xb3\xe5\xed\xc0\x9b\xec\xbc\x1e\x9d\x1f\xf0\x8c\xf1\xa9\x99\"W\r\xf8\x130-\xa0b'\xb0H\xa7@\xd0=~\x91\xc4\x15\xf0\x85\xfc]\xea\xd2,a\xef}햋\xc5N\xfa&\asSUAK\x7fZ\xc4t*\xb7\xc1\x1b\xeb\x16\x02\x0f\xa8\x16N\xee\xe6\xcc\xf2\xbd\xf4\xc8}\xb0\xb8`\xb5\x9cG\xd1uJ\x9a\x95\xf8\xd1\xe6\xac\xed\xee\xced\x1dEmZ1k\xbea\x01ʘ\xc9\v\xd2֤E\a4\xbd\"t\xbe\xfeq\xf3\f\xcd\xd1\xd1\x18C\xf4#\xee\xddFי\x80\x00\x93\xbaD\x9b\x8cXZSE\x9e\xa8Em\xa4\xf6\xf1\x81+\x89z\b\xbf\v\xdbJz\xb2\xfb?\x02:O\xb6*`\x1d\v\x13l\x11B\x1d㾀\xcf\x1a֬B\xb5f\x0e\xff\xe7\x06 \xa4ݜ\x80\xbd\xcd\x04\xfd\x9a:$N\xa8\xf5>4\xb5\xf0\x82\xbd&\xa3xS#?\x8b\x1f\x81NZ\xf2p\xcf<Ƹ\x18\xe0\x9aC\xfcr1m\xd6tp\xd3b\x9c\xa3s_\x8c\xc0ᗁȫ\x96\xf0L\xc6\x1am%],\x8bP\x1a;\xac\x18\xac\xcd\xc0\xfd\xd5d\xaab\xf4\ru\xa8Ƃ\xcc\xe1+2\xf1\xa8\xd5\xe9§\xbfY\xe9\xc7\a]0$\xad$\xe2\xe6\xa4\xf9\x13Zi\xc4\x15\xe5?\r\xc8[\b\xf6\xe6\betk\xedՉr\x90;i>ζ\xcdZ=}n2o\n\xa0\x1co\x19\xab\x02V9rM\t\x1f@HG\r\x80\x8bL\xc7`\xe9\xa0b\x83\xb0\x04oû\xd4\xe7F\x97r7V\xba\xdf\xd3\\\xf2\x98+\xac\aȭ\xe3I\x94\x9a\xc8;jk\x0eR\xa0\x9dS|\xc8R\xf2,I\xb0\xa9r\x95\x12\x95pcM/DYTŢ\xa0\xa8f\xea\x8a\r\xd7-a쀙\xd4Ƀ;\x061\xd9\xd8*\x97T\xedQ\x8b\xb6\x1b9\x93\xc6Ĭ\xe5P\xc0Q\xfa}J\x87j*\xee\xe0\xcdأ\xf5\x8a\xa7\xa9\xd7\x03ٟ\xf7H\x94\xa9\x80\"8\xe4\x16}\xf46T\xe4>\xe4J\x05\xc0\x97\xe0bB\x1d\xe6\x89f\xc5F\xad\xd9\xfd\x8a\xa71\xd0p\u0378\xb9\x85\xb9.\xf2\x1d\xb5\u038d\xc0\x16K\xb4\xa8\xfddR\xa7\x01\xc4j\xf4\x18\xf3\xba0\xdcQJ\xe7X{\xb70\a\xb4\a\x89\xc7\xc5\xd1\xd8W\xa9ws\x02|\x9e#h\x11NJŏ\xf1\xcf\x05\x95\x9f\x1f\xef\x1f\x97\xb0\x12\x02\x8cߣ%\xab\x95A5\x8e\xd6\xebo~\x8a5\xf6'\bR\xfc\xe1\xee{p1u\x8a\x9c\x1b\xb0\xd9D\xef?Q\xa3\x16\x85\"\x886\xc9*\xc6\x02UJ2v\x95\xad\x99r͔#Nu\x98\xfdE\x89\x89*\xc8TF}\xc5q2}#\xcc\x00\xbe\xcd;C\xcd+V\xcf\x135\xf3\xa6\x92|6\xd46\xb6\xc1W\"\xb2i\xbb\xa5\x16\x92S\xdbv\x1eI\xcd8\"κ\xf3\t\x18\x86\xfd\xfa\xa5\xfc1\rSR7W\xcf+\x12?\xf6i\xbb!.%\xb3\\\x11\x1dzj\xb7\x1ch\xa4\x8a\xc9\xec\x18\xe7\x98B\xb8њb\xd7\x1b`mb\xbcsÊ\xf0\xce|\xb2\r\xfc\x15'\x80\x1f\xa9\xf2)\x126\x18\xa7m$Kp\x18S\xf551\xe0zDp\xb6F{\x8b,\xeb\x15\x11\xb6E\x95\xc1z\x05۠\x85\xc2F\xa2\xe3\x1e5\xcd\x13\xb2\x14?\xfcf3\x93b\xce\xd3\b\x84\xe2+\x1e\xe4\xf8Nh\x8c\xee\xc3hG\x13\xf8m8\xd0\xc3/\xcdh\xbd\xb0\x99\xec\x97\t0J\xa9\xa8s\x9c\xc8\x13]\xc70\xbe\xbd\xfc\xb4y\xb8s\xb1\xe1G\xed\xa7\x9a\xc4#Z\x8c\xf3\x15\n\xea\xf9M\xbe\xc5\bΣ\x9dp\x80\xd6z\xd1栌\xde\r\x02'\xad|\xa7A\xfd\\r(cA\xa0\xa7Ҥw\xc0\xf7Lﰻ\xb3\xca\xf2\xbf-)\xb9\xcf\xc0g:\x0f\x91\xfa\x92{\xdcd\xd1g9\xd5ԏ\xee\x8b;\xe2\xe9\xbb\xe2F\xfaƲ\x17\x87\xa2+\xb8\x8f\xe8\x9b*M\xa0\xce}w\x7fܭ\xef\x1f\x86Ǘ\xd37 \xf1ޛ\xf37nA\xe0\xc8\\w\x87\xfe\xdb\xe1PQ\xb7z\xb5\x05\xfe\x92\xa8\xd2ec\xde\x02lk\x82\x7f+2\xef\xa6\x1c:\xff8\xf0\x1e\x19\xe3O\x1eך\f\xa2i,\u0083\xa5\xc1\xb3\xbbC\x8bIa\xaa\xb6\xdc~\x19\xb5\x1a\xfc2\xd3\xff6\xfe\xdd\xe6\x06\xbd&k\xed\xe8e\xaa\x97=\xbbf\x90\xfbo¶\xbdW^¿\xfe=\xfbO\x00\x00\x00\xff\xff\x80.\x12\xd3P\x1c\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4\x96M\x93\xdb6\x0f\xc7\xef\xfe\x14\x98y\x0e\xb9<\x92\xb3\xed\xa5\xa3[\xb3\xc9a\xa7mƳ\x9bɝ&a\x8bY\x8ad\x01\xd0[\xb7\xd3\xef\xde!)\xf9E\xb67\xdbCy\x13\t\x02\x7f\xfe@\x80j\x9af\xa1\xa2\xfd\x8a\xc46\xf8\x0eT\xb4\xf8\x87\xa0\xcf_\xdc>\xffĭ\r\xcb\xdd\xdd\xe2\xd9z\xd3\xc1}b\t\xc3#rH\xa4\xf1#n\xac\xb7b\x83_\f(\xca(Q\xdd\x02@y\x1fD\xe5iΟ\x00:x\xa1\xe0\x1cR\xb3E\xdf>\xa75\xae\x93u\x06\xa98\x9fB\xef\u07b7w?\xb4\xef\x17\x00^\r\u0601A\x87\x82k\xa5\x9fS$\xfc=!\v\xb7;tH\xa1\xb5a\xc1\x11u\xf6\xbf\xa5\x90b\aDž\xba\x7f\x8c]u\x7f,\xae>\x14W\x8f\xd5UYu\x96\xe5\x97[\x16\xbf\xda\xd1*\xbaD\xca]\x17T\f\xd8\xfamr\x8a\xae\x9a,\x00X\x87\x88\x1d|β\xa2\xd2h\x16\x00㱋\xcc\x06\x941\x05\xa4r+\xb2^\x90\xee\x83K\xc3\x04\xb0\x01\x83\xac\xc9F)\xa0\xbe\xf4X\x8e\ba\x03\xd2#\xd4p \x01\xd68*0e\x1f\xc07\x0e~\xa5\xa4\xef\xa0ͼ\xdaj\x9a\x85\x8c\x06\x15\xf5\x87\xf9\xb4\xec\xb3`\x16\xb2~{K\x02\x8b\x92ē\x88\x12\xd7\x06\x0ft\xc2\xf7\\@\xb1oc\xaf\xf8<\xfaSY\xb8\x15\xb9\xda\xec\xee*i\xdd㠺\xd16D\xf4?\xaf\x1e\xbe\xfe\xf8t6\r\xe7Z\xaf\xa4\x16,\x83\x9a\x94fp\x95\x1a\x04\x8f\x10\b\x86@\x13Un\x0fN#\x85\x88$v\xbaZu\x9c\x14\xcf\xc9\xecL»\xac\xb2Z\x81\xc9U\x83\\\xa0\x8d\x97\x00\xcdx\xb0\n\xd32\x10FBF_\xeb\xe8\xcc1d#\xe5!\xac\xbf\xa1\x96\x16\x9e\x90\xb2\x1b\xe0>$gr\xb1\xed\x90\x04\bu\xd8z\xfb\xe7\xc17\xe7s\xe6\xa0N\xc91?\xd3(\x97\xce+\a;\xe5\x12\xfe\x1f\x9470\xa8=\x10\xe6(\x90\xfc\x89\xbfb\xc2-\xfc\x961Y\xbf\t\x1d\xf4\"\x91\xbb\xe5rkej\x1a:\fC\xf2V\xf6\xcbR\xffv\x9d$\x10/\r\xee\xd0-\xd9n\x1bE\xba\xb7\x82Z\x12\xe1RE\xdb\x14\xe9\xbe4\x8ev0\xff\xa3\xb1\xcd\xf0\xbb3\xad\x17\x17\xa4\x8eR\xe8\xafd \x97yM{\xddZOq\x04\x9d\xa72\x9d\xc7OO_`\n]\x921\xa7_\xb8\x1f7\xf21\x05\x19\x98\xf5\x1b\xa4\x9a\xc4\r\x85\xa1\xf8Dob\xb0^ʇv\x16\xfd\x1c?\xa7\xf5`\x85\xa7+\x99s\xd5\xc2}餹\xa8S4Jд\xf0\xe0\xe1^\r\xe8\xee\x15\xe3\x7f\x9e\x80L\x9a\x9b\f\xf6m)8}\x04\xe6ƕ\xda\xc9\xc2Ծo\xe4\xebJ\xd1>E\xd49\x83\x19b\xdem7V\x97\xf2\x80M x\xe9\xad\ue9e2\x9d\xd1=\x14x{\xb6p\xbd\xa0\xf38\xb6\xc9\xf9\xca\xcd\xc3Cɝ%\x9c\xdd\xc2\x06.z\xee\xeb\\J3\xfc\x97dj'\x1e\xd9\xe8D\x84^N\xfa\xb3\xba\xb6\xe9\xad,\x90(\xd0\xc5\xecLԧbT^ze=\x83\xf2\xfbq#H\xaf\x04^\x90r\x19\xe8\x90r\x9fA\x03&]\xf0\x1b\xb1\x9c\xbe%\x91\x82F\xe6\xf6\xc2\xce\n\x0eW4\xbd\x92\x9d<|rN\xad\x1dv \x94\xf0Ff\x15\x91\xda\xcf\xd6ʛ\xf5\x1d\x04\xabls-\a\x87w\xfa\xbbI(\xb8}\x1a.#5\xf0\x19_\xae\xcc>\xf8\x15\x85-!ϯ|^\\Uz\x87\x9f\x817P\xbaz)/&9\xf7;sB\x91%\x90ڞr\xe5\xb4>\xf4\xef\x0e\xfe\xfa{\xf1O\x00\x00\x00\xff\xff\x045\f\xc6i\n\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4VM\x93\xdb6\f\xbd\xfbW`\xa6\x87\xb43\x91\x9c\xb4\x97\x8eo\xad\x93\xc3N\xd24\xb3N\xf7NS\xb0\xc4.E\xb2\x04\xe8\xcd\xf6\xd7w@J\xfe\x94\xbd\xdeCu\x13\t\x82\x8f\x0f\x0f\x8f\xac\xaaj\xa6\x82y\xc0Hƻ\x05\xa8`\xf0;\xa3\x93?\xaa\x1f\x7f\xa5\xda\xf8\xf9\xf6\xfd\xecѸf\x01\xcbD\xec\xfb{$\x9f\xa2\xc6\x0f\xb81ΰ\xf1n\xd6#\xabF\xb1Z\xcc\x00\x94s\x9e\x95\f\x93\xfc\x02h\xef8zk1V-\xba\xfa1\xadq\x9d\x8cm0\xe6\xe4\xe3\xd6\xdbw\xf5\xfb\x9f\xebw3\x00\xa7z\\@㟜\xf5\xaa\x89\xf8OBb\xaa\xb7h1\xfa\xda\xf8\x19\x05Ԓ\xbb\x8d>\x85\x05\xec'\xca\xdaa߂\xf9Ð澤\xc93\xd6\x10\x7f\x9a\x9a\xfdl\x86\x88`ST\xf6\x1cD\x9e$\xe3\xdadU<\x9b\x9e\x01\x90\xf6\x01\x17\xf0E`\x04\xa5\xb1\x99\x01\fG̰\xaa\xe1t\xdb\xf7%\x95\xee\xb0W\x05/\x80\x0f\xe8~\xfbz\xf7\xf0\xcb\xeah\x18\xa0A\xd2\xd1\x04\xceD\x9d`\x06C\xa0`@\x00\xecw\xa0@9P\x91\xcdFi\x86M\xf4=\xac\x95~La\x97\x15\xc0\xaf\xffF\xcd@\xec\xa3j\xf1-P\xd2\x1d(\xc9WB\xc1\xfa\x166\xc6b\xbd[\x14\xa2\x0f\x18ٌ,\x97\xef@C\a\xa3'\xc0\xdf\xc8\xd9J\x144\"\x1e$\xe0\x0eG~\xb0\x19\xe8\x00\xbf\x01\xee\fA\xc4\x10\x91\xd0\x159\x1d%\x06\tRn8A\r+\x8c\x92\x06\xa8\xf3\xc96\xa2\xb9-F\x86\x88ڷ\xce\xfc\xbb\xcbM\u0090lj\x15\x8fr\xd8\x7f\xc61F\xa7,l\x95M\xf8\x16\x94k\xa0W\xcf\x101\xf3\x94\xdcA\xbe\x1cB5\xfc\xe1#\x82q\x1b\xbf\x80\x8e9\xd0b>o\r\x8f\xbd\xa3}\xdf'g\xf8y\x9e\xdb\xc0\xac\x13\xfbH\xf3\x06\xb7h\xe7d\xdaJE\xdd\x19F\xcd)\xe2\\\x05Se\xe8.\xf7O\xdd7?ġ\xdb\xe8\xcd\x11V~\x16\x99\x11G\xe3ڃ\x89\xac\xf9+\x15\x10\xd5\x17\xc1\x94\xa5\xe5\x14{\xa2eHع\xff\xb8\xfa\x06\xe3ֹ\x18\xa7\xec\x17\xe5\xec\x16Ҿ\x04B\x98q\x1b\x8c\xa5\x88Yy\x92\x13]\x13\xbcq\x9c\x7f\xb45\xe8N駴\xee\r\xd3(f\xa9U\r\xcbl(\xb0FH\xa1Q\x8cM\rw\x0e\x96\xaaG\xbbT\x84\xff{\x01\x84i\xaa\x84\xd8\xdbJp腧\xc1\x85\xb5\x83\x89\xd1\xc9.\xd4\xeb\xa4\xd5W\x01\xb5TO\b\x94\x95fctn\r\xd8\xf8\bj\xdf\xf9\x03\x81\xf5Q\xe6\xe9\xce\xcd\xe0Tl\x91OGO\xb0|\xcbA\xb2\xfdS\xa7\x8e\x8d\xe6G\xac\xdbZ\xbc\x82\x06 \xc5=~\xaa\xcf2^\xc6\x00\x93\xea\x9dD2\x8aXh\x10^\xc5\nĤ\x0e1\x9do-\x1f\xba\xd4OoP\xc1\xef\x19\xf3g\xdf^\x9d_z\xc7\"\xf7\xabA\x0fަ\x1eWN\x05\xea\xfc\v\xb1w\x8c\xfd\x9f\x01c\xb91\xaf\x86\x8e\x17\xef\ue5ba\x12\x98\xec\xc5}\xefQ\xfc\x1e/\x9ft\b\xb8)\xcb\r\x98\x86ț\x0e\xba\\ݽ\x86\xc2\v\xe1W\x8bt\xa1m\xc7/_\xcf/kP.\xf8Q\x83\xb2\xa4\xdcY\b\x9f\xd2\x1a\xa3CF\xda\xdb\xe7\x93\xe1n2#\xc0Sgt\x97\x17f\x01\x8b3\x13ym\xb2Ͻ\x1e\xbe\xf4\xbd\x898\xd1DUn\xae\x89a\x01\x7f6|\xc1\xad.mP\r\x0er\x93\xe3\xb1\xe2D\xaf\xf0\xbc\x1c?R\xadS\x8c\xe8xȒ\xdf\x00\xa7\vn5\xbd\xd1)\xfe\xba\xff\xfc\x82\xf3}\xd8G\xe6Ǭ2\xae\xa0\t\x11+2\xad\xbc\\dN\xbc/{\xd29\x19\xe5;~I\x1d\x135YQ\xfc\x1eLi\x98\x17 ~\xdc\x05\x16\x83FW.\xdfӷbN\x88\x94\x1f6Z\x9d>\xa9\xe4[#4h\x91\xb1\x81\xf5s\xb9i\x9e\x89\xb1?ǽ\xf1\xb1W\xbc\x00\xb9\x94+6\x132r\xc9Z\xb5\xb6\xb8\x00\x8e\xe9\x92\xca&\x0f\x1e:E\x13mxt\xe6\xaf\x123%\x8c]3^U\x06\\\xbc\x0f*\xf8\x82O\x13\xa3_\xa3\xd7H\x84\xe7mt\xf1$\x93Mp6H\xf2rj\x0eX\x1a\x1e\xe4\xc3\xc8\x7f\x01\x00\x00\xff\xff\xa7\x94\xfb\xf9\xa5\r\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4VM\x93\xdb6\f\xbd\xfbW`\xa6\x87\xb43\x91\x9c\xb4\x97\x8eo\xad\x93\xc3N\xd24\xb3N\xf7NS\xb0\xc4.E\xb2\x04\xe8\xcd\xf6\xd7w@J\xfe\x94\xbd\xdeCu\x13\t\x82\x8f\x0f\x0f\x8f\xac\xaaj\xa6\x82y\xc0Hƻ\x05\xa8`\xf0;\xa3\x93?\xaa\x1f\x7f\xa5\xda\xf8\xf9\xf6\xfd\xecѸf\x01\xcbD\xec\xfb{$\x9f\xa2\xc6\x0f\xb81ΰ\xf1n\xd6#\xabF\xb1Z\xcc\x00\x94s\x9e\x95\f\x93\xfc\x02h\xef8zk1V-\xba\xfa1\xadq\x9d\x8cm0\xe6\xe4\xe3\xd6\xdbw\xf5\xfb\x9f\xebw3\x00\xa7z\\@㟜\xf5\xaa\x89\xf8OBb\xaa\xb7h1\xfa\xda\xf8\x19\x05Ԓ\xbb\x8d>\x85\x05\xec'\xca\xdaa߂\xf9Ð澤\xc93\xd6\x10\x7f\x9a\x9a\xfdl\x86\x88`ST\xf6\x1cD\x9e$\xe3\xdadU<\x9b\x9e\x01\x90\xf6\x01\x17\xf0E`\x04\xa5\xb1\x99\x01\fG̰\xaa\xe1t\xdb\xf7%\x95\xee\xb0W\x05/\x80\x0f\xe8~\xfbz\xf7\xf0\xcb\xeah\x18\xa0A\xd2\xd1\x04\xceD\x9d`\x06C\xa0`@\x00\xecw\xa0@9P\x91\xcdFi\x86M\xf4=\xac\x95~La\x97\x15\xc0\xaf\xffF\xcd@\xec\xa3j\xf1-P\xd2\x1d(\xc9WB\xc1\xfa\x166\xc6b\xbd[\x14\xa2\x0f\x18ٌ,\x97\xef@C\a\xa3'\xc0\xdf\xc8\xd9J\x144\"\x1e$\xe0\x0eG~\xb0\x19\xe8\x00\xbf\x01\xee\fA\xc4\x10\x91\xd0\x159\x1d%\x06\tRn8A\r+\x8c\x92\x06\xa8\xf3\xc96\xa2\xb9-F\x86\x88ڷ\xce\xfc\xbb\xcbM\u0090lj\x15\x8fr\xd8\x7f\xc61F\xa7,l\x95M\xf8\x16\x94k\xa0W\xcf\x101\xf3\x94\xdcA\xbe\x1cB5\xfc\xe1#\x82q\x1b\xbf\x80\x8e9\xd0b>o\r\x8f\xbd\xa3}\xdf'g\xf8y\x9e\xdb\xc0\xac\x13\xfbH\xf3\x06\xb7h\xe7d\xdaJE\xdd\x19F\xcd)\xe2\\\x05Se\xe8.\xf7O\xdd7?ġ\xdb\xe8\xcd\x11V~\x16\x99\x11G\xe3ڃ\x89\xac\xf9+\x15\x10\xd5\x17\xc1\x94\xa5\xe5\x14{\xa2eHع\xff\xb8\xfa\x06\xe3ֹ\x18\xa7\xec\x17\xe5\xec\x16Ҿ\x04B\x98q\x1b\x8c\xa5\x88Yy\x92\x13]\x13\xbcq\x9c\x7f\xb45\xe8N駴\xee\r\xd3(f\xa9U\r\xcbl(\xb0FH\xa1Q\x8cM\rw\x0e\x96\xaaG\xbbT\x84\xff{\x01\x84i\xaa\x84\xd8\xdbJp腧\xc1\x85\xb5\x83\x89\xd1\xc9.\xd4\xeb\xa4\xd5W\x01\xb5TO\b\x94\x95fctn\r\xd8\xf8\bj\xdf\xf9\x03\x81\xf5Q\xe6\xe9\xce\xcd\xe0Tl\x91OGO\xb0|\xcbA\xb2\xfdS\xa7\x8e\x8d\xe6G\xac\xdbZ\xbc\x82\x06 \xc5=~\xaa\xcf2^\xc6\x00\x93\xea\x9dD2\x8aXh\x10^\xc5\nĤ\x0e1\x9do-\x1f\xba\xd4OoP\xc1\xef\x19\xf3g\xdf^\x9d_z\xc7\"\xf7\xabA\x0fަ\x1eWN\x05\xea\xfc\v\xb1w\x8c\xfd\x9f\x01c\xb91\xaf\x86\x8e\x17\xef\ue5ba\x12\x98\xec\xc5}\xefQ\xfc\x1e/\x9ft\b\xb8)\xcb\r\x98\x86ț\x0e\xba\\ݽ\x86\xc2\v\xe1\xaf(ҝ\xdb\xf8\xe9\xb8\v\xed=~\xf9\x1a\x7fY\xab\xf2\x10\x18\xb5*K\xca݆\xf0)\xad1:d\xa4\xbd\xcd>\x19\xee&3\x02}S\xe6\x84H\xf9\x01\xa4\xd5\xe9\xd3K\xbe5B\x83\x16\x19\x1bX?\x97\x1b\xe9\x99\x18\xfbs\xdc\x1b\x1f{\xc5\v\x90˻b3!#\x97\xacUk\x8b\v\xe0\x98.\xa9l\xf2\xe0\xa1S4цGg\xfe*1S\xc2\xd85\xe3Ue\xc0\xc5{\xa3\x82/\xf841\xfa5z\x8dDx\xdeF\x17O2\xd9\x04g\x83$/\xac急\xe1\xe1>\x8c\xfc\x17\x00\x00\xff\xff\t\x15i;\xcd\r\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YKs\x1c\xb7\x11\xbe\xf3Wt\xc9\a\xc6U\x9aY[I\xa5R{\x93\xa88\xc5ĦX\xa2\xa4\x8b\xcb\a\xec\xa0g\x06\xe6\f\x80\x00\x98%7.\xff\xf7T\xe3\xb1;\x0f\xec.Ɋ\x1c\\\xc8ţ\xf1\xa1\xdf\xddS\x14\xc5\x05\xd3\xe2\v\x1a+\x94\\\x03\xd3\x02\x1f\x1dJ\xfae\xcb\xfb\xbf\xd9R\xa8\xd5\xf6\xfb\x8b{!\xf9\x1a\xae\x06\xebT\xff\x11\xad\x1aL\x85\xef\xb1\x16R8\xa1\xe4E\x8f\x8eq\xe6\xd8\xfa\x02\x80I\xa9\x1c\xa3iK?\x01*%\x9dQ]\x87\xa6hP\x96\xf7\xc3\x067\x83\xe88\x1aO<]\xbd\xfd\xae\xfc\xfeM\xf9\xdd\x05\x80d=\xaeA+\xbeU\xdd\xd0\xe3\x86U\xf7\x83\xb6\xe5\x16;4\xaa\x14\xea\xc2j\xac\x88vcԠ\xd7pX\bg\xe3\xbd\x01\xf3\xad\xe2_<\x99w\x9e\x8c_\xe9\x84u\xffʭ\xfe(\xac\xf3;t7\x18\xd6-A\xf8E+d3t\xcc,\x96/\x00l\xa54\xae\xe1\x86`hV!\xbf\x00\x88O\xf4\xb0\n`\x9c{\xa6\xb1\xee\xd6\b\xe9\xd0\\\x11\x85Ĭ\x028\xda\xca\b\xed\x89\x1e\xe1\xa1E\t\xae\x15\x16\xc2k\xe1\x81Y\x82c\x9c\x7fe\xfeb\xbfNǭc\xbd\x9e \xb82\xc8\x0eG\x03\x04\xce\x1c\xe6\x00\xec\xf9\t\xaa\x06\xd7\"q\xde+\x16\x13R\xc8\xc6O\x05I\x80S\xb0A\x0f\x119\f:\x83LcUj\xc5K\x99\x88N`\xdd\xccf\xcf\xf1\x86\xf6\xff\xafQM\x00\xdd*\xfe\x02(Ϻ7l\x9e\xdc\xfae0\x96=\xff\xc6\xc40j/'X\x17\x8a\x11\x86\x0ff'$@\xe1\f\x84\x05\x16\x8f\x86W\x1c\x18\x9d\xdc\xd1ǿ\xdf}\x82t\xb5\x17Ɯ\xfb\x9e\uf1c3\xf6 \x02b\x98\x905\x995\t\xb16\xaa\xf74Qr\xad\x84t\xfeG\xd5\t\x94s\xf6\xdba\xd3\vGr\xff\xf7\x80֑\xacJ\xb8\xf2\x99\x02\xb9\xa7A\x93\xe6\xf2\x12\xae%\\\xb1\x1e\xbb+f\xf1\xab\v\x808m\vb\xec\xd3D0Nr\xe6\x9b\x03\xd7F\v)E9\"\xafY\xdeq\xa7\xb1\"\xe9\x11\x03館E\xf4P\xb52\xc0\xe6\xdb\xcb\t\xe1\xbc\xe1\xd2\xc8z\xa7\xf9\xa6\x19\xb2w\xb93\t\x9b\x1c\xf9\xd4\xe40\xc3\xce\x05Q\x80n\xeee\xf7g\fje\x85SfG\x84\x83\x83-\x17\x14\x8e\x88\x81\x86T\x1cϼ\xe3Fq\xcc\xc1\xa6\xa3\xe0Z\x16\xb4\x95\xf2+\xf2G\x83\x94\xcb[h(\xf9,`Z\xf13\xb8\xe2\x8d\f\f\xd6hPV\x98\x1cש\xe4!\x83l\x1c֗\x18\x8f+\x05\x9c\xf0\xeaY\xc4oo\xaf\x93'OL\x8c\xd8\xdd\xf2\xde3\xfc\xa1Q\v\xec\xb8\x0ft\xe7ﾼ\xae\xc3eާ9\x05\f\xb4\xc0\x90\x06\xee\x83\x04\bi\x1d2\x0e\xaa\xceR\xa4\x9a\x04\xc8\xf0\r\xc6\x13\xaf\x83\a\x8b\xae\xf2\x10Z\x88\xf7\xc0\xc8w\n\x0e\xff\xbc\xfbp\xb3\xfaG\x8e\xf5\xfbW\x00\xab*\xb4>\vvأt\xaf\xf7\x899G+\frJ\xb3\xb1\xec\x99\x145ZW\xc6;\xd0؟\xdf\xfc\x92\xe7\x1e\xc0\x0f\xca\x00>\xb2^w\xf8\x1aD\xe0\xf8\xde-'\xa5\x116\xb0cO\x11\x1e\x84k\xc5<\x98\xee9@\xea\x15\x9f\xfd\xe0\x9f\xeb\xd8=\x82\x8a\xcf\x1d\x10:q\x8fkx\xe5Ӛ\x03\xcc\xdf\xc8v~\x7fu\x84Ꟃi\xbf\xa2M\xaf\x02\xb8}\x1c\x1e\x1b\xdd\x01d\xb0<#\x9a\x06\x0fY\xd5|\xf8\xa0B\xae\xfa[P\x868 Ո\x84'L\xd2\v\x8e\x12\xf9\x02\xf4\xcfo~9\x8ax\xca/\x10\x92\xe3#\xbc\x01\x11K\x1b\xad\xf8\xb7%|\xf2ڱ\x93\x8e=\xd2MU\xab,\x1e㬒\xdd.\xe4\xb9[\x04\xab\xa8P®+B\x1e\xc4\xe1\x81\xed\x88\vIp\xa4o\f43\ue936\xa6\xec\xe7Ӈ\xf7\x1f\xd6\x01\x19)T\xe3=1E\xcdZP6CiL\x88\xc5^\x1b\x17\xc1<\r;\x04\xf5q\n\xaa\x96\xc9\x06\xc3{\x11ꁢcy\xf9\x12;^\xa6$idR\x93\xb9\xe3\xf8\xbf\x05\xf7'>\xceg\xd0Oxܸ\xca8\xf9\xb8\xfba\x83F\xa2C\xff>\xae*KO\xabP;\xbbR[4[\x81\x0f\xab\ae\xee\x85l\nR\xcd\"\xe8\x80]\xf92u\xf5\x8d\xff\xf3\xe2\xb7\xf8\x8a\xf6\xa9\x0f\x9aT\xda_\xf3Ut\x8f]\xbd\xe8Q)\x87}z\x1c\xbb\xbc\x8b\x99\xd5\xfc,\x99\xc5C+\xaa6\x15'\xd1\xc7\x1e1&A\x990\x0f\xae\x99\xc9\xddWWeb\xe8`\bѮ\x88\xbd\xb4\x82IN\xff[a\x1dͿ\x88\x83\x83x\x92\xf9~\xbe~\xff\xc7(\xf8 ^d\xabG\x12\xf00\x1e\x8b\x03\xac\xa2g\xba\b\xbb\x99S\xbd\xa8f\xbb)+\xbd\xe6\xc4\xf8Z\xa09\x93\xc6}\x9clN\x89f&\xbf\xdd\xefyV\x1e\xe9X\x93I\xdcƭ\xc3S\xe9\xddI~M\x1b7\xac\xb1\xc0\f\x02\x83\x9ei\x92\xf3=\ue290\x10h&(\x9aS\xc0\xdewE\x80i݉l\xe0\x8ea?\xa6\xac\x91\x13T\x96\xb3\xc6\x1e{{Vj\xe3.\xd0\x19)|\x1emM28Ӈrmή'ݩ%Z\x94C\xbf\x84R\xc0\xbd҂e\xe6\rZ\xb7\xd0/Zx\xb5\xccKN\b+\xf0\xf2\f\x0fb{8S\xeaDQ\x84\xbcp_\xee\xf8\x8e`\xae\x9e8^L\x1c\x85H\xf5a\x89=Z˚s\xa6\xf8S\xd8\x15\xea\xfbx\x04\xd8F\rn_\xe0O\xdc㥍:\xf5\xbc\x1eC\xb6t\x9e\xaa3\xa3\xd2\xc6\xc6\x14\xbf\xeb\xfc\x99\xb1#8|\x93\xf3\xa86\x98O\x11^\xe2\x13\x00\xfcǦs\biO\xce\xc0\xf6\xde뤅\xc1\t\xa7|\x83\x0f\x99\xd9\xc5G\xb2\xf1\xe2U2\x99\xcc\xda\x0f\xde\x1a\x9e\xf5\xfex\xd19\x16\xc4mЪ.\x19\xb3r\xac\x039\xf4\x1b4ć\xcdΡ\x9d\xba\xf3\\7\xc7W\x81\a6\x8e\xce'\xf9\x05J\xb1\xb0\xad\x98\xf4]W\xb2.\xa7\x80\v\xab;\xb6\xcb\x10N\x0f\xf1\x99\x1e\x19\x17\xb9\x80\x83>'\xa3\xd6h\xfc\xd2s\xbbP\x1e\xd3{%\x8f\xd4%ɞ\x85t\x7f\xfdˉ\xbcPH\x87\xcd,8\xc4ub\xe7;\xba\xe5\xeb\xdcp\"\x89\xb1\x92i\xdb*w\xfd\xfe\x8c\x16\xdc\xed7&k8\xa4\x8c\xde\xf7\xf9\x9ep\xdc\x14U!'\xaa\xbdoy\x96\xa9N?Ϟ\x83:\xd9|&\n\xc5\x0fù\x18t\x87\x9a\x19\xb2t\xff\x05\xe1j\xfe\x89\xeb5X\xe1ۢ\x94y\x86T44-,\x05'J\xad\x94\xc1\x8c˄eX\x99\x04\x91)\xfc?2~d\xf5d1\xe9\x91\xf3\x11\xed\xd8Z\x1f\xcf\f\x9b\xfdg\xa35\xfc\xf6\xfb\xc5\x7f\x03\x00\x00\xff\xffY\xc0\xfaX\xc0!\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Y_\x93۶\x11\x7fק\xd8q\x1e\xae\x991\xa9\xd8\xedt:z\xb3\xef\x9aε\xc9Yc\x9d\xfd\x92\xc9\x03D\xacHD$\x80\x02\xa0tj&߽\xb3\x00A\xf1\x9f\xa4\xd3M.\xe1\x8b}\xc0b\xf1\xc3\x0f\xfb\x0f\xab$IfL\x8b\xafh\xacPr\x01L\v|r(\xe9/\x9bn\xffaS\xa1\xe6\xbbw\xb3\xad\x90|\x01\xb7\xb5u\xaa\xfa\x8cV\xd5&\xc3;\xdc\b)\x9cPrV\xa1c\x9c9\xb6\x98\x010)\x95c4l\xe9O\x80LIgTY\xa2Ir\x94\xe9\xb6^\xe3\xba\x16%G\xe3\x95ǭwߥ\xefާ\xdf\xcd\x00$\xabp\x01Z\xf1\x9d*\xeb\n\rZ\xa7\f\xdat\x87%\x1a\x95\n5\xb3\x1a3R\x9e\x1bU\xeb\x05\x1c'\xc2\xe2f\xe3\x00z\xa9\xf8W\xaf\xe7s\xd0\xe3\xa7Ja\xdd\x7f&\xa7\x7f\x10\xd6y\x11]ֆ\x95\x138\xfc\xac\x152\xafKf\xc6\xf33\x00\x9b)\x8d\vx (\x9ae\xc8g\x00\xcd9=\xb4\x04\x18\xe7\x9e9V.\x8d\x90\x0e\xcd-\xa9\x88\x8c%\xc0\xd1fFh\xe7\x99i\xf5\x80ڀ+\x90\xb6\xf4\xac2!\x85\xcc\xfdP\x80\x00N\xc1\x1a\xa1A½2\x80_\xac\x92K\xe6\x8a\x05\xa4D\\\xaa\x15Oe\xd4\xd9\xc8\x04\xce\x1f\x06\xa3\xee@\xe7\xb0\xce\b\x99\x9fB\xf6;\x83\xea\xe1Y*\xfeL$\x8f\x05z\x99\x88\xa6֥b\x1c\rm^0\xc9K\x042Pp\x86I\xbbAs\x02E\\\xf6x\xd0}$_\xa2\xbe\xce\xcc5\xec\\CE\x90\xedm\xff\xb5;tiߥ\xe2\xcd\x02h\x8c\x1a\xacc\xae\xb6`\xeb\xac\x00f\xe1\x01\xf7\xf3{\xb94*7h\xed\x04\f/\x9e\xea\x82\xd9>\x8e\x95\x9fx]\x1c\x1be*\xe6\x16 \xa4\xfb\xfb\xdfNck\x16\xa5N9V~<8\xb4=\xa4\x8f\xc3ဖ\x9c-o\xae\xffO\x81\xbb&HwJ\xf6y\xfd8\x18\x9d\x02\xdbQ\x1a\xe3m\x9a\x19\xf4\xa1\xf6QTh\x1d\xabtO뇼\xaf\x8f3\x17\x06\xc2\xf4\xee]\beY\x81\x15[4\x92J\xa3\xfc\xb0\xbc\xff\xfa\xd7Uo\x18@\x1b\xa5\xd18\x11\xa3k\xf8:ɣ3\n}foHa\x90\x02NY\x03mp\x8a0\x86\xbc\xc1\x10\x9cEX0\xa8\rZ\x94!\x8f\xf4\x14\x03\t1\tj\xfd\vf.\x85\x15\x1aR\x03\xb6Pu\xe9#\xd0\x0e\x8d\x03\x83\x99ʥ\xf8_\xabے\xefѦ%s\u0604\xf8\xe3\xe7c\xb0d%\xecXY\xe3[`\x92C\xc5\x0e`\x90v\x81Zv\xf4y\x11\x9b\u008fd!Bn\xd4\x02\n\xe7\xb4]\xcc\xe7\xb9p1if\xaa\xaaj)\xdca\xee\xf3\x9fX\xd7N\x19;\xe7\xb8\xc3rnE\x9e0\x93\x15\xc2a\xe6j\x83s\xa6E\xe2\xa1K\x9f8ӊ\x7fc\x9a4kozXGN\x17>\x9f\xeb\xce\xdc\x00%;\x10\x16X\xb34\x9c\xe2Ht\fٟ\xff\xb9z\x84\xb8\xb5\xbf\x8c!\xfb\x9e\xf7\xe3B{\xbc\x02\"L\xc8\r\x05]\xbačQ\x95\u05c9\x92k%\xa4\xf3\x7fd\xa5@9\xa4\xdf\xd6\xebJ8\xba\xf7\xff\xd6h\x1d\xddU\n\xb7\xbe\x92\xa0xYk\xb2\\\x9e½\x84[Vay\xcb,\xbe\xfa\x05\x10\xd36!b\x9fw\x05\xdd\"h(\x1cX\xebL\xc4\n\xe6\xc4}\r\xab\x92\x95ƌ\xae\x8f\x18\xa4\xa5b#2\xef\x1b\x14~\x80\x8d\xe4Ӟ\xeaiץoͲm\xadWN\x19\x96\xe3\x0f*\xe8\x1c\n\r\xb0}\x9cZ\x13\xc1\xc9N\xce\v\xca\xc1\x06ɑR\x802.\xde\x17h\xb0\xbbƠVV8e\x0e\xa48d\xcbt\xa4\xe1\xc4E\xf8#+~\xe1\x18\x14\xee\xbdC\x18ܠA\x99a\x8c\x10\xe7*\x99\x89St\x12\xfa\x18\xe2i\xea\xe1L\xf4\x9c\x04\xfcay\x1f#fd\xb8\x81\xee\xc6\xfb^\xa0\x87\xbe\x8d\xc0\x92\xfb\x84ry\xef\x9b\xfbM\xd8\xcc\xc7\x0e\xa7\x80\x81\x16\x18*\xd26\x18\x83\x90\xd6!\xe3\xa06\x93\x1a\xe9m\x00\xe4`\x06\x9b\x15oC\xa4hB\xd21\x84\x13\xf5\xc0(F\t\x0e\xff^}z\x98\xffk\x8a\xf9\xf6\x14\xc0\xb2\f\xad\xf5\xf9\x1a+\x94\xeem\x9b\xb39Za\x90S\xe1\x82iŤؠui\xb3\a\x1a\xfb\xd3\xfb\x9f\xa7\xd9\x03\xf8^\x19\xc0'V\xe9\x12߂\b\x8c\xb7\xe1/ڌ\xb0\x81\x8eV#\xec\x85+\xc40i\xb5\f\x90u5\xc7\xde\xfb\xe3:\xb6EP\xcdqk\x84Rlq\x01o|%x\x84\xf9+9\xd6ooNh\xfdKp\xa07$\xf4&\x80k\xf3]\xd7#\x8f ]\xc1\x1c8#\xf2\x1c\x8f\x85\xe8\xf0\xf3\xc1\x9bBⷠ\f1 UG\x85WL\xb7\x17\xe2\x11\xf2\x11\xe8\x9f\xde\xff|\x12q\x9f/\x10\x92\xe3\x13\xbc\a!\x037Z\xf1oSx\xf4\xd6q\x90\x8e=\xd1NY\xa1,\x9ebV\xc9\xf2\x10\xaa\xfd\x1d\x82U\x15\xc2\x1e\xcb2\t\xf5\x06\x87=;\x10\v\xf1\xe2\xc8\xde\x18hf\xdcYk\x8dU\xc6㧻O\x8b\x80\x8c\f*\xf7\xf1\x8e\xb2\xd3FP\xd5@\xe5B\xc8y\xde\x1aGI3~\xb6\x0e\xe6\xe3\x14d\x05\x939\x86\xf3\"lj\xcaB\xe9\xcdK\xfcx\x9c\xfa\xe37Q\x02\f\x03ǟ\x96D\x9fy8_\xa9>\xe3pݷ\xd6\xd9\xc3m\xeb5\x1a\x89\x0e\xfd\xf9\xb8\xca,\x1d-C\xed\xec\\\xed\xd0\xec\x04\xee\xe7{e\xb6B\xe6\t\x99f\x12l\xc0\xce\xfd\x93y\xfe\x8d\xff\xe7\xc5g\xf1\xaf\xeb\xe7\x1e\xa8\xf7\xe8\x7f\xcdS\xd1>v\xfe\xa2C\xc5Z\xf1\xf9y\xecf\xd5\x140õ\xe4\x16\xfbBdE|\x0441\xf6\x843\t\xaa8y\b\xcdL\x1e^ݔ\x89\xd0\xda\x10\xa2C\xd2\xf4\xb4\x12&9\xfd\xdf\n\xebh\xfcE\f\xd6\xe2Y\xee\xfb\xe5\xfe\xee\x8f1\xf0Z\xbc\xc8WO\x14\xba\xe1{J\x8e\xb0\x92\x8a\xe9$H3\xa7*\x91\r\xa4\xa9\xf6\xbb\xe7D\xfcF\xa0\xb9P\xc5}\xee\t\xc7*t\xa2\x8ale\xae*#\xadd\xda\x16\xca\xdd\xdf]\xc0\xb1j\x05#\x86\xe3u5\xc5c\xd45h\x02]\x87\xc7\xfb\xcb\xc3\xe9@\xd2\a\u0557\x8eȔ\x11\xb9O[\xad\xef\xfbW\x84d\x15\xeb6\xff\xba_Ŵ\x162\xbf\nk\xb7\x97v\x01藎hDy\xa1\x9b\xe7\x8a)\x9c\xbd\x1e\xdf\x18-ʺ\x1aCI`\xab\xb4`\x13\xe3tG#\xfb\xa4\x897\xe3\xba\xe6\f\x13\xc1\x00.pд\x9e&\xdeQ\x8d\xfd\x84\xbaҏ\xd0\xdb\xc5[\xd1t@\xbe֮\xe8\xd9MEr\x1fa2\xfd:\x1c\xc8h\xc5gCҺ.9\x98<:\xd4p\xa2o\xab\x83\xd9^K\xb4{\x9a\xf1\xc3\xda\xf7ۮyZ\x87\x1e_\xc3{\x88\xf0.v\xfe\xe8y\xf3\xe2\xc7u\xa6\xe8\xe9\xd0k\xcf]\xb0\x81\xdb\xf1\n\xdf\xc92\xbc\xf1\tQ\xa1\x7f\xb1\x86\xf6\xe4\x9eٸ\xc9\xd4}CG_X\xea\xb3*\xa9C\xee\v{zwl\x98(\x91C\xfb+\x8bo\xa5[\xdfҹ\x99\xaac\xa3\xa2\xda\"\xf7qc\x02\xf4x]\xec\x92r\xe60!\x15#\tY\x97%[\x97\xb8\x00g\xea\xf1\xf4\x19\xf7\xaa\xd0Z\x96_\xf2\xaf\x1f\x83Tx\xf37K\x80\xadU\xed\xdaG\x7f\xe3h\r\x157\xb6\xb1\x82\xeb\x1a\x0f\x05\xb3\x97\xa0,If\xca\xe2Z\x97?orp&\x94=\xe0~btԵ\xeeN\xdeF\x13\x9a\x98\xfb\xde[\xc7U\x044\x1b]\xe2\xa0\x11\x83B\x95Ѻ\x95\xa3\xa4TWk4D\x84o\x95GFb\xe0\x98\xea\xa2\xf8\xd7בɣ\x86\x18\v\x83\xaa\xe6=\x991雊d\xbfN\x01\x17V\x97\xec0\xa17\x9e\xc4\x17Xd\xbe\xe4GG\x8b\x89^H\xee\xef\xe7\xae\xed\xfe\xb4?\x05L\x97\x7fS?,L\xddB\xf7W\x82\xc1|\xfb\x1b\xc8\xeb\xecp\xa6䳎\x19\xf7ܰ\xb7\xea\t_\x8ax^\xf5t\xbc놮q\xa0\xeao\xf3GƨI\xa2F\x83\x1e9\xef\xe8n:\xa7ݑz\xdd\xfe.\xb0\x80_\x7f\x9b\xfd?\x00\x00\xff\xffg\b\x17r\xc1\x1f\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xdc]O\x93\xdb:r\xbfϧ@9\a'U#y\x9d\\Rs\x9b\xf8\xd9Ye\xf7\xd9Sc\x97\xdf\x19\"[\x12vH\x80\x0f\x005VR\xf9\xee\xa9n\x80\xe0\x1f\x81$\xa8\x19\xbd\xf5\x06\xb7\xa1\x80\x06\xd0\xddht7~\xc0\xacV\xab\x1b^\x89\uf80dP\xf2\x8e\xf1J\xc0\x0f\v\x12\xff2\xeb\xa7\x7f7k\xa1\xde\x1d\xdf\xdf<\t\x99߱\x0f\xb5\xb1\xaa|\x04\xa3j\x9d\xc1/\xb0\x13RX\xa1\xe4M\t\x96\xe7\xdc\xf2\xbb\x1bƸ\x94\xcar\xfcl\xf0O\xc62%\xadVE\x01z\xb5\a\xb9~\xaa\xb7\xb0\xadE\x91\x83&\xe2M\xd7\xc7?\xad\xdf\xff\xeb\xfaO7\x8cI^\xc2\x1d\xd3`\xac\xd2`\xd6G(@\xab\xb5P7\xa6\x82\fi\ued6a\xab;\xd6\xfe\xe0\xda\xf8\xfe\xdcX\x1f]s\xfaR\bc\xff\xd2\xfd\xfaWa,\xfdR\x15\xb5\xe6E\xdb\x19}4B\xee\xeb\x82\xeb\xf0\xf9\x861\x93\xa9\n\xee\xd8g\xec\xa6\xe2\x19\xe47\x8c\xf9\xa1S\xb7+?\xea\xe3{G\";@\xc9\xddx\x18S\x15\xc8\xfb\x87\xcd\xf7\x7f\xfb\xda\xfb\xccX\x0e&Ӣ\xb2\xc4\x00?6&\f\xe3\xec;\xcd\r\a@\xbcf\xf6\xc0-\xd3Pi0 \xada\xf6\x00\x8cWU!2bu\xa0ȘڅV\x86\xed\xb4*[j[\x9e=\xd5\x15\xb3\x8aqf\xb9ރe\x7f\xa9\xb7\xa0%X0,+jcA\xaf\x03\xadJ\xab\n\xb4\x15\rc]\xe9\xa8K\xe7\xeb`.oq\xba\xae\x16\xcbQO\xc0\rٳ\fr\xcf!\x1c\xad=\b\xd3Nm8\x1d?%.\x99\xda\xfe\r2\xbbf_A#\x19f\x0e\xaa.rT\xaf#hdN\xa6\xf6R\xfcw\xa0mp\xa2\xd8i\xc1-xy\xb7EH\vZ\xf2\x82\x1dyQ\xc3-\xe32g%?1\r\xd8\v\xabe\x87\x1eU1k\xf6+\x89G\xee\xd4\x1d;X[\x99\xbbw\xef\xf6\xc26\xcb$SeYKaO\xefH\xe3Ŷ\xb6J\x9bw9\x1c\xa1xg\xc4~\xc5uv\x10\x162[kx\xc7+\xb1\xa2\xa1KZ*\xeb2\xff\xa7 \xb6\xb7\xbd\xb1\xda\x13j\x9e\xb1Z\xc8}\xe7\aR\xf3\t\t\xa0\xc2;]rM\xdd,ZF\xe3'\xe4\xce\xe3ǯߺz&̐\xfb\xc4\xf7\x8e\xf2\xb5\"@\x86\t\xb9\x03\xed\x84Hچ4A\xe6\x95\x12\xd2\xd2\x1fY!@\x0e\xd9o\xeam),\xca\xfd\xf7\x1a\f*\xb4Z\xb3\x0fd;\xd8\x16X]\xe5\xdcB\xbef\x1b\xc9>\xf0\x12\x8a\x0f\xdc\xc0\xd5\x05\x80\x9c6+dl\x9a\b\xbafoX\xd9q\xad\xf3Cc\xbcF\xe4\xe5W\xff\xd7\n\xb2ފ\xc1fb\xe7\x979\xdb)\xdd3\x0e\xd8d\xdd#\x1a_\xb4X\xdc\xeaG\v6\xfce0\x94\xff\b\x15Q\x7fp\x10\xb5\x14\xbf\xd7@&έX83)g$Y3>R\x8b\xf5\xd9\xef#<\xc5\x02?\xb2\xa2\xce!\x0f\xd6\xf6l.\x83\x11\x7f\xe7d\n?]I\xb1\x95#|\x8cX;\xfa\xb7\x13\x9b \xc9P͟\x0f\";8'\x00u\x93\xe8\xb0\\\x81!Á\x8e\xeail\x92lN\xf6\xbe\x93)\xd3і\x9955\xa4\x173'mI0\xb7m\x991\xbcg\x86\xc5\x7f\x8f\xee\x9cm\xf9\xff\xc9\xd8f'\xb9@i7gM_Wi)\xa8Bg\x7f\xb3cPV\xf6t˄m\xbe\xceQ\xe4E\xd1\xe9\xff\x1fX0\xcb5~3l\xf9\xaa\x1a?)\x959\x8a(\x95\xd0\xfd?\xa0Ph\xb3\xf8\xea\xf7\x8ad\x81\xfc\xb5\xdbꖉ]\x10H~\xcbv\xa2\xb0\xa0\a\x92y\xd1zy\rf\xa4\xecwXJn\xb3\xc3\xc7\x1f\xe8٘6ϔȗac\xe7\x1271B\x7fc\x9e\xa1\xcb(|\x15\x1aJ\x17\x16\x7f#n\xb6_(\x9e\xb8\xff\xfc\v\xe4S\xecai\x9aw6\x91\xfb\xc1`\xbb]{??u\x1a\xde\xf5\t1\x93Kx\xdc2Ξ\xe0\xe4<\x16.\x19\n\x87[\xf2w\xa3\xd1\xd39s(\xf3BJ\xf6\x04'\"\xe3S)\xb3\xadSU\xc1\x95'\x88\xb8\xfb\xb1\xd2c \x8e\xc9\a\xb8\x8e\x93\xf8\x81\x18A\x81w:\xf3\x18\xa5\xc5\x1a[4?9\x96nH\x9a\xd2\xf0\xfe\x82i\x06\xb1u҇$طƉ\bW\xc1AT\x89\x13\xa5\xec\xa1\x01Z-Mb\xec;/D\x1e:rz\xbf\x91\xe3\xdep\xbf|Vv#o]DfHK~Q`>+K_\xae\xc2N7\xf0\v\x98\xe9\x1a\xd2\xf2\x92\xcel#\x1f\xba\x19\xb6\x04\xe5ve\xe3\x12)A<°\x8d\xc4\xc0\xc5\xf3\x83\U000a5bbb\xe9\xfd\xa1_\xca\xdaP\nM*\xb9\xa2\xadr\x1d\xeb\xc91;\x91\xa4\xd2=\x89\x9c\x0f-t\xea:L$\xfb\rw\x12\xd7\xdee\x80\v\x9eA\xdeD\x9b\x94\xb7\xe4\x16\xf6\"c%\xe8\xfd\xd4\xc6\xd1-\x15\xda\xf7\xb4!$Z]W\x16jX\xda\xd6\xde\x14o\xba\xf3\xf9\xc1\xacp\xe5&\xd4j\x84=[u$]9^u~F\xb4Œ\xff1\xcb]\x9e\xe7t\x84ċ\x87\x05\x16\x7f\x81,\xce\xf7~70\xb7C\x96\xbc\xc2\xf5\xfb?\xb8͑B\xff/\xab\xb8\xd0\tk\xf8\x9e\x8e\x89\n\xe8\xb5\xf5\x89\xb1n7\u06030\f\xe5{\xe4\xc5y\"<29\x85\xb6\x05\n\xb7\x91\xabݙ\xc7r˞\x0fʸ=u'\xa0\x88\xa5l\xfaE\x18\xf6\xe6\tNon\xcf\xec\xc0\x9b\x8d|\xe36\xf8\xc5\xe6&x\vJ\x16'\xf6\x86ھy\x89\x13\x94\xa8\x89\x89\xd5~\xac\x9eBJnU\xf2j\xe5\xb5תRd\xa3\xedd4=ޖ\x9e:uS\xe4mnܻ\xc7S\xb3M\xd2\xdfJ\x19\xfb\xe7x\xa2od<\x0fM\x8b\xbeO\x1bɗ\xcd\xfa\xfa>\xf7\x15\x8c1z\x80;\v\xda'\xff\x9c\x81n\"\x87\x17\xc6Tsɽ\x90\xd8\xe3!!\x8b\f\x9e\xd1&wT\x922\xc4%\xde&\xf2e\xa1\x9f\xfe\xf1G'7\x89+\x1b\xff\xeeN䵽\xe1L\x95%\x1f\x1e\x0e&\r\xf5\x83k\xd9\xe8\xb4'䤯\xf75\xad\xe7t7\xb1\xd1!:\x16|\x16\xf6 $\xe3\x8d\xd9\x00\xed\x15\x8a\xb3J\xcd[0W\x0eܰ-\x80\f9\xf5\x9fa\x9f/\x85\xdcP\a\xec\xfd\xab\xfb\x05\xace\xd7E\xe2lX\x1d\x04\x1a>\xd0N\x95\xeaR\xa9\x9c=\x1f@CO+\xce\x13\xe5\xe8i&\x92\x94\xcav\xf3\x11H\xb7R\xf9[\xc3vB\x1b\xdb\x1dh\xaa\xc2\xd5&U\x1d\x16J\x18g\xf7M\x94\xa0j{\x81\f>\xb6\xad{\xe7\xba%\xff!ʺd\xbcTu\x82S\xe0\n\xee/\xa2\f\x87\xaf^\x02\xcf\\\xd8p\x0eE\x99\x19\xabPJU\x016U\xc4[ء9ʔ4\"\a݀\x03\x9cd\x85\u0085\xbb㢨c\xc7>\xb1\xb24\xbc\x95\x1f\xb5\xbe(\xba\xfd\xe2Zv\xb2\x8d\a\xf5\xdcgP2\v\x0e\xfc\bL옰\fd\x86r\x01\xedL6u\xe1\x99A\xacIV\xcb4\x03\x8f\x05d]\xa61`E+[\xc8\xc9dZ\xb7\xfa'.\x8ak\x88\r5\xef\x93ҏ\xc0\xf3K\x120\xbfu\x9a3\x90\xa6\xd6tp\xef\xcc˳(\xd2ƌ\x92c\x05\xafev\x00\xb2S\xb2g>\x98#/\xa4\xb1\xc0Su\x01\xbd\xa6ZJ!\xf7i\xb2KNq\xb6űz\xabT\x01|\bx\x8a\x15\xe4\xf5\xe5f跶\xf5\x1fb\x86\x82\x04\xd2݅-xQy[ĭ\x85\xb2r\xebM1]\xcb\xee\xees\x05+\xb4$\x06\xf7\xa3x\xcd\xe0ZH\x91 \xd8\xc1\x99\x8b\xb0]\xcf\x12I\\ճ\xc4\x0e\x82SqI\xfal\xd3#\x80\xab\xb3\tRh\xecAk\x16x\x99[`<\xcf!w\x89ItU|\xcc\xe2\xe0e#P\x85\xe8얻\x89I\x92mJ/\"\xa5T\xac>ª\x96OR=\xcb\x15E\xf2f\xb1\x01IO\r\xbej\xf7\xf6bK\xf4GZ\xa1\xbe\xbe\xa6\xebT\xe3<]\xc1\xca$\xeb͢lȔ\x16\xcc\xd95\a]\x1e\xf9qv\x14S\xfdO4\xf6\a\xcd\x1f\x1c\xe68\x15϶\x89\xb7\xea8\x7f\xcf\a\xb0\a\xd0\r\x98yE\xb8혝nϣ\xdb8&\x00\xdcP\x7f\x1aW\xd8\x01/\a\x90\xb7x\xa0\x83^\xc0-*6\xaf\v\xeb\xe0Ǻ\x8e(Q\x12\xf0+\xee\x19\xa4 '\xe6\xf0\x12}\f`\xc0+4 @\xd5t\x12\x99\xa1\x93\xa5C\xfav\x0f\xe3\xfb\xc0\aJ\xf95#\xfd\xbb\xc3\x03\x130\r3H\x86i\xd0\xe4\x14\xbf\xceզ˱V\a}=\x8f\xa6\xfd\xb9\xd8g\xa1\xfcR\xf9u0\xea\x80\xf69\x18i2\x80\x83\x90\xe5Ɛ\x9d\x80\x05\\Č\v\xaeB\x9f\x0eD\x8a\xf7\x19\xadD\xd5\x106\x94j\xf6\xabͣۅa\xef\xd9A\xd5\x11H\xdd\x04wf\x00\x16\xe3\xb0\n\x7f\x88\x00\x96\x1f߯\xfb\xbfX\xe5A\x16\x94\xf9\x8a̎\x02\x956\x9b*d.\x8e\"\xafy\xd1[d\x1d\xb5h\xb5\x87)ͤ(b竨VM\xfb\x9e\x1a\xb1/\x95;gYl\x8e\xa6]\xc44,\xc6\xc5\b\x8c>\xc2bd\x93Zz\xe4\x90\x0e5M\xc7XL\x83\"\x96 +\x86\xb8\x89Q\xa2\xf3x\x8a\x14\xef~\x06;q\x01b\"\x11-\xf7\xe2\x03\x92\x14L\xc4EH\x88Y@Y\"\xfe\xa1\x8fl\x98&\xb9\x00\xf5\x90Ĝy\x84\xc3b\\\x83\xc7\x11L\xce#\x19\xcd\x10\xc1)L\x12\x1e\xc50L\xa1\x13\xa6Y\x1eA.\xa4c\x12&I\x13^a\x1e\x89\xf0zx\xc3\u05c8\x02\xc6M\xcd,\x9a\xe0EQB\x02^`\tJ`\x96c\x17\"\x02\u0089\xffH\xbfKq\x00\xfds\xfe\x11\xa2)\xa7\xff#\xa7\xfb#\x14'\xcf\xfcS\xcf\xf4Gh\xcfl\xbb\x93Z2\xf9㒳\xfc\x10\x86\xfcʫJ\xc8\xfd\xb9\x9e\xa4jӤ&\x9d\x01\x01\xba}\xf6T\xa9\x1b-\xf4\xe2\xacX\x97\xeeVn$&k\xd2zBZ\xb5f\xf7\xf2tF\x97\xae\x04Dc\x90\xfe\xb5-\x1cֳ(\x8a\xee\xdd$\"\xdb%\xe5o\xf9\x99xf\x00+\x8ey\xd8Q\x11*\xdd\xf3\x8e\xe7B\xb0/\x83\xea\xddDᴷ\x1ds\xb4\x85=\\\xe8m\x97uaE\x15]\xf2\x95VGAi\xc7\x03\x9c\x02?\xff\xa6\xe8V\xd0\xf6D\x94\xbe<\x86ո\x1e\x04\x0e<\xb6\x86\x9e\xa1(\x187\xe7\xd3\xcf\xdc\xc5\xd8L\xad\xe8\xae\x1bJ\xb2\xd1\a\x7f\x81\xf6\x96Vl,b\x97͕\xcd\x12\xc9\xd0\xe5Z\x13Ɉ\x8c\xeeE\xd3\xfe\xb0s\xdd\xe9\xdb\xef5\xe8\x13SG:\xd3\xf7\x0e\xd2\f\xec\xde\xd9\x15\x83\xf1[c鼹tױ\aqBk_ؽt;v\x94\xec`\x8cD\aM\\\x1b\x1b\xa15ǰg\xa4j\x94\xaaT\xa1u\\\x1f&7\xa6T\xcc\xfau#\xa5\xe5\xb1Ҭ\x97r\x95x\xe9\xf2\x88i\x82d*\x06=\xedLd\x16s~\xad\xc8i.vJv\x1a\xd30\xe5\xd7\xc0\x92/\xc0\x90/\x88\xa1\x96EQ\xc9lJ\xc1\x8a_%\x96\xbab4u\x8dx겈j\x86\xe4\x00\x03\x9e\x82\xeeN:\xc6K>\xb3I9e\x9b?9\x9eFm'\xa0\xb5\x13N\x83\xe6F\x9a\x80\xca^\x86\xc6N\xe0\xe1\x95b\xad+E[\u05c8\xb7\xae\x1bq\xcd\xc6\\\xb3\x9a3\xf3\xf32\x14\xf5Ň\f\xcdq\xf4g\x95Ã\xd2v.@x\x18֏\x1c\x01v\x82&U\xe4L6Uc'\r\xe8\xfb{\xbf\xff\xb2I\xc5O\xeb\x1a\xf7\xf7W\x95\xe3\xd8\xe6\xce\x16\x1e\a\xd5Ϯ\xd0\xee@\x83t\x0fK\xfc\xd7\xd7/\x9f\x03\xfd\x98?\xea\x9d\xde\xc1\x9b\x06\xce\xc1\xc8=s\xfc\xe9\x93\a\xdc8n\xd1\x1e\xfeʇ\x04\xbc\x12\xffIov\xcd'd\xee\x1f6T\xb5\xf1\x96譯p\xa0\x1f\xce\u07b6\x80\xbbG\xe0Ȩ\xf6ov=\x8a\x11\xd8i\xf8\x93ыI\xcd\xee%\xc60Y\x0e\x84\x84\xab\xeea\xe3F\xb7f\x9f\xd0u\x93'\xa6\x9c\xe2\x1d\x84\xceW\x15\xd7\xf6D\xdaan\xc3\x18Ɠ2\xcd\x1e2\x95:\x195\xb5\xe7oAEy\xdb<\tE\ap\xa7\xaa\x7f\x9a9\xe4\xe8%\xe3\x18\xbf=1{o\xe2\x15\xc71\xbe\x1d\xaf\x88S\x91\xcfQ\x04ī\xa5\xa4\xbc\x19z\xf8>g\xd6\x1eC\xc5i{\x86\x91l\x93։\xf0\aۓI3\x92W\xe6\x10\xc9\n\xbd̦\xd1CU\x96\xdb:q>\xaenoJ\";t\f\xd034&Jw\x9e\xed\x1b\x8c\tת#D\xdb0\xb9\xd0R\x14\xb7\x9d\xc0\xfc\x8f9\xf2L|\x16\xe4\xe2\aA\x1c{FL\x05e\x9aЌ5\xba\xd0\xf2\xe5\x82\xc3\xceY\x17.\x01\xd8:\xedw&\xbe(q\xf1[\x12\xf3̊0j\xec\x19\x89\x94\xa7\"\xfe\xae\xfc\x9c0I&;@^\x17\x90\xf0\xc0\xdb\xd7N\xd5\xf9'\xde\x1a±5\xa9\xfa\x8f\xbc!_;\xdb+:\xbc\xfd\xc7\xe4<\xd3=\xe5\x11\x88w\x97\xa4\xf3\xecݫS\x19\xfa\xf1\xa6\xce20fW\x17\rZ&\xd3\xc0-\xe4M\xf5(2\xbf\x99\xc3\x02XH|\x17Yu\x9eѻI\x90\x8c\x89\x98\xc9\t\x13\x99\xf1\xca\x12\n\x9e\xbc\x8cZk\x9a\xb2\xfbM\xedΞ\xfe\xeb\x91\x1d7Z\x1e\xce\xe8\xc18\xc6\xf22\xe2\x89\r/\x82\r[\xd0\x03\x9b:\xef\xc0w\xba/\xa4\x05TN,\xad\xcdM@T\xe6\xeb\x0emG\x86\x9c\x1f$\r9\x83#H\xa6$\xdd5\x81|\n\xbe\xfb\x8d\xd2f\xfa\b\xfa\xad\tt\bP\x84\xbe\xe2W˵\rC?\u05c8\x9d\xd2%\xb7w,\xe7\x16V\xd8\xfa\xb2\x1d2\xfez\xa1\xd6\xf3'\x1ctiŇ\xc1tӄ\xc4[\x14\xfe\xaaI\t\xc6\xf0}\xe3\xbe?\x83\x06\xb6\a\x89,\x9ez\xa0\xad\xbd\xad\xe3Wp\xc0\x9d!\xb7xfk\xee;p;e8\xfb\x89\x9d\x1b\xb8W?)\"؏\xae\x1b!-\xec\xcfN]\xfcM\xa1G\xe0f\xf8J\xec\x19#>u\xeb\xfa\x9c\x99\xe3\x81{\x92\x84;\x90\x18=*jE\x88RF\xac\x11\xf6\xbc\b\xfaU\x1d\xb8\x993\x97\x0fX'\\\xa1\xeb,\xca`)\x1fG\xc6\x14\xbfҳb\x9f\xe19\xf2\xf5\x13)=\xe5A\xe3Ki\xc56\xf2A\xab\xbd\x06s\xae\xd2+\xba\xe3!\xe4\xfe\x93\xd2\x0fE\xbd\x172@\xf0\x96U~\xe0\xda\n^\x14'7\x9eH\xdb\x0f\xcdb\x8e\xfc6\xdfz\xe4\x87)!\xf99\xcf&\x05\\\xb56\xa7\"\xa4[\xe8t\x81m\xabj\xdb]\x15oM\xbb`b\x01\xb4\xa7\xb6f\x9f\x95\x85&W/\xfaD\x05\x06\xcfƮ`\xb7Sں\x1c\xcej\xc5\xc4\xce\x1b\xeaX\xae\x81\x8b\x82|\r\xf7\xc6-: \x01]\x12v>\x1fOjZ\x15䤔\xfc\xe4\xc2R\x9ee5ځw\xc6\xf2؆\xf6\"ז\x9c\x1b\xaf\xcd)\x11\xe5\xa6[?\x84tu\xb9\x05M\x97:\xf0g\xc7:\xba[\xe7LP\xf4\x9c\x92\xd1=\xae\xce\xd5^fp9\xc7\xd3jSƇ~W\x96\x17\x9bqG\xad\x7f!!T\x0e\xb11~9\x9fF\xef5\xcfq,\xa20MS\x94Yv\xe0r\x8f\xea\xa3U\xbd?4*8f\xa9\xc7Ҩ5\xe5|*Z\xa9\xa69\U000f2d56\x9d\x94\xad?\x05\xcb\xdb\xe1N\x11\x9dfᄟ\xa9[Dnk3\xee\xdd]\xad\x98\xceļ\x9d\x91\xc6#\xfc\x8f%\x94B\x13nN2\x9b\x86\t\xbb䑘\xb8\r4Ō\xe8|\x83\x05\xbcd\xbe\xa1q\xfa|[\xaf\xb78\xb5\xbeԒɏ\xfbٯ\xc0\x0eg\xd2/\xe1\x85k9\xb6\xf0h~\x91\x91/\x12\xb7\xcf6\x80D\a\x93\xc0 \xd1\xfb\x96\xe4t,\xe3\x85\xe9y\x99sAW\xaf\xf2˼i\xea\x18}\xe9\x9f\xd7\v>\x067\xe6c\x8a?\xfc}P}p\xe7\x02=㖢\xf7a#\xcc\xf9g\xb1k\xfe-¶\x80\x7f9\xab\xf1\aߝx\xe6Z\n\xb9\x9f\x9b\xfco\xbeZ$\x1c\xf0\x14\"\x01Ad\x12!DX\x14\x104\x83\x1cy\xf9;\x04\t/\b\t\xa2\xdb\xc9\xd9GR\xe4\xbc\xc3dߓ\xff\xf2\x7f\x01\x00\x00\xff\xffT\xf5\x7f\x80\xacd\x00\x00"), diff --git a/design/pv_backup_info.md b/design/pv_backup_info.md index 771f8255c2..107305fe5b 100644 --- a/design/pv_backup_info.md +++ b/design/pv_backup_info.md @@ -61,7 +61,7 @@ type VolumeInfo struct { // CSISnapshotInfo is used for displaying the CSI snapshot status type CSISnapshotInfo struct { SnapshotHandle string // It's the storage provider's snapshot ID for CSI. - Size int64 // The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot. + Size int64 // The snapshot corresponding volume size. Driver string // The name of the CSI driver. VSCName string // The name of the VolumeSnapshotContent. @@ -70,7 +70,7 @@ type CSISnapshotInfo struct { // SnapshotDataMovementInfo is used for displaying the snapshot data mover status. type SnapshotDataMovementInfo struct { DataMover string // The data mover used by the backup. The valid values are `velero` and ``(equals to `velero`). - UploaderType string // The type of the uploader that uploads the snapshot data. The valid values are `kopia` and `restic`. It's useful for file-system backup and snapshot data mover. + UploaderType string // The type of the uploader that uploads the snapshot data. The valid values are `kopia` and `restic`. RetainedSnapshot string // The name or ID of the snapshot associated object(SAO). SAO is used to support local snapshots for the snapshot data mover, e.g. it could be a VolumeSnapshot for CSI snapshot data moign/pv_backup_info. SnapshotHandle string // It's the filesystem repository's snapshot ID. @@ -79,7 +79,6 @@ type SnapshotDataMovementInfo struct { // VeleroNativeSnapshotInfo is used for displaying the Velero native snapshot status. type VeleroNativeSnapshotInfo struct { SnapshotHandle string // It's the storage provider's snapshot ID for the Velero-native snapshot. - Size int64 // The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot. VolumeType string // The cloud provider snapshot volume type. VolumeAZ string // The cloud provider snapshot volume's availability zones. @@ -89,11 +88,12 @@ type VeleroNativeSnapshotInfo struct { // PodVolumeBackupInfo is used for displaying the PodVolumeBackup snapshot status. type PodVolumeBackupInfo struct { SnapshotHandle string // It's the file-system uploader's snapshot ID for PodVolumeBackup. - Size int64 // The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot. + Size int64 // The snapshot corresponding volume size. - UploaderType string // The type of the uploader that uploads the data. The valid values are `kopia` and `restic`. It's useful for file-system backup and snapshot data mover. + UploaderType string // The type of the uploader that uploads the data. The valid values are `kopia` and `restic`. VolumeName string // The PVC's corresponding volume name used by Pod: https://github.com/kubernetes/kubernetes/blob/e4b74dd12fa8cb63c174091d5536a10b8ec19d34/pkg/apis/core/types.go#L48 - PodName string // The Pod name mounting this PVC. The format should be /. + PodName string // The Pod name mounting this PVC. + PodNamespace string // The Pod namespace. NodeName string // The PVB-taken k8s node's name. } diff --git a/pkg/apis/velero/v1/download_request_types.go b/pkg/apis/velero/v1/download_request_types.go index 7c7be53373..07f44b38d5 100644 --- a/pkg/apis/velero/v1/download_request_types.go +++ b/pkg/apis/velero/v1/download_request_types.go @@ -25,7 +25,7 @@ type DownloadRequestSpec struct { } // DownloadTargetKind represents what type of file to download. -// +kubebuilder:validation:Enum=BackupLog;BackupContents;BackupVolumeSnapshots;BackupItemOperations;BackupResourceList;BackupResults;RestoreLog;RestoreResults;RestoreResourceList;RestoreItemOperations;CSIBackupVolumeSnapshots;CSIBackupVolumeSnapshotContents +// +kubebuilder:validation:Enum=BackupLog;BackupContents;BackupVolumeSnapshots;BackupItemOperations;BackupResourceList;BackupResults;RestoreLog;RestoreResults;RestoreResourceList;RestoreItemOperations;CSIBackupVolumeSnapshots;CSIBackupVolumeSnapshotContents;BackupVolumeInfos type DownloadTargetKind string const ( @@ -41,6 +41,7 @@ const ( DownloadTargetKindRestoreItemOperations DownloadTargetKind = "RestoreItemOperations" DownloadTargetKindCSIBackupVolumeSnapshots DownloadTargetKind = "CSIBackupVolumeSnapshots" DownloadTargetKindCSIBackupVolumeSnapshotContents DownloadTargetKind = "CSIBackupVolumeSnapshotContents" + DownloadTargetKindBackupVolumeInfos DownloadTargetKind = "BackupVolumeInfos" ) // DownloadTarget is the specification for what kind of file to download, and the name of the diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index e9c15dbc46..6760d627b1 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -71,6 +71,7 @@ func TestBackedUpItemsMatchesTarballContents(t *testing.T) { req := &Request{ Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), + PVMap: map[string]PvcPvInfo{}, } backupFile := bytes.NewBuffer([]byte{}) @@ -84,8 +85,8 @@ func TestBackedUpItemsMatchesTarballContents(t *testing.T) { builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( - builder.ForPersistentVolume("bar").Result(), - builder.ForPersistentVolume("baz").Result(), + builder.ForPersistentVolume("bar").ClaimRef("foo", "pvc1").Result(), + builder.ForPersistentVolume("baz").ClaimRef("bar", "pvc2").Result(), ), } for _, resource := range apiResources { diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 61f6834d60..ae8074521b 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -250,6 +250,10 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti namespace = metadata.GetNamespace() if groupResource == kuberesource.PersistentVolumes { + if err := ib.addVolumeInfo(obj, log); err != nil { + backupErrs = append(backupErrs, err) + } + if err := ib.takePVSnapshot(obj, log); err != nil { backupErrs = append(backupErrs, err) } @@ -685,6 +689,39 @@ func (ib *itemBackupper) unTrackSkippedPV(obj runtime.Unstructured, groupResourc } } +func (ib *itemBackupper) addVolumeInfo(obj runtime.Unstructured, log logrus.FieldLogger) error { + pv := new(corev1api.PersistentVolume) + err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pv) + if err != nil { + log.WithError(err).Warnf("Fail to convert PV") + return err + } + + if ib.backupRequest.PVMap == nil { + ib.backupRequest.PVMap = make(map[string]PvcPvInfo) + } + + pvcName := "" + pvcNamespace := "" + if pv.Spec.ClaimRef != nil { + pvcName = pv.Spec.ClaimRef.Name + pvcNamespace = pv.Spec.ClaimRef.Namespace + + ib.backupRequest.PVMap[pvcNamespace+"/"+pvcName] = PvcPvInfo{ + PVCName: pvcName, + PVCNamespace: pvcNamespace, + PV: *pv, + } + } + + ib.backupRequest.PVMap[pv.Name] = PvcPvInfo{ + PVCName: pvcName, + PVCNamespace: pvcNamespace, + PV: *pv, + } + return nil +} + // convert the input object to PV/PVC and get the PV name func getPVName(obj runtime.Unstructured, groupResource schema.GroupResource) (string, error) { if groupResource == kuberesource.PersistentVolumes { diff --git a/pkg/backup/item_backupper_test.go b/pkg/backup/item_backupper_test.go index 7bd7548bc7..481092a0ca 100644 --- a/pkg/backup/item_backupper_test.go +++ b/pkg/backup/item_backupper_test.go @@ -19,10 +19,12 @@ package backup import ( "testing" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime/schema" "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/volume" "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" @@ -237,3 +239,55 @@ func TestRandom(t *testing.T) { err2 := runtime.DefaultUnstructuredConverter.FromUnstructured(o, pvc) t.Logf("err1: %v, err2: %v", err1, err2) } + +func TestAddVolumeInfo(t *testing.T) { + tests := []struct { + name string + pv *corev1api.PersistentVolume + expectedVolumeInfo map[string]PvcPvInfo + }{ + { + name: "PV has ClaimRef", + pv: builder.ForPersistentVolume("testPV").ClaimRef("testNS", "testPVC").Result(), + expectedVolumeInfo: map[string]PvcPvInfo{ + "testPV": { + PVCName: "testPVC", + PVCNamespace: "testNS", + PV: *builder.ForPersistentVolume("testPV").ClaimRef("testNS", "testPVC").Result(), + }, + "testNS/testPVC": { + PVCName: "testPVC", + PVCNamespace: "testNS", + PV: *builder.ForPersistentVolume("testPV").ClaimRef("testNS", "testPVC").Result(), + }, + }, + }, + { + name: "PV has no ClaimRef", + pv: builder.ForPersistentVolume("testPV").Result(), + expectedVolumeInfo: map[string]PvcPvInfo{ + "testPV": { + PVCName: "", + PVCNamespace: "", + PV: *builder.ForPersistentVolume("testPV").Result(), + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ib := itemBackupper{} + ib.backupRequest = new(Request) + ib.backupRequest.VolumeInfos.VolumeInfos = make([]volume.VolumeInfo, 0) + + pvObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pv) + require.NoError(t, err) + logger := logrus.StandardLogger() + + err = ib.addVolumeInfo(&unstructured.Unstructured{Object: pvObj}, logger) + require.NoError(t, err) + require.Equal(t, tc.expectedVolumeInfo, ib.backupRequest.PVMap) + }) + } +} diff --git a/pkg/backup/pv_skip_tracker.go b/pkg/backup/pv_skip_tracker.go index 859456a374..64241a2406 100644 --- a/pkg/backup/pv_skip_tracker.go +++ b/pkg/backup/pv_skip_tracker.go @@ -10,6 +10,14 @@ type SkippedPV struct { Reasons []PVSkipReason `json:"reasons"` } +func (s *SkippedPV) SerializeSkipReasons() string { + ret := "" + for _, reason := range s.Reasons { + ret = ret + reason.Approach + ": " + reason.Reason + ";" + } + return ret +} + type PVSkipReason struct { Approach string `json:"approach"` Reason string `json:"reason"` diff --git a/pkg/backup/pv_skip_tracker_test.go b/pkg/backup/pv_skip_tracker_test.go index 9fdcb034f5..16de8f555c 100644 --- a/pkg/backup/pv_skip_tracker_test.go +++ b/pkg/backup/pv_skip_tracker_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSummary(t *testing.T) { @@ -41,3 +42,14 @@ func TestSummary(t *testing.T) { } assert.Equal(t, expected, tracker.Summary()) } + +func TestSerializeSkipReasons(t *testing.T) { + tracker := NewSkipPVTracker() + //tracker.Track("pv5", "", "skipped due to policy") + tracker.Track("pv3", podVolumeApproach, "it's set to opt-out") + tracker.Track("pv3", csiSnapshotApproach, "not applicable for CSI ") + + for _, skippedPV := range tracker.Summary() { + require.Equal(t, "csiSnapshot: not applicable for CSI ;podvolume: it's set to opt-out;", skippedPV.SerializeSkipReasons()) + } +} diff --git a/pkg/backup/request.go b/pkg/backup/request.go index 44bc5578f2..6735c23a28 100644 --- a/pkg/backup/request.go +++ b/pkg/backup/request.go @@ -20,6 +20,8 @@ import ( "fmt" "sort" + corev1api "k8s.io/api/core/v1" + "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/resourcepolicies" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -52,6 +54,16 @@ type Request struct { itemOperationsList *[]*itemoperation.BackupOperation ResPolicies *resourcepolicies.Policies SkippedPVTracker *skipPVTracker + // A map contains the backup-included PV detail content. + // The key is PV name or PVC name(The format is PVC-namespace/PVC-name) + PVMap map[string]PvcPvInfo + VolumeInfos volume.VolumeInfos +} + +type PvcPvInfo struct { + PVCName string + PVCNamespace string + PV corev1api.PersistentVolume } // GetItemOperationsList returns ItemOperationsList, initializing it if necessary diff --git a/pkg/backup/snapshots.go b/pkg/backup/snapshots.go index a5c6597051..e9724b9e33 100644 --- a/pkg/backup/snapshots.go +++ b/pkg/backup/snapshots.go @@ -4,7 +4,6 @@ import ( "context" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" - snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/sets" kbclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -17,25 +16,23 @@ import ( // Common function to update the status of CSI snapshots // returns VolumeSnapshot, VolumeSnapshotContent, VolumeSnapshotClasses referenced -func UpdateBackupCSISnapshotsStatus(client kbclient.Client, volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister, backup *velerov1api.Backup, backupLog logrus.FieldLogger) (volumeSnapshots []snapshotv1api.VolumeSnapshot, volumeSnapshotContents []snapshotv1api.VolumeSnapshotContent, volumeSnapshotClasses []snapshotv1api.VolumeSnapshotClass) { +func UpdateBackupCSISnapshotsStatus(client kbclient.Client, globalCRClient kbclient.Client, backup *velerov1api.Backup, backupLog logrus.FieldLogger) (volumeSnapshots []snapshotv1api.VolumeSnapshot, volumeSnapshotContents []snapshotv1api.VolumeSnapshotContent, volumeSnapshotClasses []snapshotv1api.VolumeSnapshotClass) { if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) { backupLog.Info("backup SnapshotMoveData is set to true, skip VolumeSnapshot resource persistence.") } else if features.IsEnabled(velerov1api.CSIFeatureFlag) { selector := label.NewSelectorForBackup(backup.Name) vscList := &snapshotv1api.VolumeSnapshotContentList{} - if volumeSnapshotLister != nil { - tmpVSs, err := volumeSnapshotLister.List(label.NewSelectorForBackup(backup.Name)) - if err != nil { - backupLog.Error(err) - } - for _, vs := range tmpVSs { - volumeSnapshots = append(volumeSnapshots, *vs) - } + vsList := new(snapshotv1api.VolumeSnapshotList) + err := globalCRClient.List(context.TODO(), vsList, &kbclient.ListOptions{ + LabelSelector: label.NewSelectorForBackup(backup.Name), + }) + if err != nil { + backupLog.Error(err) } + volumeSnapshots = append(volumeSnapshots, vsList.Items...) - err := client.List(context.Background(), vscList, &kbclient.ListOptions{LabelSelector: selector}) - if err != nil { + if err := client.List(context.Background(), vscList, &kbclient.ListOptions{LabelSelector: selector}); err != nil { backupLog.Error(err) } if len(vscList.Items) >= 0 { diff --git a/pkg/builder/volume_snapshot_builder.go b/pkg/builder/volume_snapshot_builder.go index bbaedd16ef..0abc48d2ac 100644 --- a/pkg/builder/volume_snapshot_builder.go +++ b/pkg/builder/volume_snapshot_builder.go @@ -18,6 +18,7 @@ package builder import ( snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -68,7 +69,21 @@ func (v *VolumeSnapshotBuilder) BoundVolumeSnapshotContentName(vscName string) * return v } +// SourcePVC set the built VolumeSnapshot's spec.Source.PersistentVolumeClaimName. func (v *VolumeSnapshotBuilder) SourcePVC(name string) *VolumeSnapshotBuilder { v.object.Spec.Source.PersistentVolumeClaimName = &name return v } + +// RestoreSize set the built VolumeSnapshot's status.RestoreSize. +func (v *VolumeSnapshotBuilder) RestoreSize(size string) *VolumeSnapshotBuilder { + resourceSize := resource.MustParse(size) + v.object.Status.RestoreSize = &resourceSize + return v +} + +// VolumeSnapshotClass set the built VolumeSnapshot's spec.VolumeSnapshotClassName value. +func (v *VolumeSnapshotBuilder) VolumeSnapshotClass(name string) *VolumeSnapshotBuilder { + v.object.Spec.VolumeSnapshotClassName = &name + return v +} diff --git a/pkg/builder/volume_snapshot_content_builder.go b/pkg/builder/volume_snapshot_content_builder.go index 734eeedf3d..bbfbe5477f 100644 --- a/pkg/builder/volume_snapshot_content_builder.go +++ b/pkg/builder/volume_snapshot_content_builder.go @@ -59,6 +59,7 @@ func (v *VolumeSnapshotContentBuilder) DeletionPolicy(policy snapshotv1api.Delet return v } +// VolumeSnapshotRef sets the built VolumeSnapshotContent's spec.VolumeSnapshotRef value. func (v *VolumeSnapshotContentBuilder) VolumeSnapshotRef(namespace, name string) *VolumeSnapshotContentBuilder { v.object.Spec.VolumeSnapshotRef = v1.ObjectReference{ APIVersion: "snapshot.storage.k8s.io/v1", @@ -68,3 +69,18 @@ func (v *VolumeSnapshotContentBuilder) VolumeSnapshotRef(namespace, name string) } return v } + +// VolumeSnapshotClassName sets the built VolumeSnapshotContent's spec.VolumeSnapshotClassName value. +func (v *VolumeSnapshotContentBuilder) VolumeSnapshotClassName(name string) *VolumeSnapshotContentBuilder { + v.object.Spec.VolumeSnapshotClassName = &name + return v +} + +// ObjectMeta applies functional options to the VolumeSnapshotContent's ObjectMeta. +func (v *VolumeSnapshotContentBuilder) ObjectMeta(opts ...ObjectMetaOpt) *VolumeSnapshotContentBuilder { + for _, opt := range opts { + opt(v.object) + } + + return v +} diff --git a/pkg/client/factory.go b/pkg/client/factory.go index 9ff2040c65..9fcb097fbd 100644 --- a/pkg/client/factory.go +++ b/pkg/client/factory.go @@ -24,6 +24,7 @@ import ( k8scheme "k8s.io/client-go/kubernetes/scheme" kbclient "sigs.k8s.io/controller-runtime/pkg/client" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime" @@ -158,6 +159,9 @@ func (f *factory) KubebuilderClient() (kbclient.Client, error) { if err := apiextv1.AddToScheme(scheme); err != nil { return nil, err } + if err := snapshotv1api.AddToScheme(scheme); err != nil { + return nil, err + } kubebuilderClient, err := kbclient.New(clientConfig, kbclient.Options{ Scheme: scheme, }) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index fb9d96cb38..3d91f9474e 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -23,21 +23,16 @@ import ( "net/http" "net/http/pprof" "os" - "reflect" "strings" "time" logrusr "github.com/bombsimon/logrusr/v3" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" - snapshotv1client "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" - snapshotv1informers "github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions" - snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -244,15 +239,17 @@ func NewCommand(f client.Factory) *cobra.Command { } type server struct { - namespace string - metricsAddress string - kubeClientConfig *rest.Config - kubeClient kubernetes.Interface - discoveryClient discovery.DiscoveryInterface - discoveryHelper velerodiscovery.Helper - dynamicClient dynamic.Interface - csiSnapshotClient *snapshotv1client.Clientset - csiSnapshotLister snapshotv1listers.VolumeSnapshotLister + namespace string + metricsAddress string + kubeClientConfig *rest.Config + kubeClient kubernetes.Interface + discoveryClient discovery.DiscoveryInterface + discoveryHelper velerodiscovery.Helper + dynamicClient dynamic.Interface + // controller-runtime client. the difference from the controller-manager's client + // is that the the controller-manager's client is limited to list namespaced-scoped + // resources in the namespace where Velero is installed, or the cluster-scoped + // resources. The crClient doesn't have the limitation. crClient ctrlclient.Client ctx context.Context cancelFunc context.CancelFunc @@ -399,23 +396,6 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s featureVerifier: featureVerifier, } - // Setup CSI snapshot client and lister - var csiSnapClient *snapshotv1client.Clientset - if features.IsEnabled(velerov1api.CSIFeatureFlag) { - csiSnapClient, err = snapshotv1client.NewForConfig(clientConfig) - if err != nil { - cancelFunc() - return nil, err - } - s.csiSnapshotClient = csiSnapClient - - s.csiSnapshotLister, err = s.getCSIVolumeSnapshotListers() - if err != nil { - cancelFunc() - return nil, err - } - } - return s, nil } @@ -615,40 +595,6 @@ func (s *server) initRepoManager() error { return nil } -func (s *server) getCSIVolumeSnapshotListers() (vsLister snapshotv1listers.VolumeSnapshotLister, err error) { - _, err = s.discoveryClient.ServerResourcesForGroupVersion(snapshotv1api.SchemeGroupVersion.String()) - switch { - case apierrors.IsNotFound(err): - // CSI is enabled, but the required CRDs aren't installed, so halt. - s.logger.Warnf("The '%s' feature flag was specified, but CSI API group [%s] was not found.", velerov1api.CSIFeatureFlag, snapshotv1api.SchemeGroupVersion.String()) - err = nil - case err == nil: - wrapper := NewCSIInformerFactoryWrapper(s.csiSnapshotClient) - - s.logger.Debug("Creating CSI listers") - // Access the wrapped factory directly here since we've already done the feature flag check above to know it's safe. - vsLister = wrapper.factory.Snapshot().V1().VolumeSnapshots().Lister() - - // start the informers & and wait for the caches to sync - wrapper.Start(s.ctx.Done()) - s.logger.Info("Waiting for informer caches to sync") - csiCacheSyncResults := wrapper.WaitForCacheSync(s.ctx.Done()) - s.logger.Info("Done waiting for informer caches to sync") - - for informer, synced := range csiCacheSyncResults { - if !synced { - err = errors.Errorf("cache was not synced for informer %v", informer) - return - } - s.logger.WithField("informer", informer).Info("Informer cache synced") - } - case err != nil: - s.logger.Errorf("fail to find snapshot v1 schema: %s", err) - } - - return vsLister, err -} - func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string) error { s.logger.Info("Starting controllers") @@ -775,10 +721,10 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.metrics, backupStoreGetter, s.config.formatFlag.Parse(), - s.csiSnapshotLister, s.credentialFileStore, s.config.maxConcurrentK8SConnections, s.config.defaultSnapshotMoveData, + s.crClient, ).SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", controller.Backup) } @@ -837,7 +783,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string cmd.CheckError(err) r := controller.NewBackupFinalizerReconciler( s.mgr.GetClient(), - s.csiSnapshotLister, + s.crClient, clock.RealClock{}, backupper, newPluginManager, @@ -1027,37 +973,6 @@ func (s *server) runProfiler() { } } -// CSIInformerFactoryWrapper is a proxy around the CSI SharedInformerFactory that checks the CSI feature flag before performing operations. -type CSIInformerFactoryWrapper struct { - factory snapshotv1informers.SharedInformerFactory -} - -func NewCSIInformerFactoryWrapper(c snapshotv1client.Interface) *CSIInformerFactoryWrapper { - // If no namespace is specified, all namespaces are watched. - // This is desirable for VolumeSnapshots, as we want to query for all VolumeSnapshots across all namespaces using this informer - w := &CSIInformerFactoryWrapper{} - - if features.IsEnabled(velerov1api.CSIFeatureFlag) { - w.factory = snapshotv1informers.NewSharedInformerFactoryWithOptions(c, 0) - } - return w -} - -// Start proxies the Start call to the CSI SharedInformerFactory. -func (w *CSIInformerFactoryWrapper) Start(stopCh <-chan struct{}) { - if features.IsEnabled(velerov1api.CSIFeatureFlag) { - w.factory.Start(stopCh) - } -} - -// WaitForCacheSync proxies the WaitForCacheSync call to the CSI SharedInformerFactory. -func (w *CSIInformerFactoryWrapper) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { - if features.IsEnabled(velerov1api.CSIFeatureFlag) { - return w.factory.WaitForCacheSync(stopCh) - } - return nil -} - // if there is a restarting during the reconciling of backups/restores/etc, these CRs may be stuck in progress status // markInProgressCRsFailed tries to mark the in progress CRs as failed when starting the server to avoid the issue func markInProgressCRsFailed(ctx context.Context, cfg *rest.Config, scheme *runtime.Scheme, namespace string, log logrus.FieldLogger) { diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 746c9d7890..e8a8c2eec8 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -21,12 +21,11 @@ import ( "context" "fmt" "os" + "strconv" "strings" "time" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" - snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" - snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" @@ -43,14 +42,18 @@ import ( "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/storage" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" + "github.com/vmware-tanzu/velero/pkg/itemoperation" + "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/plugin/framework" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/util/encode" @@ -84,11 +87,10 @@ type backupReconciler struct { metrics *metrics.ServerMetrics backupStoreGetter persistence.ObjectBackupStoreGetter formatFlag logging.Format - volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister - volumeSnapshotClient snapshotterClientSet.Interface credentialFileStore credentials.FileStore maxConcurrentK8SConnections int defaultSnapshotMoveData bool + globalCRClient kbclient.Client } func NewBackupReconciler( @@ -110,10 +112,10 @@ func NewBackupReconciler( metrics *metrics.ServerMetrics, backupStoreGetter persistence.ObjectBackupStoreGetter, formatFlag logging.Format, - volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister, credentialStore credentials.FileStore, maxConcurrentK8SConnections int, defaultSnapshotMoveData bool, + globalCRClient kbclient.Client, ) *backupReconciler { b := &backupReconciler{ ctx: ctx, @@ -135,10 +137,10 @@ func NewBackupReconciler( metrics: metrics, backupStoreGetter: backupStoreGetter, formatFlag: formatFlag, - volumeSnapshotLister: volumeSnapshotLister, credentialFileStore: credentialStore, maxConcurrentK8SConnections: maxConcurrentK8SConnections, defaultSnapshotMoveData: defaultSnapshotMoveData, + globalCRClient: globalCRClient, } b.updateTotalBackupMetric() return b @@ -317,6 +319,7 @@ func (b *backupReconciler) prepareBackupRequest(backup *velerov1api.Backup, logg request := &pkgbackup.Request{ Backup: backup.DeepCopy(), // don't modify items in the cache SkippedPVTracker: pkgbackup.NewSkipPVTracker(), + PVMap: map[string]pkgbackup.PvcPvInfo{}, } // set backup major version - deprecated, use Status.FormatVersion @@ -665,7 +668,7 @@ func (b *backupReconciler) runBackup(backup *pkgbackup.Request) error { backup.Status.VolumeSnapshotsCompleted++ } } - volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses := pkgbackup.UpdateBackupCSISnapshotsStatus(b.kbClient, b.volumeSnapshotLister, backup.Backup, backupLog) + volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses := pkgbackup.UpdateBackupCSISnapshotsStatus(b.kbClient, b.globalCRClient, backup.Backup, backupLog) // Iterate over backup item operations and update progress. // Any errors on operations at this point should be added to backup errors. @@ -734,6 +737,8 @@ func (b *backupReconciler) runBackup(backup *pkgbackup.Request) error { if logFile, err := backupLog.GetPersistFile(); err != nil { fatalErrs = append(fatalErrs, errors.Wrap(err, "error getting backup log file")) } else { + backup.VolumeInfos.VolumeInfos = generateVolumeInfo(backup, volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses, b.globalCRClient, backupLog) + if errs := persistBackup(backup, backupFile, logFile, backupStore, volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses, results); len(errs) > 0 { fatalErrs = append(fatalErrs, errs...) } @@ -796,7 +801,6 @@ func persistBackup(backup *pkgbackup.Request, ) []error { persistErrs := []error{} backupJSON := new(bytes.Buffer) - volumeInfos := make([]volume.VolumeInfo, 0) if err := encode.To(backup.Backup, "json", backupJSON); err != nil { persistErrs = append(persistErrs, errors.Wrap(err, "error encoding backup")) @@ -843,7 +847,7 @@ func persistBackup(backup *pkgbackup.Request, persistErrs = append(persistErrs, errs...) } - volumeInfoJSON, errs := encode.ToJSONGzip(volumeInfos, "backup volumes information") + volumeInfoJSON, errs := encode.ToJSONGzip(backup.VolumeInfos, "backup volumes information") if errs != nil { persistErrs = append(persistErrs, errs...) } @@ -908,3 +912,328 @@ func oldAndNewFilterParametersUsedTogether(backupSpec velerov1api.BackupSpec) bo return haveOldResourceFilterParameters && haveNewResourceFilterParameters } + +func generateVolumeInfo(backup *pkgbackup.Request, csiVolumeSnapshots []snapshotv1api.VolumeSnapshot, + csiVolumeSnapshotContents []snapshotv1api.VolumeSnapshotContent, csiVolumesnapshotClasses []snapshotv1api.VolumeSnapshotClass, + crClient kbclient.Client, logger logrus.FieldLogger) []volume.VolumeInfo { + volumeInfos := make([]volume.VolumeInfo, 0) + + skippedVolumeInfos := generateVolumeInfoForSkippedPV(backup, logger) + volumeInfos = append(volumeInfos, skippedVolumeInfos...) + + nativeSnapshotVolumeInfos := generateVolumeInfoForVeleroNativeSnapshot(backup, logger) + volumeInfos = append(volumeInfos, nativeSnapshotVolumeInfos...) + + csiVolumeInfos := generateVolumeInfoForCSIVolumeSnapshot(backup, csiVolumeSnapshots, csiVolumeSnapshotContents, csiVolumesnapshotClasses, logger) + volumeInfos = append(volumeInfos, csiVolumeInfos...) + + pvbVolumeInfos := generateVolumeInfoFromPVB(backup, crClient, logger) + volumeInfos = append(volumeInfos, pvbVolumeInfos...) + + dataUploadVolumeInfos := generateVolumeInfoFromDataUpload(backup, crClient, logger) + volumeInfos = append(volumeInfos, dataUploadVolumeInfos...) + + return volumeInfos +} + +// generateVolumeInfoForSkippedPV generate VolumeInfos for SkippedPV. +func generateVolumeInfoForSkippedPV(backup *pkgbackup.Request, logger logrus.FieldLogger) []volume.VolumeInfo { + tmpVolumeInfos := make([]volume.VolumeInfo, 0) + + for _, skippedPV := range backup.SkippedPVTracker.Summary() { + if pvcPVInfo, ok := backup.PVMap[skippedPV.Name]; ok { + volumeInfo := volume.VolumeInfo{ + PVCName: pvcPVInfo.PVCName, + PVCNamespace: pvcPVInfo.PVCNamespace, + PVName: skippedPV.Name, + SnapshotDataMoved: false, + Skipped: true, + SkippedReason: skippedPV.SerializeSkipReasons(), + PVInfo: volume.PVInfo{ + ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy), + Labels: pvcPVInfo.PV.Labels, + }, + } + tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) + } else { + logger.Warnf("Cannot find info for PV %s", skippedPV.Name) + continue + } + } + + return tmpVolumeInfos +} + +// generateVolumeInfoForVeleroNativeSnapshot generate VolumeInfos for Velero native snapshot +func generateVolumeInfoForVeleroNativeSnapshot(backup *pkgbackup.Request, logger logrus.FieldLogger) []volume.VolumeInfo { + tmpVolumeInfos := make([]volume.VolumeInfo, 0) + + for _, nativeSnapshot := range backup.VolumeSnapshots { + var iops int64 + if nativeSnapshot.Spec.VolumeIOPS != nil { + iops = *nativeSnapshot.Spec.VolumeIOPS + } + + if pvcPVInfo, ok := backup.PVMap[nativeSnapshot.Spec.PersistentVolumeName]; ok { + volumeInfo := volume.VolumeInfo{ + BackupMethod: volume.NativeSnapshot, + PVCName: pvcPVInfo.PVCName, + PVCNamespace: pvcPVInfo.PVCNamespace, + PVName: pvcPVInfo.PV.Name, + SnapshotDataMoved: false, + Skipped: false, + NativeSnapshotInfo: volume.NativeSnapshotInfo{ + SnapshotHandle: nativeSnapshot.Status.ProviderSnapshotID, + VolumeType: nativeSnapshot.Spec.VolumeType, + VolumeAZ: nativeSnapshot.Spec.VolumeAZ, + IOPS: strconv.FormatInt(iops, 10), + }, + PVInfo: volume.PVInfo{ + ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy), + Labels: pvcPVInfo.PV.Labels, + }, + } + + tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) + } else { + logger.Warnf("cannot find info for PV %s", nativeSnapshot.Spec.PersistentVolumeName) + continue + } + } + + return tmpVolumeInfos +} + +// generateVolumeInfoForCSIVolumeSnapshot generate VolumeInfos for CSI VolumeSnapshot +func generateVolumeInfoForCSIVolumeSnapshot(backup *pkgbackup.Request, csiVolumeSnapshots []snapshotv1api.VolumeSnapshot, + csiVolumeSnapshotContents []snapshotv1api.VolumeSnapshotContent, csiVolumesnapshotClasses []snapshotv1api.VolumeSnapshotClass, + logger logrus.FieldLogger) []volume.VolumeInfo { + tmpVolumeInfos := make([]volume.VolumeInfo, 0) + + for _, volumeSnapshot := range csiVolumeSnapshots { + var volumeSnapshotClass *snapshotv1api.VolumeSnapshotClass + var volumeSnapshotContent *snapshotv1api.VolumeSnapshotContent + + // This is protective logic. The passed-in VS should be all related + // to this backup. + if volumeSnapshot.Labels[velerov1api.BackupNameLabel] != backup.Name { + continue + } + + if volumeSnapshot.Spec.VolumeSnapshotClassName == nil { + logger.Warnf("Cannot find VolumeSnapshotClass for VolumeSnapshot %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Name) + continue + } + + if volumeSnapshot.Status == nil || volumeSnapshot.Status.BoundVolumeSnapshotContentName == nil { + logger.Warnf("Cannot fine VolumeSnapshotContent for VolumeSnapshot %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Name) + continue + } + + if volumeSnapshot.Spec.Source.PersistentVolumeClaimName == nil { + logger.Warnf("VolumeSnapshot %s/%s doesn't have a source PVC", volumeSnapshot.Namespace, volumeSnapshot.Name) + continue + } + + for index := range csiVolumesnapshotClasses { + if *volumeSnapshot.Spec.VolumeSnapshotClassName == csiVolumesnapshotClasses[index].Name { + volumeSnapshotClass = &csiVolumesnapshotClasses[index] + } + } + + for index := range csiVolumeSnapshotContents { + if *volumeSnapshot.Status.BoundVolumeSnapshotContentName == csiVolumeSnapshotContents[index].Name { + volumeSnapshotContent = &csiVolumeSnapshotContents[index] + } + } + + if volumeSnapshotClass == nil || volumeSnapshotContent == nil { + logger.Warnf("fail to get VolumeSnapshotContent or VolumeSnapshotClass for VolumeSnapshot: %s/%s", + volumeSnapshot.Namespace, volumeSnapshot.Name) + continue + } + + var operation itemoperation.BackupOperation + for _, op := range *backup.GetItemOperationsList() { + if op.Spec.ResourceIdentifier.GroupResource.String() == kuberesource.VolumeSnapshots.String() && + op.Spec.ResourceIdentifier.Name == volumeSnapshot.Name && + op.Spec.ResourceIdentifier.Namespace == volumeSnapshot.Namespace { + operation = *op + } + } + + var size int64 + if volumeSnapshot.Status.RestoreSize != nil { + size = volumeSnapshot.Status.RestoreSize.Value() + } + snapshotHandle := "" + if volumeSnapshotContent.Status.SnapshotHandle != nil { + snapshotHandle = *volumeSnapshotContent.Status.SnapshotHandle + } + if pvcPVInfo, ok := backup.PVMap[volumeSnapshot.Namespace+"/"+*volumeSnapshot.Spec.Source.PersistentVolumeClaimName]; ok { + volumeInfo := volume.VolumeInfo{ + BackupMethod: volume.CSISnapshot, + PVCName: pvcPVInfo.PVCName, + PVCNamespace: pvcPVInfo.PVCNamespace, + PVName: pvcPVInfo.PV.Name, + Skipped: false, + SnapshotDataMoved: false, + PreserveLocalSnapshot: true, + OperationID: operation.Spec.OperationID, + StartTimestamp: &volumeSnapshot.CreationTimestamp, + CSISnapshotInfo: volume.CSISnapshotInfo{ + VSCName: *volumeSnapshot.Status.BoundVolumeSnapshotContentName, + Size: size, + Driver: volumeSnapshotClass.Driver, + SnapshotHandle: snapshotHandle, + }, + PVInfo: volume.PVInfo{ + ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy), + Labels: pvcPVInfo.PV.Labels, + }, + } + + tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) + } else { + logger.Warnf("cannot find info for PVC %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Spec.Source.PersistentVolumeClaimName) + continue + } + } + + return tmpVolumeInfos +} + +// generateVolumeInfoFromPVB generate VolumeInfo for PVB. +func generateVolumeInfoFromPVB(backup *pkgbackup.Request, crClient kbclient.Client, logger logrus.FieldLogger) []volume.VolumeInfo { + tmpVolumeInfos := make([]volume.VolumeInfo, 0) + + for _, pvb := range backup.PodVolumeBackups { + volumeInfo := volume.VolumeInfo{ + BackupMethod: volume.PodVolumeBackup, + SnapshotDataMoved: false, + Skipped: false, + StartTimestamp: pvb.Status.StartTimestamp, + PVBInfo: volume.PodVolumeBackupInfo{ + SnapshotHandle: pvb.Status.SnapshotID, + Size: pvb.Status.Progress.TotalBytes, + UploaderType: pvb.Spec.UploaderType, + VolumeName: pvb.Spec.Volume, + PodName: pvb.Spec.Pod.Name, + PodNamespace: pvb.Spec.Pod.Namespace, + NodeName: pvb.Spec.Node, + }, + } + + pod := new(corev1api.Pod) + pvcName := "" + err := crClient.Get(context.TODO(), kbclient.ObjectKey{Namespace: pvb.Spec.Pod.Namespace, Name: pvb.Spec.Pod.Name}, pod) + if err != nil { + logger.WithError(err).Warn("Fail to get pod for PodVolumeBackup: ", pvb.Name) + continue + } + for _, volume := range pod.Spec.Volumes { + if volume.Name == pvb.Spec.Volume && volume.PersistentVolumeClaim != nil { + pvcName = volume.PersistentVolumeClaim.ClaimName + } + } + + if pvcName != "" { + if pvcPVInfo, ok := backup.PVMap[pod.Namespace+"/"+pvcName]; ok { + volumeInfo.PVCName = pvcPVInfo.PVCName + volumeInfo.PVCNamespace = pvcPVInfo.PVCNamespace + volumeInfo.PVName = pvcPVInfo.PV.Name + volumeInfo.PVInfo = volume.PVInfo{ + ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy), + Labels: pvcPVInfo.PV.Labels, + } + } else { + logger.Warnf("Cannot find info for PVC %s/%s", pod.Namespace, pvcName) + continue + } + } else { + logger.Debug("The PVB %s doesn't have a corresponding PVC", pvb.Name) + } + + tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) + } + + return tmpVolumeInfos +} + +// generateVolumeInfoFromDataUpload generate VolumeInfo for DataUpload. +func generateVolumeInfoFromDataUpload(backup *pkgbackup.Request, crClient kbclient.Client, logger logrus.FieldLogger) []volume.VolumeInfo { + tmpVolumeInfos := make([]volume.VolumeInfo, 0) + vsClassList := new(snapshotv1api.VolumeSnapshotClassList) + if err := crClient.List(context.TODO(), vsClassList); err != nil { + logger.WithError(err).Errorf("cannot list VolumeSnapshotClass %s", err.Error()) + return tmpVolumeInfos + } + + for _, operation := range *backup.GetItemOperationsList() { + if operation.Spec.ResourceIdentifier.GroupResource.String() == kuberesource.PersistentVolumeClaims.String() { + var duIdentifier velero.ResourceIdentifier + + for _, identifier := range operation.Spec.PostOperationItems { + if identifier.GroupResource.String() == "datauploads.velero.io" { + duIdentifier = identifier + } + } + if duIdentifier.Empty() { + logger.Warnf("cannot find DataUpload for PVC %s/%s backup async operation", + operation.Spec.ResourceIdentifier.Namespace, operation.Spec.ResourceIdentifier.Name) + continue + } + + dataUpload := new(velerov2alpha1.DataUpload) + err := crClient.Get( + context.TODO(), + kbclient.ObjectKey{ + Namespace: duIdentifier.Namespace, + Name: duIdentifier.Name}, + dataUpload, + ) + if err != nil { + logger.Warnf("fail to get DataUpload for operation %s: %s", operation.Spec.OperationID, err.Error()) + continue + } + + driverUsedByVSClass := "" + for index := range vsClassList.Items { + if vsClassList.Items[index].Name == dataUpload.Spec.CSISnapshot.SnapshotClass { + driverUsedByVSClass = vsClassList.Items[index].Driver + } + } + + if pvcPVInfo, ok := backup.PVMap[operation.Spec.ResourceIdentifier.Namespace+"/"+operation.Spec.ResourceIdentifier.Name]; ok { + volumeInfo := volume.VolumeInfo{ + BackupMethod: volume.CSISnapshot, + PVCName: pvcPVInfo.PVCName, + PVCNamespace: pvcPVInfo.PVCNamespace, + PVName: pvcPVInfo.PV.Name, + SnapshotDataMoved: true, + Skipped: false, + OperationID: operation.Spec.OperationID, + StartTimestamp: operation.Status.Created, + CSISnapshotInfo: volume.CSISnapshotInfo{ + Driver: driverUsedByVSClass, + }, + SnapshotDataMovementInfo: volume.SnapshotDataMovementInfo{ + DataMover: dataUpload.Spec.DataMover, + UploaderType: "kopia", + }, + PVInfo: volume.PVInfo{ + ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy), + Labels: pvcPVInfo.PV.Labels, + }, + } + + tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) + } else { + logger.Warnf("Cannot find info for PVC %s/%s", operation.Spec.ResourceIdentifier.Namespace, operation.Spec.ResourceIdentifier.Name) + continue + } + } + } + + return tmpVolumeInfos +} diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index df2e22a22a..736209e4a1 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -29,15 +29,16 @@ import ( "github.com/google/go-cmp/cmp" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" - snapshotfake "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/fake" - snapshotinformers "github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + corev1api "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/version" "k8s.io/utils/clock" @@ -45,11 +46,14 @@ import ( ctrl "sigs.k8s.io/controller-runtime" kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/vmware-tanzu/velero/pkg/backup" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" + "github.com/vmware-tanzu/velero/pkg/volume" fakeClient "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/discovery" @@ -61,6 +65,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/plugin/framework" pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/boolptr" @@ -1062,12 +1067,11 @@ func TestProcessBackupCompletions(t *testing.T) { CSIVolumeSnapshotsCompleted: 0, }, }, - volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { - name: "backup with snapshot data movement set to false when CSI feature is enabled", - backup: defaultBackup().SnapshotMoveData(false).Result(), - //backup: defaultBackup().Result(), + name: "backup with snapshot data movement set to false when CSI feature is enabled", + backup: defaultBackup().SnapshotMoveData(false).Result(), backupLocation: defaultBackupLocation, defaultVolumesToFsBackup: false, expectedResult: &velerov1api.Backup{ @@ -1103,7 +1107,7 @@ func TestProcessBackupCompletions(t *testing.T) { CSIVolumeSnapshotsCompleted: 0, }, }, - volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { name: "backup with snapshot data movement not set when CSI feature is enabled", @@ -1143,7 +1147,7 @@ func TestProcessBackupCompletions(t *testing.T) { CSIVolumeSnapshotsCompleted: 0, }, }, - volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { name: "backup with snapshot data movement set to true and defaultSnapshotMoveData set to false", @@ -1184,7 +1188,7 @@ func TestProcessBackupCompletions(t *testing.T) { CSIVolumeSnapshotsCompleted: 0, }, }, - volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { name: "backup with snapshot data movement set to false and defaultSnapshotMoveData set to true", @@ -1225,7 +1229,7 @@ func TestProcessBackupCompletions(t *testing.T) { CSIVolumeSnapshotsCompleted: 0, }, }, - volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { name: "backup with snapshot data movement not set and defaultSnapshotMoveData set to true", @@ -1266,35 +1270,43 @@ func TestProcessBackupCompletions(t *testing.T) { CSIVolumeSnapshotsCompleted: 0, }, }, - volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, } + snapshotHandle := "testSnapshotID" + for _, test := range tests { t.Run(test.name, func(t *testing.T) { formatFlag := logging.FormatText var ( - logger = logging.DefaultLogger(logrus.DebugLevel, formatFlag) - pluginManager = new(pluginmocks.Manager) - backupStore = new(persistencemocks.BackupStore) - backupper = new(fakeBackupper) - snapshotClient = snapshotfake.NewSimpleClientset() - sharedInformer = snapshotinformers.NewSharedInformerFactory(snapshotClient, 0) - snapshotLister = sharedInformer.Snapshot().V1().VolumeSnapshots().Lister() + logger = logging.DefaultLogger(logrus.DebugLevel, formatFlag) + pluginManager = new(pluginmocks.Manager) + backupStore = new(persistencemocks.BackupStore) + backupper = new(fakeBackupper) + fakeGlobalClient = velerotest.NewFakeControllerRuntimeClient(t) ) var fakeClient kbclient.Client // add the test's backup storage location if it's different than the default if test.backupLocation != nil && test.backupLocation != defaultBackupLocation { - fakeClient = velerotest.NewFakeControllerRuntimeClient(t, test.backupLocation) + fakeClient = velerotest.NewFakeControllerRuntimeClient(t, test.backupLocation, + builder.ForVolumeSnapshotClass("testClass").Driver("testDriver").Result(), + builder.ForVolumeSnapshotContent("testVSC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).VolumeSnapshotClassName("testClass").Status(&snapshotv1api.VolumeSnapshotContentStatus{ + SnapshotHandle: &snapshotHandle, + }).Result(), + ) } else { - fakeClient = velerotest.NewFakeControllerRuntimeClient(t) + fakeClient = velerotest.NewFakeControllerRuntimeClient(t, + builder.ForVolumeSnapshotClass("testClass").Driver("testDriver").Result(), + builder.ForVolumeSnapshotContent("testVSC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).VolumeSnapshotClassName("testClass").Status(&snapshotv1api.VolumeSnapshotContentStatus{ + SnapshotHandle: &snapshotHandle, + }).Result(), + ) } if test.volumeSnapshot != nil { - snapshotClient.SnapshotV1().VolumeSnapshots(test.volumeSnapshot.Namespace).Create(context.Background(), test.volumeSnapshot, metav1.CreateOptions{}) - sharedInformer.Snapshot().V1().VolumeSnapshots().Informer().GetStore().Add(test.volumeSnapshot) - sharedInformer.WaitForCacheSync(make(chan struct{})) + require.NoError(t, fakeGlobalClient.Create(context.TODO(), test.volumeSnapshot)) } apiServer := velerotest.NewAPIServer(t) @@ -1328,8 +1340,7 @@ func TestProcessBackupCompletions(t *testing.T) { backupStoreGetter: NewFakeSingleObjectBackupStoreGetter(backupStore), backupper: backupper, formatFlag: formatFlag, - volumeSnapshotClient: snapshotClient, - volumeSnapshotLister: snapshotLister, + globalCRClient: fakeGlobalClient, } pluginManager.On("GetBackupItemActionsV2").Return(nil, nil) @@ -1731,3 +1742,749 @@ func TestPatchResourceWorksWithStatus(t *testing.T) { } } +func TestGenerateVolumeInfoForSkippedPV(t *testing.T) { + tests := []struct { + name string + skippedPVName string + pvMap map[string]backup.PvcPvInfo + expectedVolumeInfos []volume.VolumeInfo + }{ + { + name: "Cannot find info for PV", + skippedPVName: "testPV", + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "Normal Skipped PV info", + skippedPVName: "testPV", + pvMap: map[string]backup.PvcPvInfo{ + "velero/testPVC": { + PVCName: "testPVC", + PVCNamespace: "velero", + PV: corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPV", + Labels: map[string]string{"a": "b"}, + }, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, + }, + }, + }, + "testPV": { + PVCName: "testPVC", + PVCNamespace: "velero", + PV: corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPV", + Labels: map[string]string{"a": "b"}, + }, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, + }, + }, + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{ + { + PVCName: "testPVC", + PVCNamespace: "velero", + PVName: "testPV", + Skipped: true, + SkippedReason: "CSI: skipped for PodVolumeBackup;", + PVInfo: volume.PVInfo{ + ReclaimPolicy: "Delete", + Labels: map[string]string{ + "a": "b", + }, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + request := new(backup.Request) + request.SkippedPVTracker = backup.NewSkipPVTracker() + if tc.skippedPVName != "" { + request.SkippedPVTracker.Track(tc.skippedPVName, "CSI", "skipped for PodVolumeBackup") + } + if tc.pvMap != nil { + request.PVMap = tc.pvMap + } + logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON) + + volumeInfos := generateVolumeInfoForSkippedPV(request, logger) + require.Equal(t, tc.expectedVolumeInfos, volumeInfos) + }) + } +} + +func TestGenerateVolumeInfoForCSIVolumeSnapshot(t *testing.T) { + resourceQuantity := resource.MustParse("100Gi") + now := metav1.Now() + tests := []struct { + name string + volumeSnapshot snapshotv1api.VolumeSnapshot + volumeSnapshotContent snapshotv1api.VolumeSnapshotContent + volumeSnapshotClass snapshotv1api.VolumeSnapshotClass + pvMap map[string]backup.PvcPvInfo + operation *itemoperation.BackupOperation + expectedVolumeInfos []volume.VolumeInfo + }{ + { + name: "VS doesn't have VolumeSnapshotClass name", + volumeSnapshot: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testVS", + Namespace: "velero", + }, + Spec: snapshotv1api.VolumeSnapshotSpec{}, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "VS doesn't have status", + volumeSnapshot: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testVS", + Namespace: "velero", + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + VolumeSnapshotClassName: stringPtr("testClass"), + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "VS doesn't have PVC", + volumeSnapshot: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testVS", + Namespace: "velero", + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + VolumeSnapshotClassName: stringPtr("testClass"), + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: stringPtr("testContent"), + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "Cannot find VSC for VS", + volumeSnapshot: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testVS", + Namespace: "velero", + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + VolumeSnapshotClassName: stringPtr("testClass"), + Source: snapshotv1api.VolumeSnapshotSource{ + PersistentVolumeClaimName: stringPtr("testPVC"), + }, + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: stringPtr("testContent"), + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "Cannot find VolumeInfo for PVC", + volumeSnapshot: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testVS", + Namespace: "velero", + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + VolumeSnapshotClassName: stringPtr("testClass"), + Source: snapshotv1api.VolumeSnapshotSource{ + PersistentVolumeClaimName: stringPtr("testPVC"), + }, + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: stringPtr("testContent"), + }, + }, + volumeSnapshotClass: *builder.ForVolumeSnapshotClass("testClass").Driver("pd.csi.storage.gke.io").Result(), + volumeSnapshotContent: *builder.ForVolumeSnapshotContent("testContent").Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: stringPtr("testSnapshotHandle")}).Result(), + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "Normal VolumeSnapshot case", + volumeSnapshot: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testVS", + Namespace: "velero", + CreationTimestamp: now, + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + VolumeSnapshotClassName: stringPtr("testClass"), + Source: snapshotv1api.VolumeSnapshotSource{ + PersistentVolumeClaimName: stringPtr("testPVC"), + }, + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: stringPtr("testContent"), + RestoreSize: &resourceQuantity, + }, + }, + volumeSnapshotClass: *builder.ForVolumeSnapshotClass("testClass").Driver("pd.csi.storage.gke.io").Result(), + volumeSnapshotContent: *builder.ForVolumeSnapshotContent("testContent").Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: stringPtr("testSnapshotHandle")}).Result(), + pvMap: map[string]backup.PvcPvInfo{ + "velero/testPVC": { + PVCName: "testPVC", + PVCNamespace: "velero", + PV: corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPV", + Labels: map[string]string{"a": "b"}, + }, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, + }, + }, + }, + }, + operation: &itemoperation.BackupOperation{ + Spec: itemoperation.BackupOperationSpec{ + OperationID: "testID", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: "snapshot.storage.k8s.io", + Resource: "volumesnapshots", + }, + Namespace: "velero", + Name: "testVS", + }, + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{ + { + PVCName: "testPVC", + PVCNamespace: "velero", + PVName: "testPV", + BackupMethod: volume.CSISnapshot, + OperationID: "testID", + StartTimestamp: &now, + PreserveLocalSnapshot: true, + CSISnapshotInfo: volume.CSISnapshotInfo{ + Driver: "pd.csi.storage.gke.io", + SnapshotHandle: "testSnapshotHandle", + Size: 107374182400, + VSCName: "testContent", + }, + PVInfo: volume.PVInfo{ + ReclaimPolicy: "Delete", + Labels: map[string]string{ + "a": "b", + }, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + request := new(backup.Request) + request.Backup = new(velerov1api.Backup) + if tc.pvMap != nil { + request.PVMap = tc.pvMap + } + operationList := request.GetItemOperationsList() + if tc.operation != nil { + *operationList = append(*operationList, tc.operation) + } + logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON) + + volumeInfos := generateVolumeInfoForCSIVolumeSnapshot(request, []snapshotv1api.VolumeSnapshot{tc.volumeSnapshot}, []snapshotv1api.VolumeSnapshotContent{tc.volumeSnapshotContent}, []snapshotv1api.VolumeSnapshotClass{tc.volumeSnapshotClass}, logger) + require.Equal(t, tc.expectedVolumeInfos, volumeInfos) + }) + } + +} + +func TestGenerateVolumeInfoForVeleroNativeSnapshot(t *testing.T) { + tests := []struct { + name string + nativeSnapshot volume.Snapshot + pvMap map[string]backup.PvcPvInfo + expectedVolumeInfos []volume.VolumeInfo + }{ + { + name: "Native snapshot's IPOS pointer is nil", + nativeSnapshot: volume.Snapshot{ + Spec: volume.SnapshotSpec{ + PersistentVolumeName: "testPV", + VolumeIOPS: nil, + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "Cannot find info for the PV", + nativeSnapshot: volume.Snapshot{ + Spec: volume.SnapshotSpec{ + PersistentVolumeName: "testPV", + VolumeIOPS: int64Ptr(100), + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "Normal native snapshot", + pvMap: map[string]backup.PvcPvInfo{ + "testPV": { + PVCName: "testPVC", + PVCNamespace: "velero", + PV: corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPV", + Labels: map[string]string{"a": "b"}, + }, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, + }, + }, + }, + }, + nativeSnapshot: volume.Snapshot{ + Spec: volume.SnapshotSpec{ + PersistentVolumeName: "testPV", + VolumeIOPS: int64Ptr(100), + VolumeType: "ssd", + VolumeAZ: "us-central1-a", + }, + Status: volume.SnapshotStatus{ + ProviderSnapshotID: "pvc-b31e3386-4bbb-4937-95d-7934cd62-b0a1-494b-95d7-0687440e8d0c", + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{ + { + PVCName: "testPVC", + PVCNamespace: "velero", + PVName: "testPV", + BackupMethod: volume.NativeSnapshot, + PVInfo: volume.PVInfo{ + ReclaimPolicy: "Delete", + Labels: map[string]string{ + "a": "b", + }, + }, + NativeSnapshotInfo: volume.NativeSnapshotInfo{ + SnapshotHandle: "pvc-b31e3386-4bbb-4937-95d-7934cd62-b0a1-494b-95d7-0687440e8d0c", + VolumeType: "ssd", + VolumeAZ: "us-central1-a", + IOPS: "100", + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + request := new(backup.Request) + request.VolumeSnapshots = append(request.VolumeSnapshots, &tc.nativeSnapshot) + if tc.pvMap != nil { + request.PVMap = tc.pvMap + } + logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON) + + volumeInfos := generateVolumeInfoForVeleroNativeSnapshot(request, logger) + require.Equal(t, tc.expectedVolumeInfos, volumeInfos) + }) + } +} + +func TestGenerateVolumeInfoFromPVB(t *testing.T) { + tests := []struct { + name string + pvb *velerov1api.PodVolumeBackup + pod *corev1api.Pod + pvMap map[string]backup.PvcPvInfo + expectedVolumeInfos []volume.VolumeInfo + }{ + { + name: "cannot find PVB's pod, should fail", + pvb: builder.ForPodVolumeBackup("velero", "testPVB").PodName("testPod").PodNamespace("velero").Result(), + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "PVB doesn't have a related PVC", + pvb: builder.ForPodVolumeBackup("velero", "testPVB").PodName("testPod").PodNamespace("velero").Result(), + pod: builder.ForPod("velero", "testPod").Containers(&corev1api.Container{ + Name: "test", + VolumeMounts: []corev1api.VolumeMount{ + { + Name: "testVolume", + MountPath: "/data", + }, + }, + }).Volumes( + &corev1api.Volume{ + Name: "", + VolumeSource: corev1api.VolumeSource{ + HostPath: &corev1api.HostPathVolumeSource{}, + }, + }, + ).Result(), + expectedVolumeInfos: []volume.VolumeInfo{ + { + PVCName: "", + PVCNamespace: "", + PVName: "", + BackupMethod: volume.PodVolumeBackup, + PVBInfo: volume.PodVolumeBackupInfo{ + PodName: "testPod", + PodNamespace: "velero", + }, + }, + }, + }, + { + name: "Backup doesn't have information for PVC", + pvb: builder.ForPodVolumeBackup("velero", "testPVB").PodName("testPod").PodNamespace("velero").Result(), + pod: builder.ForPod("velero", "testPod").Containers(&corev1api.Container{ + Name: "test", + VolumeMounts: []corev1api.VolumeMount{ + { + Name: "testVolume", + MountPath: "/data", + }, + }, + }).Volumes( + &corev1api.Volume{ + Name: "", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "testPVC", + }, + }, + }, + ).Result(), + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "PVB's volume has a PVC", + pvMap: map[string]backup.PvcPvInfo{ + "velero/testPVC": { + PVCName: "testPVC", + PVCNamespace: "velero", + PV: corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPV", + Labels: map[string]string{"a": "b"}, + }, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, + }, + }, + }, + }, + pvb: builder.ForPodVolumeBackup("velero", "testPVB").PodName("testPod").PodNamespace("velero").Result(), + pod: builder.ForPod("velero", "testPod").Containers(&corev1api.Container{ + Name: "test", + VolumeMounts: []corev1api.VolumeMount{ + { + Name: "testVolume", + MountPath: "/data", + }, + }, + }).Volumes( + &corev1api.Volume{ + Name: "", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "testPVC", + }, + }, + }, + ).Result(), + expectedVolumeInfos: []volume.VolumeInfo{ + { + PVCName: "testPVC", + PVCNamespace: "velero", + PVName: "testPV", + BackupMethod: volume.PodVolumeBackup, + PVBInfo: volume.PodVolumeBackupInfo{ + PodName: "testPod", + PodNamespace: "velero", + }, + PVInfo: volume.PVInfo{ + ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), + Labels: map[string]string{"a": "b"}, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + crClient := velerotest.NewFakeControllerRuntimeClient(t) + logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON) + request := new(pkgbackup.Request) + request.PodVolumeBackups = append(request.PodVolumeBackups, tc.pvb) + if tc.pvMap != nil { + request.PVMap = tc.pvMap + } + if tc.pod != nil { + require.NoError(t, crClient.Create(context.TODO(), tc.pod)) + } + + volumeInfos := generateVolumeInfoFromPVB(request, crClient, logger) + require.Equal(t, tc.expectedVolumeInfos, volumeInfos) + }) + } +} + +func TestGenerateVolumeInfoFromDataUpload(t *testing.T) { + now := metav1.Now() + tests := []struct { + name string + volumeSnapshotClass *snapshotv1api.VolumeSnapshotClass + dataUpload *velerov2alpha1.DataUpload + operation *itemoperation.BackupOperation + pvMap map[string]backup.PvcPvInfo + expectedVolumeInfos []volume.VolumeInfo + }{ + { + name: "Operation is not for PVC", + operation: &itemoperation.BackupOperation{ + Spec: itemoperation.BackupOperationSpec{ + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: "", + Resource: "configmaps", + }, + }, + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "Operation doesn't have DataUpload PostItemOperation", + operation: &itemoperation.BackupOperation{ + Spec: itemoperation.BackupOperationSpec{ + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: "", + Resource: "persistentvolumeclaims", + }, + Namespace: "velero", + Name: "testPVC", + }, + PostOperationItems: []velero.ResourceIdentifier{ + { + GroupResource: schema.GroupResource{ + Group: "", + Resource: "configmaps", + }, + }, + }, + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "DataUpload cannot be found for operation", + operation: &itemoperation.BackupOperation{ + Spec: itemoperation.BackupOperationSpec{ + OperationID: "testOperation", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: "", + Resource: "persistentvolumeclaims", + }, + Namespace: "velero", + Name: "testPVC", + }, + PostOperationItems: []velero.ResourceIdentifier{ + { + GroupResource: schema.GroupResource{ + Group: "velero.io", + Resource: "datauploads", + }, + Namespace: "velero", + Name: "testDU", + }, + }, + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "VolumeSnapshotClass cannot be found for operation", + dataUpload: builder.ForDataUpload("velero", "testDU").DataMover("velero").CSISnapshot(&velerov2alpha1.CSISnapshotSpec{ + VolumeSnapshot: "testVS", + }).SnapshotID("testSnapshotHandle").Result(), + operation: &itemoperation.BackupOperation{ + Spec: itemoperation.BackupOperationSpec{ + OperationID: "testOperation", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: "", + Resource: "persistentvolumeclaims", + }, + Namespace: "velero", + Name: "testPVC", + }, + PostOperationItems: []velero.ResourceIdentifier{ + { + GroupResource: schema.GroupResource{ + Group: "velero.io", + Resource: "datauploads", + }, + Namespace: "velero", + Name: "testDU", + }, + }, + }, + }, + pvMap: map[string]backup.PvcPvInfo{ + "velero/testPVC": { + PVCName: "testPVC", + PVCNamespace: "velero", + PV: corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPV", + Labels: map[string]string{"a": "b"}, + }, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, + }, + }, + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{ + { + PVCName: "testPVC", + PVCNamespace: "velero", + PVName: "testPV", + BackupMethod: volume.CSISnapshot, + SnapshotDataMoved: true, + OperationID: "testOperation", + SnapshotDataMovementInfo: volume.SnapshotDataMovementInfo{ + DataMover: "velero", + UploaderType: "kopia", + }, + PVInfo: volume.PVInfo{ + ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), + Labels: map[string]string{"a": "b"}, + }, + }, + }, + }, + { + name: "Normal DataUpload case", + dataUpload: builder.ForDataUpload("velero", "testDU").DataMover("velero").CSISnapshot(&velerov2alpha1.CSISnapshotSpec{ + VolumeSnapshot: "testVS", + SnapshotClass: "testClass", + }).SnapshotID("testSnapshotHandle").Result(), + volumeSnapshotClass: builder.ForVolumeSnapshotClass("testClass").Driver("pd.csi.storage.gke.io").Result(), + operation: &itemoperation.BackupOperation{ + Spec: itemoperation.BackupOperationSpec{ + OperationID: "testOperation", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: "", + Resource: "persistentvolumeclaims", + }, + Namespace: "velero", + Name: "testPVC", + }, + PostOperationItems: []velero.ResourceIdentifier{ + { + GroupResource: schema.GroupResource{ + Group: "velero.io", + Resource: "datauploads", + }, + Namespace: "velero", + Name: "testDU", + }, + }, + }, + Status: itemoperation.OperationStatus{ + Created: &now, + }, + }, + pvMap: map[string]backup.PvcPvInfo{ + "velero/testPVC": { + PVCName: "testPVC", + PVCNamespace: "velero", + PV: corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPV", + Labels: map[string]string{"a": "b"}, + }, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, + }, + }, + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{ + { + PVCName: "testPVC", + PVCNamespace: "velero", + PVName: "testPV", + BackupMethod: volume.CSISnapshot, + SnapshotDataMoved: true, + OperationID: "testOperation", + StartTimestamp: &now, + CSISnapshotInfo: volume.CSISnapshotInfo{ + Driver: "pd.csi.storage.gke.io", + }, + SnapshotDataMovementInfo: volume.SnapshotDataMovementInfo{ + DataMover: "velero", + UploaderType: "kopia", + }, + PVInfo: volume.PVInfo{ + ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), + Labels: map[string]string{"a": "b"}, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + request := new(backup.Request) + operationList := request.GetItemOperationsList() + if tc.operation != nil { + *operationList = append(*operationList, tc.operation) + } + if tc.pvMap != nil { + request.PVMap = tc.pvMap + } + logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON) + + crClient := velerotest.NewFakeControllerRuntimeClient(t) + if tc.dataUpload != nil { + crClient.Create(context.TODO(), tc.dataUpload) + } + + if tc.volumeSnapshotClass != nil { + crClient.Create(context.TODO(), tc.volumeSnapshotClass) + } + + volumeInfos := generateVolumeInfoFromDataUpload(request, crClient, logger) + require.Equal(t, tc.expectedVolumeInfos, volumeInfos) + }) + } +} + +func int64Ptr(val int) *int64 { + i := int64(val) + return &i +} + +func stringPtr(str string) *string { + return &str +} diff --git a/pkg/controller/backup_finalizer_controller.go b/pkg/controller/backup_finalizer_controller.go index eb99f6ee53..ea9c0364b2 100644 --- a/pkg/controller/backup_finalizer_controller.go +++ b/pkg/controller/backup_finalizer_controller.go @@ -29,8 +29,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" kbclient "sigs.k8s.io/controller-runtime/pkg/client" - snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1" - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/metrics" @@ -42,21 +40,21 @@ import ( // backupFinalizerReconciler reconciles a Backup object type backupFinalizerReconciler struct { - client kbclient.Client - volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister - clock clocks.WithTickerAndDelayedExecution - backupper pkgbackup.Backupper - newPluginManager func(logrus.FieldLogger) clientmgmt.Manager - backupTracker BackupTracker - metrics *metrics.ServerMetrics - backupStoreGetter persistence.ObjectBackupStoreGetter - log logrus.FieldLogger + client kbclient.Client + globalCRClient kbclient.Client + clock clocks.WithTickerAndDelayedExecution + backupper pkgbackup.Backupper + newPluginManager func(logrus.FieldLogger) clientmgmt.Manager + backupTracker BackupTracker + metrics *metrics.ServerMetrics + backupStoreGetter persistence.ObjectBackupStoreGetter + log logrus.FieldLogger } // NewBackupFinalizerReconciler initializes and returns backupFinalizerReconciler struct. func NewBackupFinalizerReconciler( client kbclient.Client, - volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister, + globalCRClient kbclient.Client, clock clocks.WithTickerAndDelayedExecution, backupper pkgbackup.Backupper, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, @@ -67,6 +65,7 @@ func NewBackupFinalizerReconciler( ) *backupFinalizerReconciler { return &backupFinalizerReconciler{ client: client, + globalCRClient: globalCRClient, clock: clock, backupper: backupper, newPluginManager: newPluginManager, @@ -191,7 +190,7 @@ func (r *backupFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Requ backup.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} recordBackupMetrics(log, backup, outBackupFile, r.metrics, true) - pkgbackup.UpdateBackupCSISnapshotsStatus(r.client, r.volumeSnapshotLister, backup, log) + pkgbackup.UpdateBackupCSISnapshotsStatus(r.client, r.globalCRClient, backup, log) // update backup metadata in object store backupJSON := new(bytes.Buffer) if err := encode.To(backup, "json", backupJSON); err != nil { diff --git a/pkg/controller/backup_finalizer_controller_test.go b/pkg/controller/backup_finalizer_controller_test.go index f759d03187..74f6da57c5 100644 --- a/pkg/controller/backup_finalizer_controller_test.go +++ b/pkg/controller/backup_finalizer_controller_test.go @@ -23,7 +23,6 @@ import ( "testing" "time" - snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -44,14 +43,13 @@ import ( "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" - velerotestmocks "github.com/vmware-tanzu/velero/pkg/test/mocks" ) -func mockBackupFinalizerReconciler(fakeClient kbclient.Client, fakeVolumeSnapshotLister snapshotv1listers.VolumeSnapshotLister, fakeClock *testclocks.FakeClock) (*backupFinalizerReconciler, *fakeBackupper) { +func mockBackupFinalizerReconciler(fakeClient kbclient.Client, fakeGlobalClient kbclient.Client, fakeClock *testclocks.FakeClock) (*backupFinalizerReconciler, *fakeBackupper) { backupper := new(fakeBackupper) return NewBackupFinalizerReconciler( fakeClient, - fakeVolumeSnapshotLister, + fakeGlobalClient, fakeClock, backupper, func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, @@ -164,9 +162,9 @@ func TestBackupFinalizerReconcile(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) - fakeVolumeSnapshotLister := velerotestmocks.NewVolumeSnapshotLister(t) + fakeGlobalClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) - reconciler, backupper := mockBackupFinalizerReconciler(fakeClient, fakeVolumeSnapshotLister, fakeClock) + reconciler, backupper := mockBackupFinalizerReconciler(fakeClient, fakeGlobalClient, fakeClock) pluginManager.On("CleanupClients").Return(nil) backupStore.On("GetBackupItemOperations", test.backup.Name).Return(test.backupOperations, nil) backupStore.On("GetBackupContents", mock.Anything).Return(io.NopCloser(bytes.NewReader([]byte("hello world"))), nil) diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index 26bddc5656..48a48dbf17 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -608,6 +608,8 @@ func (s *objectBackupStore) GetDownloadURL(target velerov1api.DownloadTarget) (s return s.objectStore.CreateSignedURL(s.bucket, s.layout.getCSIVolumeSnapshotContentsKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindBackupResults: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupResultsKey(target.Name), DownloadURLTTL) + case velerov1api.DownloadTargetKindBackupVolumeInfos: + return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupVolumeInfoKey(target.Name), DownloadURLTTL) default: return "", errors.Errorf("unsupported download target kind %q", target.Kind) } diff --git a/pkg/persistence/object_store_test.go b/pkg/persistence/object_store_test.go index 6c149f188c..ba6a7bcb83 100644 --- a/pkg/persistence/object_store_test.go +++ b/pkg/persistence/object_store_test.go @@ -768,6 +768,13 @@ func TestGetDownloadURL(t *testing.T) { velerov1api.DownloadTargetKindRestoreResourceList: "restores/b-cool-20170913154901-20170913154902/restore-b-cool-20170913154901-20170913154902-resource-list.json.gz", }, }, + { + name: "", + targetName: "my-backup", + expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{ + velerov1api.DownloadTargetKindBackupVolumeInfos: "backups/my-backup/my-backup-volumeinfos.json.gz", + }, + }, } for _, test := range tests { diff --git a/pkg/volume/volume_info_common.go b/pkg/volume/volume_info_common.go index cfca31df9b..14ede0c6b1 100644 --- a/pkg/volume/volume_info_common.go +++ b/pkg/volume/volume_info_common.go @@ -51,6 +51,8 @@ type VolumeInfo struct { SnapshotDataMoved bool `json:"snapshotDataMoved"` // Whether the local snapshot is preserved after snapshot is moved. + // The local snapshot may be a result of CSI snapshot backup(no data movement) + // or a CSI snapshot data movement plus preserve local snapshot. PreserveLocalSnapshot bool `json:"preserveLocalSnapshot"` // Whether the Volume is skipped in this backup. @@ -69,6 +71,7 @@ type VolumeInfo struct { SnapshotDataMovementInfo SnapshotDataMovementInfo `json:"snapshotDataMovementInfo,omitempty"` NativeSnapshotInfo NativeSnapshotInfo `json:"nativeSnapshotInfo,omitempty"` PVBInfo PodVolumeBackupInfo `json:"pvbInfo,omitempty"` + PVInfo PVInfo `json:"pvInfo,omitempty"` } // CSISnapshotInfo is used for displaying the CSI snapshot status @@ -76,7 +79,7 @@ type CSISnapshotInfo struct { // It's the storage provider's snapshot ID for CSI. SnapshotHandle string `json:"snapshotHandle"` - // The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot. + // The snapshot corresponding volume size. Size int64 `json:"size"` // The name of the CSI driver. @@ -91,7 +94,7 @@ type SnapshotDataMovementInfo struct { // The data mover used by the backup. The valid values are `velero` and ``(equals to `velero`). DataMover string `json:"dataMover"` - // The type of the uploader that uploads the snapshot data. The valid values are `kopia` and `restic`. It's useful for file-system backup and snapshot data mover. + // The type of the uploader that uploads the snapshot data. The valid values are `kopia` and `restic`. UploaderType string `json:"uploaderType"` // The name or ID of the snapshot associated object(SAO). @@ -111,9 +114,6 @@ type NativeSnapshotInfo struct { // It's the storage provider's snapshot ID for the Velero-native snapshot. SnapshotHandle string `json:"snapshotHandle"` - // The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot. - Size int64 `json:"size"` - // The cloud provider snapshot volume type. VolumeType string `json:"volumeType"` @@ -129,19 +129,32 @@ type PodVolumeBackupInfo struct { // It's the file-system uploader's snapshot ID for PodVolumeBackup. SnapshotHandle string `json:"snapshotHandle"` - // The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot. + // The snapshot corresponding volume size. Size int64 `json:"size"` - // The type of the uploader that uploads the data. The valid values are `kopia` and `restic`. It's useful for file-system backup and snapshot data mover. + // The type of the uploader that uploads the data. The valid values are `kopia` and `restic`. UploaderType string `json:"uploaderType"` // The PVC's corresponding volume name used by Pod // https://github.com/kubernetes/kubernetes/blob/e4b74dd12fa8cb63c174091d5536a10b8ec19d34/pkg/apis/core/types.go#L48 VolumeName string `json:"volumeName"` - // The Pod name mounting this PVC. The format should be /. + // The Pod name mounting this PVC. PodName string `json:"podName"` + // The Pod namespace + PodNamespace string `json:"podNamespace"` + // The PVB-taken k8s node's name. NodeName string `json:"nodeName"` } + +// PVInfo is used to store some PV information modified after creation. +// Those information are lost after PV recreation. +type PVInfo struct { + // ReclaimPolicy of PV. It could be different from the referenced StorageClass. + ReclaimPolicy string `json:"reclaimPolicy"` + + // The PV's labels should be kept after recreation. + Labels map[string]string `json:"labels"` +}