From 31b80c41c4dcd30519146625de9c65676d0711f2 Mon Sep 17 00:00:00 2001 From: SergeyRyabinin Date: Thu, 14 Dec 2023 23:37:02 +0000 Subject: [PATCH] CancelAll to also cancel directory upload --- .../include/aws/core/platform/FileSystem.h | 4 ++-- .../source/platform/linux-shared/FileSystem.cpp | 6 +++--- .../include/aws/transfer/TransferHandle.h | 12 ++++++------ .../include/aws/transfer/TransferManager.h | 8 ++++---- .../source/transfer/TransferManager.cpp | 9 +++++++-- 5 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/aws-cpp-sdk-core/include/aws/core/platform/FileSystem.h b/src/aws-cpp-sdk-core/include/aws/core/platform/FileSystem.h index 5eeacd2fbd3..056a98b42b1 100644 --- a/src/aws-cpp-sdk-core/include/aws/core/platform/FileSystem.h +++ b/src/aws-cpp-sdk-core/include/aws/core/platform/FileSystem.h @@ -108,8 +108,8 @@ namespace FileSystem Aws::String path; Aws::String relativePath; - FileType fileType; - int64_t fileSize; + FileType fileType = FileType::None; + int64_t fileSize = 0; }; /** diff --git a/src/aws-cpp-sdk-core/source/platform/linux-shared/FileSystem.cpp b/src/aws-cpp-sdk-core/source/platform/linux-shared/FileSystem.cpp index 71debad7447..131dfdad2d0 100644 --- a/src/aws-cpp-sdk-core/source/platform/linux-shared/FileSystem.cpp +++ b/src/aws-cpp-sdk-core/source/platform/linux-shared/FileSystem.cpp @@ -120,7 +120,7 @@ static const char* FILE_SYSTEM_UTILS_LOG_TAG = "FileSystemUtils"; AWS_LOGSTREAM_TRACE(FILE_SYSTEM_UTILS_LOG_TAG, "Calling stat on path " << entry.path); - struct stat dirInfo; + struct stat dirInfo = {}; if(!lstat(entry.path.c_str(), &dirInfo)) { if(S_ISDIR(dirInfo.st_mode)) @@ -139,7 +139,7 @@ static const char* FILE_SYSTEM_UTILS_LOG_TAG = "FileSystemUtils"; entry.fileType = FileType::File; } - entry.fileSize = static_cast(dirInfo.st_size); + entry.fileSize = static_cast(dirInfo.st_size); AWS_LOGSTREAM_DEBUG(FILE_SYSTEM_UTILS_LOG_TAG, "file size detected as " << entry.fileSize); } else @@ -150,7 +150,7 @@ static const char* FILE_SYSTEM_UTILS_LOG_TAG = "FileSystemUtils"; return entry; } - DIR* m_dir; + DIR* m_dir = nullptr; }; Aws::String GetHomeDirectory() diff --git a/src/aws-cpp-sdk-transfer/include/aws/transfer/TransferHandle.h b/src/aws-cpp-sdk-transfer/include/aws/transfer/TransferHandle.h index beff479bb29..40a90437227 100644 --- a/src/aws-cpp-sdk-transfer/include/aws/transfer/TransferHandle.h +++ b/src/aws-cpp-sdk-transfer/include/aws/transfer/TransferHandle.h @@ -81,17 +81,17 @@ namespace Aws void SetChecksum(const Aws::String& checksum) { m_checksum = checksum; } private: - int m_partId; + int m_partId = 0; Aws::String m_eTag; - uint64_t m_currentProgressInBytes; - uint64_t m_bestProgressInBytes; - uint64_t m_sizeInBytes; - uint64_t m_rangeBegin; + uint64_t m_currentProgressInBytes = 0; + uint64_t m_bestProgressInBytes = 0; + uint64_t m_sizeInBytes = 0; + uint64_t m_rangeBegin = 0; std::atomic m_downloadPartStream; std::atomic m_downloadBuffer; - bool m_lastPart; + bool m_lastPart = false; Aws::String m_checksum; }; diff --git a/src/aws-cpp-sdk-transfer/include/aws/transfer/TransferManager.h b/src/aws-cpp-sdk-transfer/include/aws/transfer/TransferManager.h index 8d16c91c8e3..23dce0dc9ef 100644 --- a/src/aws-cpp-sdk-transfer/include/aws/transfer/TransferManager.h +++ b/src/aws-cpp-sdk-transfer/include/aws/transfer/TransferManager.h @@ -52,13 +52,13 @@ namespace Aws * you are using for your client configuration. This executor will be used in a different context than the s3 client is used. * It is not a bug to use the same executor, but at least be aware that this is how the manager will be used. */ - Aws::Utils::Threading::Executor* transferExecutor; + Aws::Utils::Threading::Executor* transferExecutor = nullptr; /** * When true, TransferManager will calculate the MD5 digest of the content being uploaded. * The digest is sent to S3 via an HTTP header enabling the service to perform integrity checks. * This option is disabled by default. Defer to checksumAlgorithm to use other checksum algorithms. */ - bool computeContentMD5; + bool computeContentMD5 = false; /** * If you have special arguments you want passed to our put object calls, put them here. We will copy the template for each put object call * overriding the body stream, bucket, and key. If object metadata is passed through, we will override that as well. @@ -89,12 +89,12 @@ namespace Aws * allocate for all transfer buffers. default is 50MB. * If you are using Aws::Utils::Threading::PooledThreadExecutor for transferExecutor, this size should be greater than bufferSize * poolSize. */ - uint64_t transferBufferMaxHeapSize; + uint64_t transferBufferMaxHeapSize = 10 * MB5; /** * Defaults to 5MB. If you are uploading large files, (larger than 50GB, this needs to be specified to be something larger than 5MB. Also keep in mind that you may need * to increase your max heap size if this is something you plan on increasing. */ - uint64_t bufferSize; + uint64_t bufferSize = MB5; /** * Callback to receive progress updates for uploads. diff --git a/src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp b/src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp index 1c3b0f1fea1..b4b8cac8aa9 100644 --- a/src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp +++ b/src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp @@ -276,10 +276,16 @@ namespace Aws void TransferManager::UploadDirectory(const Aws::String& directory, const Aws::String& bucketName, const Aws::String& prefix, const Aws::Map& metadata) { assert(m_transferConfig.transferInitiatedCallback); + auto handle = Aws::MakeShared(CLASS_TAG, bucketName, prefix); // fake handle auto self = shared_from_this(); - auto visitor = [self, bucketName, prefix, metadata](const Aws::FileSystem::DirectoryTree*, const Aws::FileSystem::DirectoryEntry& entry) + auto visitor = [self, bucketName, prefix, metadata, handle](const Aws::FileSystem::DirectoryTree*, const Aws::FileSystem::DirectoryEntry& entry) { + if (!handle || !handle->ShouldContinue()) + { + return false; // Allow to cancel directory upload + } + if (entry && entry.fileType == Aws::FileSystem::FileType::File) { Aws::StringStream ssKey; @@ -298,7 +304,6 @@ namespace Aws return true; }; - auto handle = Aws::MakeShared(CLASS_TAG, bucketName, prefix); // fake handle AddTask(handle); m_transferConfig.transferExecutor->Submit( [directory, visitor, self, handle]()