From cd4eeec32a32af529135ce6bbd1261a8aac6f9ec Mon Sep 17 00:00:00 2001 From: d33bs Date: Mon, 14 Feb 2022 16:33:41 -0700 Subject: [PATCH 1/4] default basename for objstrfput to address #36 --- src/cupyopt/objectstore_tasks.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/cupyopt/objectstore_tasks.py b/src/cupyopt/objectstore_tasks.py index d1b52eb..cd2cf48 100644 --- a/src/cupyopt/objectstore_tasks.py +++ b/src/cupyopt/objectstore_tasks.py @@ -1,6 +1,7 @@ """ object store functions """ import logging +import os from typing import Any from typing_extensions import Literal @@ -211,27 +212,31 @@ def __init__( self, client: Minio = None, bucket_name: str = None, - object_name: str = None, file_path: str = None, + object_name: str = None, **kwargs: Any ): self.client = client self.bucket_name = bucket_name - self.object_name = object_name self.file_path = file_path + self.object_name = object_name super().__init__(**kwargs) - @defaults_from_attrs("client", "bucket_name", "object_name", "file_path") + @defaults_from_attrs("client", "bucket_name", "file_path", "object_name") def run( self, client: Minio, bucket_name: str, - object_name: str, file_path: str, + object_name: str = None, ): + # if no object_name is provided, default to file_path basename + if not object_name: + object_name = os.path.basename(file_path) + # upload file as object client.fput_object( bucket_name=bucket_name, From 63843e363bd35c032089c050ecce163da72c9b49 Mon Sep 17 00:00:00 2001 From: d33bs Date: Mon, 14 Feb 2022 16:37:13 -0700 Subject: [PATCH 2/4] return filepath for objstrfget to address #35 --- src/cupyopt/objectstore_tasks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/cupyopt/objectstore_tasks.py b/src/cupyopt/objectstore_tasks.py index cd2cf48..993e67b 100644 --- a/src/cupyopt/objectstore_tasks.py +++ b/src/cupyopt/objectstore_tasks.py @@ -266,7 +266,9 @@ def __init__( super().__init__(**kwargs) @defaults_from_attrs("client", "bucket_name", "object_name", "file_path") - def run(self, client: Minio, bucket_name: str, object_name: str, file_path: str): + def run( + self, client: Minio, bucket_name: str, object_name: str, file_path: str + ) -> str: # get object as file client.fget_object( @@ -281,3 +283,5 @@ def run(self, client: Minio, bucket_name: str, object_name: str, file_path: str) bucket_name, file_path, ) + + return file_path From 6269372adb57718fe56a6b8c46b12236cd2afa7b Mon Sep 17 00:00:00 2001 From: d33bs Date: Mon, 14 Feb 2022 16:53:01 -0700 Subject: [PATCH 3/4] bump version --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 829b863..52c21c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ minio nose pandas pandavro==1.6.0 -prefect==0.15.11 +prefect==0.15.13 pyarrow==6.0.1 pygrok pylint diff --git a/setup.py b/setup.py index f6505a1..3dcec4d 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ # This call to setup() does all the work setuptools.setup( name="cupyopt", - version="0.15.11.2", + version="0.15.13.1", description="CU Python Opinionated Prefect Tasks", long_description=README, long_description_content_type="text/x-rst", From 6581fc6a45f87e0901b29c8574a324485e8b6510 Mon Sep 17 00:00:00 2001 From: d33bs Date: Tue, 15 Feb 2022 08:02:35 -0700 Subject: [PATCH 4/4] remove vscode dir --- .vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index de288e1..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.formatting.provider": "black" -} \ No newline at end of file