diff --git a/OSCAR_flow/upload/upload.cwl b/OSCAR_flow/upload/upload.cwl new file mode 100755 index 0000000..87c88b8 --- /dev/null +++ b/OSCAR_flow/upload/upload.cwl @@ -0,0 +1,23 @@ +#!/usr/bin/env cwl-runner + +cwlVersion: v1.0 +class: CommandLineTool +baseCommand: python +arguments: [$(inputs.uploadScript),"--bucket",$(inputs.bucket),"--filename", $(inputs.file), "--endpoint",$(inputs.minioEndpoint),"--accesskey",$(inputs.minioAccesskey),"--secretkey",$(inputs.minioSecretkey)] +inputs: + file: + type: File? + uploadScript: + type: File? + bucket: + type: string + minioEndpoint: + type: string + minioAccesskey: + type: string + minioSecretkey: + type: string +outputs: + example_out: + type: stdout +stdout: output.txt diff --git a/OSCAR_flow/upload/upload.py b/OSCAR_flow/upload/upload.py new file mode 100644 index 0000000..257229e --- /dev/null +++ b/OSCAR_flow/upload/upload.py @@ -0,0 +1,33 @@ +from minio import Minio +import argparse + + +parser = argparse.ArgumentParser() + +parser.add_argument("--bucket") +parser.add_argument("--filename") +parser.add_argument("--endpoint") +parser.add_argument("--accesskey") +parser.add_argument("--secretkey") + +args = parser.parse_args() +variables = vars(args) +#print(variables) + +bucket=variables['bucket'] +filename=variables['filename'] +endpoint=variables['endpoint'] +accesskey=variables['accesskey'] +secretkey=variables['secretkey'] + +#print (bucket.split("/")[0],) +#print('/'.join(sys.argv[1].split("/")[1:])) +#print(filename.split("/")[-1]) +#print(filename) + +# Create client with access and secret key. +client = Minio(str(endpoint), accesskey, secretkey) + +result = client.fput_object( + bucket.split("/")[0], '/'.join(bucket.split("/")[1:])+"/"+filename.split("/")[-1], filename, +) \ No newline at end of file diff --git a/OSCAR_flow/wait_output/wait.cwl b/OSCAR_flow/wait_output/wait.cwl new file mode 100755 index 0000000..756d96f --- /dev/null +++ b/OSCAR_flow/wait_output/wait.cwl @@ -0,0 +1,21 @@ +#!/usr/bin/env cwl-runner + +cwlVersion: v1.0 +class: CommandLineTool +baseCommand: python +arguments: [$(inputs.waitScript),"--bucket",$(inputs.bucket), "--endpoint",$(inputs.minioEndpoint),"--accesskey",$(inputs.minioAccesskey),"--secretkey",$(inputs.minioSecretkey)] +inputs: + waitScript: + type: File? + bucket: + type: string + minioEndpoint: + type: string + minioAccesskey: + type: string + minioSecretkey: + type: string +outputs: + example_out: + type: stdout +stdout: output.txt diff --git a/OSCAR_flow/wait_output/wait.py b/OSCAR_flow/wait_output/wait.py new file mode 100644 index 0000000..84cc86e --- /dev/null +++ b/OSCAR_flow/wait_output/wait.py @@ -0,0 +1,36 @@ +from minio import Minio +import argparse + +parser = argparse.ArgumentParser() + +parser.add_argument("--bucket") +parser.add_argument("--filename") +parser.add_argument("--endpoint") +parser.add_argument("--accesskey") +parser.add_argument("--secretkey") + +args = parser.parse_args() +variables = vars(args) +#print(variables) + +bucket=variables['bucket'] +filename=variables['filename'] +endpoint=variables['endpoint'] +accesskey=variables['accesskey'] +secretkey=variables['secretkey'] + + +# Create client with access and secret key. +client = Minio(str(endpoint), accesskey, secretkey) + +with client.listen_bucket_notification( + bucket.split("/")[0], + prefix='/'.join(bucket.split("/")[1:]), + events=["s3:ObjectCreated:*", "s3:ObjectRemoved:*"], +) as events: + for event in events: + print(event["Records"][0]["s3"]["object"]["key"]) + break + #if(resultfile in event["Records"][0]["s3"]["object"]["key"]): + # info=event + # break \ No newline at end of file diff --git a/OSCAR_flow/workflow-minio.cwl b/OSCAR_flow/workflow-minio.cwl new file mode 100755 index 0000000..e98d22d --- /dev/null +++ b/OSCAR_flow/workflow-minio.cwl @@ -0,0 +1,51 @@ +#!/usr/bin/env cwl-runner + +cwlVersion: v1.2 +class: Workflow + +requirements: + InlineJavascriptRequirement: {} + +inputs: + file: + type: File? + uploadScript: + type: File? + waitScript: + type: File? + minioEndpoint: + type: string + minioAccesskey: + type: string + minioSecretkey: + type: string + bucketInput: + type: string + bucketOutput: + type: string +outputs: + out: + type: File? + outputSource: wait/example_out +steps: + upload: + run: upload/upload.cwl + in: + file: file + uploadScript: uploadScript + bucket: bucketInput + minioEndpoint: minioEndpoint + minioAccesskey: minioAccesskey + minioSecretkey: minioSecretkey + out: [example_out] + wait: + run: wait_output/wait.cwl + in: + waitScript: waitScript + bucket: bucketOutput + minioEndpoint: minioEndpoint + minioAccesskey: minioAccesskey + minioSecretkey: minioSecretkey + data: + source: upload/example_out + out: [example_out] diff --git a/OSCAR_flow/workflow.yaml b/OSCAR_flow/workflow.yaml new file mode 100755 index 0000000..5e94fe0 --- /dev/null +++ b/OSCAR_flow/workflow.yaml @@ -0,0 +1,21 @@ +file: + class: File + path: wflow_input.tar + + +minioEndpoint: +minioAccesskey: +minioSecretkey: + + +bucketInput: wflow/in +bucketOutput: wflow/out + + +uploadScript: + class: File + path: upload/upload.py + +waitScript: + class: File + path: wait_output/wait.py \ No newline at end of file