fix dockerfile publishing

catalystneuro · Sep 28, 2024 · 6bd2999 · 6bd2999
1 parent b119181
commit 6bd2999
Show file tree

Hide file tree

Showing 6 changed files with 14 additions and 6 deletions.
diff --git a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml
@@ -31,7 +31,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           push: true  # Push is a shorthand for --output=type=registry
-          tags: ghcr.io/catalystneuro/neuroconv:yaml_variable
+          tags: ghcr.io/catalystneuro/neuroconv_yaml_variable:latest
           context: .
           file: dockerfiles/neuroconv_latest_yaml_variable
           provenance: false
diff --git a/dockerfiles/neuroconv_latest_yaml_variable b/dockerfiles/neuroconv_latest_yaml_variable
@@ -1,4 +1,4 @@
 FROM ghcr.io/catalystneuro/neuroconv:latest
 LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv
 LABEL org.opencontainers.image.description="A docker image for the most recent official release of the NeuroConv package. Modified to take in environment variables for the YAML conversion specification and other command line arguments."
-CMD echo "$NEUROCONV_YAML" > run.yml && python -m neuroconv run.yml --data-folder-path "$NEUROCONV_DATA_PATH" --output-folder-path "$NEUROCONV_OUTPUT_PATH" --overwrite
+CMD printf "$NEUROCONV_YAML" > ./run.yml && python -m neuroconv run.yml --data-folder-path "$NEUROCONV_DATA_PATH" --output-folder-path "$NEUROCONV_OUTPUT_PATH" --overwrite
diff --git a/src/neuroconv/tools/aws/_deploy_neuroconv_batch_job.py b/src/neuroconv/tools/aws/_deploy_neuroconv_batch_job.py
@@ -135,7 +135,7 @@ def deploy_neuroconv_batch_job(
         message = f"Unable to create EFS volume '{efs_volume_name}' after {max_iterations} attempts!"
         raise ValueError(message)
 
-    docker_image = "ghcr.io/catalystneuro/neuroconv_latest_yaml_variable:latest"
+    docker_image = "ghcr.io/catalystneuro/neuroconv_yaml_variable:latest"
 
     with open(file=yaml_specification_file_path, mode="r") as io:
         yaml_specification_file_stream = io.read()
@@ -147,6 +147,8 @@ def deploy_neuroconv_batch_job(
         docker_image=docker_image,
         environment_variables={
             "NEUROCONV_YAML": yaml_specification_file_stream,
+            # TODO: would prefer this to use subfolders for source and output, but need logic for YAML
+            # related code to create them if missing (hard to send EFS this command directly)
             "NEUROCONV_DATA_PATH": "/mnt/efs/source",
             "NEUROCONV_OUTPUT_PATH": "/mnt/efs/output",
         },

diff --git a/src/neuroconv/tools/aws/_submit_aws_batch_job.py b/src/neuroconv/tools/aws/_submit_aws_batch_job.py
@@ -642,7 +642,7 @@ def _ensure_job_definition_exists_and_get_arn(
                 },
             },
         ]
-        mountPoints = [{"containerPath": "/mnt/efs/", "readOnly": False, "sourceVolume": "neuroconv_batch_efs_mounted"}]
+        mountPoints = [{"containerPath": "/mnt/efs", "readOnly": False, "sourceVolume": "neuroconv_batch_efs_mounted"}]
 
     # batch_client.register_job_definition is not synchronous and so we need to wait a bit afterwards
     batch_client.register_job_definition(

diff --git a/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py b/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py
@@ -73,10 +73,16 @@ def run_conversion_from_yaml(
 
     if data_folder_path is None:
         data_folder_path = Path(specification_file_path).parent
+    else:
+        data_folder_path = Path(data_folder_path)
+        data_folder_path.mkdir(exist_ok=True)
+
     if output_folder_path is None:
-        output_folder_path = Path(specification_file_path).parent
+        output_folder_path = specification_file_path.parent
     else:
         output_folder_path = Path(output_folder_path)
+        output_folder_path.mkdir(exist_ok=True)
+
     specification = load_dict_from_file(file_path=specification_file_path)
     schema_folder = Path(__file__).parent.parent.parent / "schemas"
     specification_schema = load_dict_from_file(file_path=schema_folder / "yaml_conversion_specification_schema.json")

diff --git a/tests/test_on_data/test_yaml/neuroconv_deployment_aws_tools_tests.py b/tests/test_on_data/test_yaml/neuroconv_deployment_aws_tools_tests.py
@@ -168,7 +168,7 @@ def test_deploy_neuroconv_batch_job(self):
         self.efs_id = efs_volume["FileSystemId"]
 
         # Check normal job completion
-        assert job["jobName"] == job_name
+        assert job["jobName"] == f"{job_name}_neuroconv_deployment"
         assert "neuroconv_batch_queue" in job["jobQueue"]
         assert "fs-" in job["jobDefinition"]
         assert job["status"] == "SUCCEEDED"