examples: added validation for examples

usnistgov · Aug 30, 2023 · 23b2b38 · 23b2b38
1 parent 9fda3ec
commit 23b2b38
Show file tree

Hide file tree

Showing 5 changed files with 42 additions and 35 deletions.
diff --git a/examples/tensorflow-mnist-classifier/src/MLproject b/examples/tensorflow-mnist-classifier/src/MLproject
@@ -32,7 +32,7 @@ entry_points:
       norm: { type: string, default: "inf" }
       seed: { type: float, default: -1 }
     command: >
-      PYTHONPATH=$DIOPTRA_PLUGIN_DIR run-experiment fgm.yml
+      PYTHONPATH=$DIOPTRA_PLUGIN_DIR validate-experiment fgm.yml && PYTHONPATH=$DIOPTRA_PLUGIN_DIR run-experiment fgm.yml
       -P data_dir={data_dir}
       -P image_size={image_size}
       -P adv_tar_name={adv_tar_name}
@@ -55,7 +55,7 @@ entry_points:
       adv_data_dir: { type: string, default: "adv_testing" }
       seed: { type: float, default: -1 }
     command: >
-      PYTHONPATH=$DIOPTRA_PLUGIN_DIR run-experiment infer.yml
+      PYTHONPATH=$DIOPTRA_PLUGIN_DIR validate-experiment infer.yml && PYTHONPATH=$DIOPTRA_PLUGIN_DIR run-experiment infer.yml
       -P run_id={run_id}
       -P image_size={image_size}
       -P model_name={model_name}

diff --git a/examples/tensorflow-mnist-classifier/src/fgm.yml b/examples/tensorflow-mnist-classifier/src/fgm.yml
@@ -28,7 +28,7 @@ types:
     image_size:
         tuple: [integer, integer, integer]
     norm:
-        union: [integer, float, string]
+        union: [integer, number, string]
 
 parameters:
     data_dir: /dioptra/data/Mnist/testing
@@ -37,7 +37,7 @@ parameters:
     adv_data_dir: adv_testing
     distance_metrics_filename: distance_metrics.csv
     model_name: mnist_le_net
-    model_version: "1"
+    model_version: 1
     clip_values: [0, 1]
     batch_size: 32
     eps: 0.3
@@ -89,9 +89,13 @@ tasks:
     load_classifier:
         plugin: dioptra_builtins.registry.art.load_wrapped_tensorflow_keras_classifier
         inputs:
-            - name: string
+            - name: name
+              type: string
+              required: true
             - version: integer
-            - classifier_kwargs: kwargs_null
+            - name: classifier_kwargs
+              type: kwargs
+              required: false
         outputs: 
             classifier: keras_classifier
 
@@ -113,7 +117,7 @@ tasks:
               type: distance_metrics_null
               required: false
             - name: rescale
-              type: float
+              type: number
               required: false
             - name: batch_size
               type: integer
@@ -122,13 +126,13 @@ tasks:
               type: string
               required: false
             - name: eps
-              type: float
+              type: number
               required: false
             - name: eps_step
-              type: float
+              type: number
               required: false
             - name: minimal
-              type: bool
+              type: boolean
               required: false
             - name: norm
               type: norm
@@ -144,7 +148,9 @@ tasks:
             - name: tarball_write_mode
               type: string
               required: false
-            - working_dir: path_string_null
+            - name: working_dir
+              type: path_string_null
+              required: false
 
     upload_artifact_dataframe:
         plugin: dioptra_builtins.artifacts.mlflow.upload_data_frame_artifact

diff --git a/examples/tensorflow-mnist-classifier/src/infer.yml b/examples/tensorflow-mnist-classifier/src/infer.yml
@@ -2,6 +2,8 @@ types:
     rng:
     path:
     sequential:
+    number_null:
+        union: [number, "null"]
     string_null:
         union: [string, "null"]
     path_string:
@@ -18,23 +20,15 @@ types:
     kwargs_null:
         union: [kwargs, "null"]
     keras_classifier:
-    distance_metric_request:
-        mapping: [string, string]
-    distance_metrics_requests: 
-        list: distance_metric_request
     eval_metric:
         mapping: [string, any]
     eval_metric_results:
-        mapping: [string, float]
-    distance_metrics: 
-        list: distance_metric
-    distance_metrics_null:
-        union: [distance_metrics, "null"]
+        mapping: [string, number]
     dataframe:
     image_size:
         tuple: [integer, integer, integer]
     norm:
-        union: [integer, float, string]
+        union: [integer, number, string]
 
 
 
@@ -44,7 +38,7 @@ parameters:
     adv_tar_name: testing_adversarial_fgm.tar.gz
     adv_data_dir: adv_testing
     model_name: mnist_le_net
-    model_version: "1"
+    model_version: 1
     seed: -1
 
 
@@ -96,7 +90,7 @@ tasks:
     extract_tarfile:
         plugin: dioptra_builtins.artifacts.utils.extract_tarfile
         inputs:
-            - filepath: 
+            - filepath: path_string
             - name: tarball_read_mode
               type: string
               required: false
@@ -108,14 +102,14 @@ tasks:
         plugin: dioptra_builtins.data.tensorflow.create_image_dataset
         inputs: 
             - data_dir: string
-            - subset: str_null
+            - subset: string_null
             - image_size: image_size
             - seed: integer
             - name: rescale
-              type: float
+              type: number
               required: false
             - name: validation_split
-              type: float
+              type: number_null
               required: false
             - name: batch_size
               type: integer
@@ -125,11 +119,14 @@ tasks:
               required: false
         outputs: 
             iterator: directory_iterator
+
     load_classifier:
         plugin: dioptra_builtins.registry.mlflow.load_tensorflow_keras_classifier
         inputs:
-            - name: string
-            - version: int
+            - name: name
+              type: string
+              required: true
+            - version: integer
         outputs:
             classifier: sequential
 
@@ -143,7 +140,7 @@ tasks:
     log_results:
         plugin: dioptra_builtins.tracking.mlflow.log_metrics
         inputs:
-            metrics: eval_metric_results
+            - metrics: eval_metric_results
 
 graph:
 

diff --git a/examples/tensorflow-mnist-pixel-threshold/src/MLproject b/examples/tensorflow-mnist-pixel-threshold/src/MLproject
@@ -31,7 +31,7 @@ entry_points:
       seed: { type: float, default: -1 }
       clip_values: { type: string, default: "[0,1]" }
     command: >
-      PYTHONPATH=$DIOPTRA_PLUGIN_DIR run-experiment pt.yml
+      PYTHONPATH=$DIOPTRA_PLUGIN_DIR validate-experiment pt.yml && PYTHONPATH=$DIOPTRA_PLUGIN_DIR run-experiment pt.yml
       -P data_dir={data_dir}
       -P image_size={image_size}
       -P adv_tar_name={adv_tar_name}
@@ -58,7 +58,7 @@ entry_points:
       validation_split: { type: float, default: 0.2 }
       seed: { type: float, default: -1 }
     command: >
-      PYTHONPATH=$DIOPTRA_PLUGIN_DIR run-experiment train.yml
+      PYTHONPATH=$DIOPTRA_PLUGIN_DIR validate-experiment train.yml && PYTHONPATH=$DIOPTRA_PLUGIN_DIR run-experiment train.yml
       -P training_dir={training_dir}
       -P testing_dir={testing_dir}
       -P image_size={image_size}

diff --git a/examples/tensorflow-mnist-pixel-threshold/src/pt.yml b/examples/tensorflow-mnist-pixel-threshold/src/pt.yml
@@ -35,7 +35,7 @@ parameters:
     adv_data_dir: adv_testing
     distance_metrics_filename: distance_metrics.csv
     model_name: mnist_le_net
-    model_version: "1"
+    model_version: 1
     clip_values: [0, 1]
     batch_size: 32
     th: 1
@@ -85,7 +85,9 @@ tasks:
     load_classifier:
         plugin: dioptra_builtins.registry.art.load_wrapped_tensorflow_keras_classifier
         inputs:
-            - name: string
+            - name: name
+              type: string
+              required: true
             - version: integer
             - classifier_kwargs: kwargs_null
         outputs: 
@@ -109,7 +111,7 @@ tasks:
               type: distance_metrics_null
               required: false
             - name: rescale
-              type: float
+              type: number
               required: false
             - name: batch_size
               type: integer
@@ -134,7 +136,9 @@ tasks:
             - name: tarball_write_mode
               type: string
               required: false
-            - working_dir: path_string_null
+            - name: working_dir
+              type: path_string_null
+              required: false
 
     upload_artifact_dataframe:
         plugin: dioptra_builtins.artifacts.mlflow.upload_data_frame_artifact