input and output files need to be in order, instead of a set's random…

… order
WIPACrepo · Oct 17, 2024 · 557e883 · 557e883
1 parent 47d70d5
commit 557e883
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 13 deletions.
diff --git a/iceprod/core/exe.py b/iceprod/core/exe.py
@@ -150,18 +150,18 @@ def scope_env(cfg: ConfigParser, obj: dict, upperenv: Optional[Env] = None, logg
     """
     env: Env = {
         'parameters': {},
-        'input_files': set(),
-        'output_files': set(),
-        'upper_input_files': set(),
-        'upper_output_files': set(),
+        'input_files': [],
+        'output_files': [],
+        'upper_input_files': [],
+        'upper_output_files': [],
         'environment': {
             'OS_ARCH': '$OS_ARCH',
         }
     }
     if upperenv:
         env['parameters'].update(upperenv['parameters'])
-        env['upper_input_files'] = upperenv['input_files'] | upperenv['upper_input_files']
-        env['upper_output_files'] = upperenv['output_files'] | upperenv['upper_output_files']
+        env['upper_input_files'] = upperenv['input_files'] + upperenv['upper_input_files']
+        env['upper_output_files'] = upperenv['output_files'] + upperenv['upper_output_files']
 
     logger = logger if logger else logging.getLogger()
 
@@ -183,15 +183,15 @@ def scope_env(cfg: ConfigParser, obj: dict, upperenv: Optional[Env] = None, logg
                 if d['movement'] in ('input','both'):
                     ret = downloadData(d, cfg=cfg, logger=logger)
                     if ret:
-                        env['input_files'].add(ret)
+                        env['input_files'].append(ret)
                 if d['movement'] in ('output','both'):
                     ret = uploadData(d, cfg=cfg, logger=logger)
                     if ret:
-                        env['output_files'].add(ret)
+                        env['output_files'].append(ret)
 
         # add input and output to parseable options
-        cfg.config['options']['input'] = ' '.join(d.local for d in (env['input_files'] | env['upper_input_files']))
-        cfg.config['options']['output'] = ' '.join(d.local for d in (env['output_files'] | env['upper_output_files']))
+        cfg.config['options']['input'] = ' '.join(d.local for d in (env['input_files'] + env['upper_input_files']))
+        cfg.config['options']['output'] = ' '.join(d.local for d in (env['output_files'] + env['upper_output_files']))
         logging.info('input: %r', cfg.config['options']['input'])
         logging.info('output: %r', cfg.config['options']['output'])
 
@@ -598,7 +598,7 @@ def toStr(a):
                     local=filename,
                     transfer=Transfer.TRUE,
                 )
-                env['input_files'].add(data)
+                env['input_files'].append(data)
                 self.infiles.add(data)
 
         # run the module

diff --git a/tests/core/exe_test.py b/tests/core/exe_test.py
@@ -67,8 +67,8 @@ def test_scope_env():
 
         # test parsing data
         with iceprod.core.exe.scope_env(c, t.dataset.config['tasks'][0], env) as tenv:
-            assert tenv['input_files'] == {Data('https://foo.bar/baz', 'baz', Transfer.TRUE)}
-        assert env['input_files'] == set()
+            assert tenv['input_files'] == [Data('https://foo.bar/baz', 'baz', Transfer.TRUE)]
+        assert env['input_files'] == []
 
 
 def test_download_data():