updated and new settings to support accelerator build arg

README.md

@@ -645,17 +645,24 @@ scontrol_command = /usr/bin/scontrol
 #### `[submitted_job_comments]` section
 
 The `[submitted_job_comments]` section specifies templates for messages about newly submitted jobs.
-```
-initial_comment = New job on instance `{app_name}` for architecture `{arch_name}` for repository `{repo_id}` in job dir `{symlink}`
-```
-`initial_comment` is used to create a comment to a PR when a new job has been created.
-
 ```
 awaits_release = job id `{job_id}` awaits release by job manager
 ```
 `awaits_release` is used to provide a status update of a job (shown as a row in the job's status
 table).
 
+```
+initial_comment = New job on instance `{app_name}` for architecture `{arch_name}`{accelerator_spec} for repository `{repo_id}` in job dir `{symlink}`
+```
+`initial_comment` is used to create a comment to a PR when a new job has been
+created. Note, the part '{accelerator_spec}' is only filled-in by the bot if the
+argument 'accelerator' to the `bot: build` command have been used.
+```
+with_accelerator =  " and accelerator {accelerator}"
+```
+`with_accelerator` is used to provide information about the accelerator the job
+should build for if and only if the argument `accelerator:X/Y` has been provided.
+
 #### `[new_job_comments]` section
 
 The `[new_job_comments]` section sets templates for messages about jobs whose `hold` flag was released.

app.cfg.example

@@ -243,8 +243,9 @@ scontrol_command = /usr/bin/scontrol
 #         are removed, the output (in PR comments) will lack important
 #         information.
 [submitted_job_comments]
-initial_comment = New job on instance `{app_name}` for architecture `{arch_name}` for repository `{repo_id}` in job dir `{symlink}`
 awaits_release = job id `{job_id}` awaits release by job manager
+initial_comment = New job on instance `{app_name}` for CPU micro-architecture `{arch_name}`{accelerator_spec} for repository `{repo_id}` in job dir `{symlink}`
+with_accelerator = " and accelerator {accelerator}"
 
 
 [new_job_comments]

tasks/build.py

@@ -33,6 +33,7 @@
 # Local application imports (anything from EESSI/eessi-bot-software-layer)
 from connections import github
 from tools import config, cvmfs_repository, job_metadata, pr_comments, run_cmd
+import tools.filter as tools_filter
 
 
 # defaults (used if not specified via, eg, 'app.cfg')
@@ -46,7 +47,7 @@
 _ERROR_NONE = "none"
 
 
-Job = namedtuple('Job', ('working_dir', 'arch_target', 'repo_id', 'slurm_opts', 'year_month', 'pr_id'))
+Job = namedtuple('Job', ('working_dir', 'arch_target', 'repo_id', 'slurm_opts', 'year_month', 'pr_id', 'accelerator'))
 
 # global repo_cfg
 repo_cfg = {}
@@ -474,6 +475,12 @@ def prepare_jobs(pr, cfg, event_info, action_filter):
     #      call to just before download_pr
     year_month, pr_id, run_dir = create_pr_dir(pr, cfg, event_info)
 
+    accelerator = "none"
+    # determine accelerator from action_filter argument
+    accelerators = action_filter.get_filter_by_component(tools_filter.FILTER_COMPONENT_ACCEL)
+    if len(accelerators) > 0:
+        accelerator = accelerators[0]
+
     jobs = []
     for arch, slurm_opt in arch_map.items():
         arch_dir = arch.replace('/', '_')
@@ -501,6 +508,15 @@ def prepare_jobs(pr, cfg, event_info, action_filter):
                     continue
                 else:
                     log(f"{fn}(): context DOES satisfy filter(s), going on with job")
+            # we reached this point when the filter matched (otherwise we
+            # 'continue' with the next repository)
+            # for each match of the filter we create a specific job directory
+            #   however, matching CPU architectures works differently to handling
+            #   accelerators; multiple CPU architectures defined in arch_target_map
+            #   can match the (CPU) architecture component of a filter; in
+            #   contrast, the value of the accelerator filter is just passed down
+            #   to scripts in bot/ directory of the pull request (see function
+            #   prepare_job_cfg and creation of Job tuple below)
             job_dir = os.path.join(run_dir, arch_dir, repo_id)
             os.makedirs(job_dir, exist_ok=True)
             log(f"{fn}(): job_dir '{job_dir}'")
@@ -514,10 +530,12 @@ def prepare_jobs(pr, cfg, event_info, action_filter):
             cpu_target = '/'.join(arch.split('/')[1:])
             os_type = arch.split('/')[0]
             log(f"{fn}(): arch = '{arch}' => cpu_target = '{cpu_target}' , os_type = '{os_type}'")
-            prepare_job_cfg(job_dir, build_env_cfg, repocfg, repo_id, cpu_target, os_type)
+
+            log(f"{fn}(): accelerator = '{accelerator}'")
+            prepare_job_cfg(job_dir, build_env_cfg, repocfg, repo_id, cpu_target, os_type, accelerator)
 
             # enlist jobs to proceed
-            job = Job(job_dir, arch, repo_id, slurm_opt, year_month, pr_id)
+            job = Job(job_dir, arch, repo_id, slurm_opt, year_month, pr_id, accelerator)
             jobs.append(job)
 
     log(f"{fn}(): {len(jobs)} jobs to proceed after applying white list")
@@ -527,7 +545,7 @@ def prepare_jobs(pr, cfg, event_info, action_filter):
     return jobs
 
 
-def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir, os_type):
+def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir, os_type, accelerator):
     """
     Set up job configuration file 'job.cfg' in directory <job_dir>/cfg
 
@@ -538,6 +556,7 @@ def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir,
         repo_id (string): identifier of the repository to build for
         software_subdir (string): software subdirectory to build for (e.g., 'x86_64/generic')
         os_type (string): type of the os (e.g., 'linux')
+        accelerator (string): defines accelerator to build for (e.g., 'nvidia/cc80')
 
     Returns:
         None (implicitly)
@@ -563,6 +582,7 @@ def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir,
     # [architecture]
     # software_subdir = software_subdir
     # os_type = os_type
+    # accelerator = accelerator
     job_cfg = configparser.ConfigParser()
     job_cfg[job_metadata.JOB_CFG_SITE_CONFIG_SECTION] = {}
     build_env_to_job_cfg_keys = {
@@ -607,6 +627,7 @@ def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir,
     job_cfg[job_cfg_arch_section] = {}
     job_cfg[job_cfg_arch_section][job_metadata.JOB_CFG_ARCHITECTURE_SOFTWARE_SUBDIR] = software_subdir
     job_cfg[job_cfg_arch_section][job_metadata.JOB_CFG_ARCHITECTURE_OS_TYPE] = os_type
+    job_cfg[job_cfg_arch_section][job_metadata.JOB_CFG_ARCHITECTURE_ACCELERATOR] = accelerator
 
     # copy contents of directory containing repository configuration to directory
     # containing job configuration/metadata
@@ -726,11 +747,19 @@ def create_pr_comment(job, job_id, app_name, pr, gh, symlink):
     # obtain arch from job.arch_target which has the format OS/ARCH
     arch_name = '-'.join(job.arch_target.split('/')[1:])
 
+    submitted_job_comments_cfg = config.read_config()[config.SECTION_SUBMITTED_JOB_COMMENTS]
+
+    # obtain accelerator from job.accelerator
+    accelerator = job.accelerator
+    accelerator_spec_str = ''
+    if not accelerator is 'none':
+        accelerator_spec = f"{submitted_job_comments_cfg[config.SUBMITTED_JOB_COMMENTS_SETTING_WITH_ACCELERATOR]}"
+        accelerator_spec_str = accelerator_spec.format(accelerator=accelerator)
+
     # get current date and time
     dt = datetime.now(timezone.utc)
 
     # construct initial job comment
-    submitted_job_comments_cfg = config.read_config()[config.SECTION_SUBMITTED_JOB_COMMENTS]
     job_comment = (f"{submitted_job_comments_cfg[config.SUBMITTED_JOB_COMMENTS_SETTING_INITIAL_COMMENT]}"
                    f"\n|date|job status|comment|\n"
                    f"|----------|----------|------------------------|\n"
@@ -741,7 +770,8 @@ def create_pr_comment(job, job_id, app_name, pr, gh, symlink):
                        arch_name=arch_name,
                        symlink=symlink,
                        repo_id=job.repo_id,
-                       job_id=job_id)
+                       job_id=job_id,
+                       accelerator_spec=accelerator_spec_str)
 
     # create comment to pull request
     repo_name = pr.base.repo.full_name

tools/config.py

@@ -105,8 +105,9 @@
 RUNNING_JOB_COMMENTS_SETTING_RUNNING_JOB = 'running_job'
 
 SECTION_SUBMITTED_JOB_COMMENTS = 'submitted_job_comments'
-SUBMITTED_JOB_COMMENTS_SETTING_INITIAL_COMMENT = 'initial_comment'
 SUBMITTED_JOB_COMMENTS_SETTING_AWAITS_RELEASE = 'awaits_release'
+SUBMITTED_JOB_COMMENTS_SETTING_INITIAL_COMMENT = 'initial_comment'
+SUBMITTED_JOB_COMMENTS_SETTING_WITH_ACCELERATOR = 'with_accelerator'
 
 SECTION_CLEAN_UP = 'clean_up'
 CLEAN_UP_SETTING_TRASH_BIN_ROOT_DIR = 'trash_bin_dir'

tools/job_metadata.py

@@ -33,6 +33,7 @@
 JOB_CFG_ARCHITECTURE_SECTION = "architecture"
 JOB_CFG_ARCHITECTURE_OS_TYPE = "os_type"
 JOB_CFG_ARCHITECTURE_SOFTWARE_SUBDIR = "software_subdir"
+JOB_CFG_ARCHITECTURE_ACCELERATOR = "accelerator"
 
 JOB_CFG_REPOSITORY_SECTION = "repository"
 JOB_CFG_REPOSITORY_CONTAINER = "container"