JCF: Issue #361: very rough draft of a fddatautilities build; works l…

…ocally on daq.fnal.gov

fddatautilities is an umbrella package which depends on a couple of
far-detector-focused packages as well as the core set of packages
(currently named "dunedaq" as it has for a couple of years, but soon
to be renamed "coredaq"). dunedaq has been given a build variant
called "subset" which when it assumes the value of "datautilities"
will only depend on the subset of core packages which fddatautilities
needs.

Note that it may make sense to have the variant value match up
perfectly with the name of the target package it's intended to
support.

In order to not try and solve everything at once, the package.py files
for the umbrella packages needed for this build are located in a
temporary subdirectory called issue361_dev_files. With this
subdirectory I can sidestep worrying about the make-release-repo.py
algorithm whenever I want to modify and test the relevant package.py
files for the umbrella packages during the development process.

I've successfully build fddatautilities using the files in this commit
via calling build-release.sh manually inside of a container on daq.fnal.gov
which is volume-mounted to a v2.1 externals area.

.github/workflows/wf-setup-tools.sh

@@ -1,35 +1,32 @@
 
-if [[ -z $BASE_RELEASE_DIR || -z $DET_RELEASE_DIR || -z $OS ]]; then
-    echo "You need to define the release's BASE_RELEASE_DIR, DET_RELEASE_DIR and OS variables for this script to source correctly; returning..." >&2
+if [[ -z $CORE_RELEASE_DIR || -z $FULL_RELEASE_DIR || -z $OS ]]; then
+    echo "You need to define the release's CORE_RELEASE_DIR, FULL_RELEASE_DIR and OS variables for this script to source correctly; returning..." >&2
     return 1
 fi
 
-if [[ $BASE_RELEASE_DIR =~ "_DEV_" ]]; then
-    export RELEASE_TYPE="nightly"
-elif [[ $BASE_RELEASE_DIR =~ "_PROD4_" ]]; then
-    export RELEASE_TYPE="production_v4"
-elif [[ $BASE_RELEASE_DIR =~ "/candidate" ]]; then
-    export RELEASE_TYPE="candidate"
-elif [[ $BASE_RELEASE_DIR =~ "/release" ]]; then
+if [[ $CORE_RELEASE_DIR =~ "/release" ]]; then
     export RELEASE_TYPE="frozen"
+elif [[ $CORE_RELEASE_DIR =~ "/candidate" ]]; then
+    export RELEASE_TYPE="candidate"
+elif [[ $CORE_RELEASE_DIR =~ "_PROD4_" ]]; then
+    export RELEASE_TYPE="production_v4"
 else
-    echo "Provided BASE_RELEASE_DIR \"${BASE_RELEASE_DIR}\" appears nonstandard and cannot be parsed; returning..." >&2
-    return 10
+    export RELEASE_TYPE="nightly"
 fi
 
-echo "Deduced release type \"${RELEASE_TYPE}\" from the name of the base release directory"
+echo "Deduced release type \"${RELEASE_TYPE}\" from the name of the core release directory"
 
-export BASE_RELEASE_TAG=$( basename $BASE_RELEASE_DIR )
-echo "Assuming base release tag is $BASE_RELEASE_TAG (i.e. the same name as the lowest-level directory in the path ${BASE_RELEASE_DIR})"
+export CORE_RELEASE_TAG=$( basename $CORE_RELEASE_DIR )
+echo "Assuming core release tag is $CORE_RELEASE_TAG (i.e. the same name as the lowest-level directory in the path ${CORE_RELEASE_DIR})"
 
-export DET_RELEASE_TAG=$( basename $DET_RELEASE_DIR )
-echo "Assuming detector release tag is $DET_RELEASE_TAG (i.e. the same name as the lowest-level directory in the path ${DET_RELEASE_DIR})"
+export FULL_RELEASE_TAG=$( basename $FULL_RELEASE_DIR )
+echo "Assuming full release tag is $FULL_RELEASE_TAG (i.e. the same name as the lowest-level directory in the path ${FULL_RELEASE_DIR})"
 
 if [[ $OS == almalinux9 && $RELEASE_TYPE == production_v4 ]]; then  # Alma9 v4 production nightly, externals v2.0
     export EXT_VERSION=v2.0
 elif [[ $OS == almalinux9 && $RELEASE_TYPE == nightly ]]; then      # Alma9 v5 development nightly, externals v2.1
     export EXT_VERSION=v2.1
-elif [[ $OS == almalinux9 && $DET_RELEASE_TAG =~ "v4." ]]; then        # Alma9 v4 candidate or frozen, externals v2.0
+elif [[ $OS == almalinux9 && $FULL_RELEASE_TAG =~ "v4." ]]; then        # Alma9 v4 candidate or frozen, externals v2.0
     export EXT_VERSION=v2.0
 elif [[ $OS == almalinux9 ]]; then                                  # Alma9 v5 candidate or frozen, externals v2.1 
     export EXT_VERSION=v2.1
@@ -45,8 +42,8 @@ echo "Using externals version $EXT_VERSION"
 export SPACK_VERSION=0.20.0
 export SPACK_EXTERNALS=/cvmfs/dunedaq.opensciencegrid.org/spack/externals/ext-${EXT_VERSION}
 
-export DET_SPACK_AREA=$DET_RELEASE_DIR
-export BASE_SPACK_AREA=$BASE_RELEASE_DIR
+export FULL_SPACK_AREA=$FULL_RELEASE_DIR
+export CORE_SPACK_AREA=$CORE_RELEASE_DIR
 
 function get_spack() {
 
@@ -65,8 +62,8 @@ function daqify_spack_environment() {
 
    release_level=$1
 
-   if [[ -z $release_level || ("$release_level" != "base" && "$release_level" != "det") ]]; then
-       echo "You need to pass daqify_spack_environment either \"base\" or \"det\" to specify whether to set up for a base release or a detector release" >&2
+   if [[ -z $release_level || ("$release_level" != "core" && "$release_level" != "full") ]]; then
+       echo "You need to pass daqify_spack_environment either \"core\" or \"full\" to specify whether to set up for a core release or a full release" >&2
       return 4 
    fi
 
@@ -86,11 +83,11 @@ function daqify_spack_environment() {
    echo "*********** spack arch ************ "
    spack arch
 
-   if [[ "$release_level" == "base" ]]; then
+   if [[ "$release_level" == "core" ]]; then
 
    cat <<EOF > $SPACK_ROOT/etc/spack/defaults/repos.yaml 
 repos:
-  - ${BASE_SPACK_AREA}/spack-${SPACK_VERSION}/spack-repo
+  - ${CORE_SPACK_AREA}/spack-${SPACK_VERSION}/spack-repo
   - ${SPACK_EXTERNALS}/spack-${SPACK_VERSION}/spack-repo-externals
   - \$spack/var/spack/repos/builtin
 EOF
@@ -103,20 +100,20 @@ upstreams:
     install_tree: ${SPACK_EXTERNALS}/spack-${SPACK_VERSION}/opt/spack
 EOF
 
-    elif [[ "$release_level" == "det" ]]; then
+    else
 
    cat <<EOF > $SPACK_ROOT/etc/spack/defaults/repos.yaml 
 repos:
-  - ${DET_SPACK_AREA}/spack-${SPACK_VERSION}/spack-repo
-  - ${BASE_SPACK_AREA}/spack-${SPACK_VERSION}/spack-repo
+  - ${FULL_SPACK_AREA}/spack-${SPACK_VERSION}/spack-repo
+  - ${CORE_SPACK_AREA}/spack-${SPACK_VERSION}/spack-repo
   - ${SPACK_EXTERNALS}/spack-${SPACK_VERSION}/spack-repo-externals
   - \$spack/var/spack/repos/builtin
 EOF
 
    cat <<EOF  >> $SPACK_ROOT/etc/spack/defaults/upstreams.yaml  
 upstreams:
-  ${BASE_RELEASE_TAG}:
-    install_tree: ${BASE_SPACK_AREA}/spack-${SPACK_VERSION}/opt/spack
+  ${CORE_RELEASE_TAG}:
+    install_tree: ${CORE_SPACK_AREA}/spack-${SPACK_VERSION}/opt/spack
   spack-externals:
     install_tree: ${SPACK_EXTERNALS}/spack-${SPACK_VERSION}/opt/spack
 EOF
@@ -125,10 +122,10 @@ EOF
 
     spack repo list
 
-    if [[ "$release_level" == "base" ]]; then
-        SPACK_AREA=$BASE_SPACK_AREA
+    if [[ "$release_level" == "core" ]]; then
+        SPACK_AREA=$CORE_SPACK_AREA
     else
-        SPACK_AREA=$DET_SPACK_AREA
+        SPACK_AREA=$FULL_SPACK_AREA
     fi
 
     cp $SPACK_EXTERNALS/spack-${SPACK_VERSION}/etc/spack/defaults/linux/compilers.yaml \
@@ -142,42 +139,41 @@ EOF
     sed -i 's/host_compatible: true/host_compatible: false/g' $SPACK_ROOT/etc/spack/defaults/concretizer.yaml
 }
 
-function get_release_yaml() {
+# get_release_yaml will either take "core" as an argument and deduce
+# the needed release.yaml file, or it will take the type of the full
+# release and do it (e.g., "nddaq")
 
-    release_level=$1
+function get_release_yaml() {
 
-    if [[ -z $release_level || ("$release_level" != "base" && "$release_level" != "fd" && "$release_level" != "nd") ]]; then
-        echo "You need to pass \"get_release_yaml\" either \"base\", \"fd\" or \"nd\" to get it to find the release's YAML file. Returning..." >&2
-        return 1
-    fi
+    release_info=$1
 
     version=""
     if [[ $RELEASE_TYPE == "candidate" || $RELEASE_TYPE == "frozen" ]]; then
-        if [[ $release_level == "base" ]]; then
-            version=$( echo $BASE_RELEASE_TAG | sed -r 's/.*(v[0-9]+\.[0-9]+\.[0-9]+).*/\1/' )  
+        if [[ $release_info == "core" ]]; then
+            version=$( echo $CORE_RELEASE_TAG | sed -r 's/.*(v[0-9]+\.[0-9]+\.[0-9]+).*/\1/' )  
         else
-            version=$( echo $DET_RELEASE_TAG | sed -r 's/.*(v[0-9]+\.[0-9]+\.[0-9]+).*/\1/' )
+            version=$( echo $FULL_RELEASE_TAG | sed -r 's/.*(v[0-9]+\.[0-9]+\.[0-9]+).*/\1/' )
 	fi
     fi
 
-    if [[ $release_level == "base" ]]; then
-
-        if [[ $RELEASE_TYPE == "nightly" ]]; then
-            echo -n "configs/dunedaq/dunedaq-develop/release.yaml"
-        elif [[ $RELEASE_TYPE == "production_v4" ]]; then
-            echo -n "configs/dunedaq/dunedaq-production_v4/release.yaml"
-	elif [[ $RELEASE_TYPE == "candidate" || $RELEASE_TYPE == "frozen" ]]; then
+    if [[ $release_info == "core" ]]; then
+
+	if [[ $RELEASE_TYPE == "candidate" || $RELEASE_TYPE == "frozen" ]]; then
             echo -n "configs/dunedaq/dunedaq-${version}/release.yaml"
+	elif [[ $RELEASE_TYPE == "production_v4" ]]; then
+            echo -n "configs/dunedaq/dunedaq-production_v4/release.yaml"
+	else
+	    echo -n "configs/dunedaq/dunedaq-develop/release.yaml"
         fi
 
     else
 
-        if [[ $RELEASE_TYPE == "nightly" ]]; then
-            echo -n "configs/${release_level}daq/${release_level}daq-develop/release.yaml"
-        elif [[ $RELEASE_TYPE == "production_v4" ]]; then
-            echo -n "configs/${release_level}daq/${release_level}daq-production_v4/release.yaml"
-        elif [[ $RELEASE_TYPE == "candidate" || $RELEASE_TYPE == "frozen" ]]; then
-            echo -n "configs/${release_level}daq/${release_level}daq-${version}/release.yaml"
+	if [[ $RELEASE_TYPE == "candidate" || $RELEASE_TYPE == "frozen" ]]; then
+            echo -n "configs/${release_info}/${release_info}-${version}/release.yaml"
+	elif [[ $RELEASE_TYPE == "production_v4" ]]; then
+            echo -n "configs/${release_info}/${release_info}-production_v4/release.yaml"
+        else
+            echo -n "configs/${release_info}/${release_info}-develop/release.yaml"
         fi
     fi
 }

configs/fddatautilities/fddatautilities-develop/dbt-build-order.cmake

@@ -0,0 +1,77 @@
+# "build_order" lists packages in the order you'd want CMake to see
+# them (via "add_subdirectory") during a simultaneous build. This is
+# due to their dependencies: e.g., you'd want CMake to see
+# daq-buildtools first in order to create daq-buildtoolsConfig.cmake
+# so "find_package(daq-builtools)" will work for all the other
+# packages, and so on. If a new package is introduced into the
+# development area, the developer is encouraged to add it to its
+# appropriate place in this list
+
+
+set(build_order "daq-cmake"
+                "ers"
+                "erskafka"
+                "logging"
+                "opmonlib"
+                "cmdlib"
+                "rcif"
+                "restcmd"
+                "utilities"
+                "ipm"
+                "serialization"
+                "iomanager"
+		"okssystem"
+		"oksdbinterfaces"
+		"oks"
+		"genconfig"
+		"oksutils"
+		"dal"
+		"coredal"
+                "appdal"
+		"oksconfig"
+		"dbe"
+                "appfwk"
+		"hermesmodules"
+                "daqconf"
+                "fddaqconf"
+                "nddaqconf"
+                "listrev"
+                "detdataformats"
+		"trgdataformats"
+                "daqdataformats"
+                "detchannelmaps"
+                "dfmessages"
+                "triggeralgs"
+                "timing"
+                "timinglibs"
+                "hdf5libs"
+                "trgtools"
+                "readoutlibs"
+                "hsilibs"
+                "readoutmodules"
+                "trigger"
+                "dfmodules"
+                "kafkaopmon"
+		# FD packages
+                "fddetdataformats"
+                "fdreadoutlibs"
+                "fdreadoutmodules"
+                "tpgtools"
+                "flxlibs"
+                "dqm"
+                "wibmod"
+                "sspmodules"
+                "uhallibs"
+                "rawdatautils"
+                "dqmtools"
+                "ctbmodules"
+                "dpdklibs"
+		"snbmodules"
+		"daqsystemtest"
+		# ND packages
+                "nddetdataformats"
+                "ndreadoutlibs"
+                "ndreadoutmodules"
+                "lbrulibs"
+)
+