From c508ba5ea1738dd278cea175a981a7df74cfff27 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 27 Nov 2023 10:43:21 +0100 Subject: [PATCH 1/7] Limit SYSUT allocations according to Cobol compiler requirements (#447) Signed-off-by: Dennis Behm --- languages/Cobol.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/languages/Cobol.groovy b/languages/Cobol.groovy index 1fdf48ad..1a31f7c7 100644 --- a/languages/Cobol.groovy +++ b/languages/Cobol.groovy @@ -197,7 +197,7 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe compile.dd(new DDStatement().name("SYSPRINT").options(props.cobol_printTempOptions)) compile.dd(new DDStatement().name("SYSMDECK").options(props.cobol_tempOptions)) - (1..17).toList().each { num -> + (1..15).toList().each { num -> compile.dd(new DDStatement().name("SYSUT$num").options(props.cobol_tempOptions)) } From 10effe3e18f966e592c33780a9083872ea66097e Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 27 Nov 2023 10:47:31 +0100 Subject: [PATCH 2/7] Document the zAppBuild and DBB Toolkit versions in build log and build result (#449) * Print zAppBuild version number * Add zAppBuild version and DBB toolkit version to build result Signed-off-by: Dennis Behm --- build.groovy | 16 +++++++++++++--- version.properties | 14 +++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/build.groovy b/build.groovy index 984298f7..039d1fdf 100644 --- a/build.groovy +++ b/build.groovy @@ -96,10 +96,17 @@ def initializeBuildProcess(String[] args) { // print and store property dbb toolkit version in use def dbbToolkitVersion = VersionInfo.getInstance().getVersion() - props.dbbToolkitVersion = dbbToolkitVersion def dbbToolkitBuildDate = VersionInfo.getInstance().getDate() - if (props.verbose) println "** zAppBuild running on DBB Toolkit Version ${dbbToolkitVersion} ${dbbToolkitBuildDate} " + props.dbbToolkitVersion = dbbToolkitVersion + props.dbbToolkitBuildDate = dbbToolkitBuildDate + File versionFile = new File("${props.zAppBuildDir}/version.properties") + if (versionFile.exists()) { + props.load(versionFile) + if (props.zappbuild_version) println "** Running zAppBuild Version ${props.zappbuild_version} " + } + if (props.verbose) println "** Running DBB Toolkit Version ${dbbToolkitVersion} ${dbbToolkitBuildDate} " + // verify required dbb toolkit buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, props.requiredDBBToolkitVersion) @@ -722,7 +729,10 @@ def finalizeBuildProcess(Map args) { buildResult.setProperty("filesProcessed", String.valueOf(args.count)) buildResult.setState(buildResult.COMPLETE) - + // add zAppBuild and DBB toolkit version info + if (props.zappbuild_version) buildResult.setProperty("zAppBuildVersion", props.zappbuild_version) + buildResult.setProperty("DBBToolkitVersion" , "${props.dbbToolkitVersion} ${props.dbbToolkitBuildDate}") + // store build result properties in BuildReport.json PropertiesRecord buildReportRecord = new PropertiesRecord("DBB.BuildResultProperties") def buildResultProps = buildResult.getPropertyNames() diff --git a/version.properties b/version.properties index 48f92f7d..b53aacdd 100644 --- a/version.properties +++ b/version.properties @@ -1 +1,13 @@ -version=3.5.0 \ No newline at end of file +# internal zAppbuild version number that is bumped up for each new release +# delivered via the public github repository +# https://github.com/IBM/dbb-zappbuild. +# Helps to understand from which version a copy was taken. +zappbuild_baseVersion=3.6.0 + +# use this property to add you own version suffix to indicate +# contributed enhancements and your own revisions +zappbuild_customVersion=BASE + +# zappbuild version string that is +# combining the base and custom version properties +zappbuild_version=${zappbuild_baseVersion}-${zappbuild_customVersion} \ No newline at end of file From 5454f881293d591b04f677f402c6fc2129c52572 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 12 Dec 2023 18:16:22 +0100 Subject: [PATCH 3/7] Handle exceptions thrown by the DBB toolkit (#448) * Catch exceptions thrown by the DBB toolkit and elegantly handle them * Update build exception handling in Transfer.groovy to stop the build * Handle assertion errors via exception handling * add truncation test case to test framework Signed-off-by: Dennis Behm --- build.groovy | 41 +++- languages/Transfer.groovy | 6 +- .../MortgageApplication/cobol/epsnbrvl.cbl | 186 ++++++++++++++++ .../MortgageApplication/test.properties | 13 ++ .../fullBuild_fileTruncation.groovy | 113 ++++++++++ utilities/BuildUtilities.groovy | 205 ++++++++++-------- 6 files changed, 457 insertions(+), 107 deletions(-) create mode 100644 test/applications/MortgageApplication/cobol/epsnbrvl.cbl create mode 100644 test/testScripts/fullBuild_fileTruncation.groovy diff --git a/build.groovy b/build.groovy index 039d1fdf..d36952f3 100644 --- a/build.groovy +++ b/build.groovy @@ -31,7 +31,15 @@ props.startTime = startTime.format("yyyyMMdd.HHmmss.SSS") println("\n** Build start at $props.startTime") // initialize build -initializeBuildProcess(args) +try { + initializeBuildProcess(args) +} catch ( AssertionError e ) { + String errorMsg = e.getMessage() + println(errorMsg) + props.error = "true" + buildUtils.updateBuildResult(errorMsg:errorMsg) + finalizeBuildProcess(start:startTime, 0) +} // create build list List buildList = new ArrayList() @@ -57,13 +65,21 @@ else { scriptPath = script // Use the ScriptMappings class to get the files mapped to the build script def buildFiles = ScriptMappings.getMappedList(script, buildList) - if (buildFiles.size() > 0) { - if (scriptPath.startsWith('/')) - runScript(new File("${scriptPath}"), ['buildList':buildFiles]) - else - runScript(new File("languages/${scriptPath}"), ['buildList':buildFiles]) + try { + if (buildFiles.size() > 0) { + if (scriptPath.startsWith('/')) + runScript(new File("${scriptPath}"), ['buildList':buildFiles]) + else + runScript(new File("languages/${scriptPath}"), ['buildList':buildFiles]) + } + processCounter = processCounter + buildFiles.size() + } catch (BuildException | AssertionError e ) { + String errorMsg = e.getMessage() + println(errorMsg) + props.error = "true" + buildUtils.updateBuildResult(errorMsg:errorMsg) + finalizeBuildProcess(start:startTime, count:processCounter) } - processCounter = processCounter + buildFiles.size() } } else if(props.scanLoadmodules && props.scanLoadmodules.toBoolean()){ println ("** Scanning load modules.") @@ -77,10 +93,6 @@ if (processCounter == 0) finalizeBuildProcess(start:startTime, count:processCounter) -// if error occurred signal process error -if (props.error) - System.exit(1) - // end script @@ -336,6 +348,8 @@ def populateBuildProperties(def opts) { // need to support IDz user build parameters if (opts.srcDir) props.workspace = opts.srcDir if (opts.wrkDir) props.outDir = opts.wrkDir + + // assert workspace buildUtils.assertBuildProperties('workspace,outDir') // load build.properties @@ -677,7 +691,6 @@ def createBuildList() { return buildList } - def finalizeBuildProcess(Map args) { println "***************** Finalization of the build process *****************" @@ -779,6 +792,10 @@ def finalizeBuildProcess(Map args) { if (props.preview) println("** Build ran in preview mode.") println("** Total files processed : ${args.count}") println("** Total build time : $duration\n") + + // if error occurred signal process error + if (props.error) + System.exit(1) } diff --git a/languages/Transfer.groovy b/languages/Transfer.groovy index e3c91446..0e9ce300 100644 --- a/languages/Transfer.groovy +++ b/languages/Transfer.groovy @@ -122,10 +122,8 @@ buildList.each { buildFile -> buildUtils.updateBuildResult(errorMsg:errorMsg) } } catch (BuildException e) { // Catch potential exceptions like file truncation - String errorMsg = "*! The CopyToPDS failed with an exception ${e.getMessage()}." - println(errorMsg) - props.error = "true" - buildUtils.updateBuildResult(errorMsg:errorMsg) + String errorMsg = "*! (Transfer.groovy) CopyToPDS of file ${buildFile} failed with an exception \n ${e.getMessage()}." + throw new BuildException(errorMsg) } } else { String errorMsg = "*! Target dataset for $buildFile could not be obtained from file properties. " diff --git a/test/applications/MortgageApplication/cobol/epsnbrvl.cbl b/test/applications/MortgageApplication/cobol/epsnbrvl.cbl new file mode 100644 index 00000000..ddc7219a --- /dev/null +++ b/test/applications/MortgageApplication/cobol/epsnbrvl.cbl @@ -0,0 +1,186 @@ + ID DIVISION. + PROGRAM-ID. EPSNBRVL + * THIS IS A CALLED PROGRAM EXAMPLE FOR DEMONSTRATION + * + * THIS PROGRAM WILL BE CALLED BY ANOTHER, RECEIVE + * THE FOLLOWING INFOMATION AND RETURN A MONTLY PAYMENT AMOUNT + * INPUT: + * ORIGINAL PRINCIPLE AMOUNT + * TRUNCATION TRUNCATION TRUNCATION TRUNCATION TRUNCATION TRUNCATION TRUNCATION TRUNCATION TRUNCATION + * NUMBER OF YEARS + * NUMBER OF MONTHS + * INTEREST RATE + * OUTPUT: + * MONTHLY PAYMENT + * + * (C) 2020 IBM + ENVIRONMENT DIVISION. + CONFIGURATION SECTION. + SOURCE-COMPUTER. FLEX-ES. + OBJECT-COMPUTER. FLEX-ES. + DATA DIVISION. + WORKING-STORAGE SECTION. + * + 01 WS-STATIC-DATA. + 03 STATIC-ERRORS. + 05 FILLER PIC 99 VALUE 1. + 05 FILLER PIC X(80) + VALUE 'NO NUMBER PRESENT'. + 05 FILLER PIC 99 VALUE 2. + 05 FILLER PIC X(80) + VALUE 'SPACES IN NUMBER'. + 05 FILLER PIC 99 VALUE 3. + 05 FILLER PIC X(80) + VALUE 'TOO MANY DEICMAL POINTS'. + 05 FILLER PIC 99 VALUE 4. + 05 FILLER PIC X(80) + VALUE 'YEARS INDICATED, BUT YEARS ZERO OR LESS'. + 05 FILLER PIC 99 VALUE 5. + 05 FILLER PIC X(80) + VALUE 'ZERO OR LESS MONTHS'. + 05 FILLER PIC 99 VALUE 6. + 05 FILLER PIC X(80) + VALUE ' '. + 05 FILLER PIC 99 VALUE 7. + 05 FILLER PIC X(80) + VALUE ' '. + 05 FILLER PIC 99 VALUE 8. + 05 FILLER PIC X(80) + VALUE ' '. + 05 FILLER PIC 99 VALUE 9. + 05 FILLER PIC X(80) + VALUE ' '. + 05 FILLER PIC 99 VALUE 10. + 05 FILLER PIC X(80) + VALUE ' '. + 03 STATIC-ERROR-TBL REDEFINES STATIC-ERRORS. + 05 STATIC-ERROR-TABLE OCCURS 10 TIMES. + 07 ERROR-INDICATOR PIC 99. + 07 ERROR-TEXT PIC X(80). + 01 WS-WORK-AMOUNTS. + 03 WS-LEADING-SPACES PIC 9(4) COMP VALUE 1. + 03 WS-TRAILING-SPACES PIC 9(4) COMP VALUE 0. + 03 WS-END-SPACE PIC 9(4) COMP VALUE 0. + 03 WS-DECIMAL-SPACE PIC 99 VALUE 0. + 03 WS-IDX PIC 9(2) COMP. + 03 WS-DEC-IDX PIC 9(2) COMP. + 03 WS-NUM-IDX PIC 9(2) COMP. + + 03 WS-MAX-NUMBER-LGTH PIC 9(2) COMP. + 03 WS-MAX-FIELD PIC 9(2) COMP. + 03 WS-DEC-ADJUST PIC 9. + + + LINKAGE SECTION. + * + COPY EPSNBRPM. + + PROCEDURE DIVISION USING EPS-NUMBER-VALIDATION. + * + A000-MAINLINE. + MOVE EPSPARM-MAX-LENGTH TO WS-IDX. + MOVE LENGTH OF EPSPARM-VALIDATE-DATA TO WS-MAX-FIELD + IF WS-IDX > WS-MAX-FIELD + MOVE WS-MAX-FIELD TO WS-IDX + ELSE + MOVE WS-IDX TO WS-MAX-FIELD + END-IF. + + MOVE ZERO TO WS-END-SPACE. + MOVE SPACES TO EPSPARM-RETURN-ERROR. + MOVE ZERO TO EPSPARM-BINARY-NUMBER + EPSPARM-NUMBER + EPSPARM-DECIMAL. + + * FIND TRAILING SPACES + PERFORM UNTIL WS-IDX = 0 + IF EPSPARM-VALIDATE-DATA(WS-IDX:1) = SPACES + ADD 1 TO WS-TRAILING-SPACES + SUBTRACT 1 FROM WS-IDX + ELSE + MOVE WS-IDX TO WS-END-SPACE + MOVE 0 TO WS-IDX + END-IF + END-PERFORM. + + * FIND LEADING SPACES + MOVE 1 TO WS-LEADING-SPACES. + + IF WS-END-SPACE NOT = 0 + MOVE 1 TO WS-IDX + PERFORM UNTIL WS-IDX >= WS-END-SPACE + IF EPSPARM-VALIDATE-DATA(WS-IDX:1) = SPACES + ADD 1 TO WS-LEADING-SPACES + ADD 1 TO WS-IDX + ELSE + COMPUTE WS-IDX = WS-END-SPACE + 1 + END-IF + END-PERFORM + ELSE + MOVE STATIC-ERROR-TABLE(1) TO EPSPARM-RETURN-ERROR + END-IF. + + MOVE WS-LEADING-SPACES TO WS-IDX. + MOVE 1 TO WS-DEC-IDX. + MOVE 0 TO WS-DECIMAL-SPACE. + + * FIND DECIMAL POINT + PERFORM A002-COMPUTE-DECIMAL + UNTIL WS-IDX > WS-END-SPACE + . + + IF WS-DECIMAL-SPACE > 0 + COMPUTE WS-END-SPACE = WS-DECIMAL-SPACE - 1 + END-IF. + + * VALIDATE NO INTERNAL BLANKS + MOVE WS-END-SPACE TO WS-IDX. + MOVE LENGTH OF EPSPARM-NUMBER TO WS-NUM-IDX. + * SUBTRACT 1 FROM WS-NUM-IDX. + + PERFORM A001-COMPUTE-INTEGER + UNTIL WS-IDX < WS-LEADING-SPACES + . + + IF EPSPARM-RETURN-ERROR = SPACES + COMPUTE EPSPARM-BINARY-NUMBER = EPSPARM-NUMBER + + EPSPARM-DECIMAL + END-IF. + GOBACK + . + + A001-COMPUTE-INTEGER. + IF EPSPARM-VALIDATE-DATA(WS-IDX:1) = ',' + SUBTRACT 1 FROM WS-IDX + ELSE + IF EPSPARM-VALIDATE-DATA(WS-IDX:1) = SPACE + OR EPSPARM-VALIDATE-DATA(WS-IDX:1) IS NOT NUMERIC + MOVE STATIC-ERROR-TABLE(2) TO EPSPARM-RETURN-ERROR + MOVE 0 TO WS-IDX + ELSE + MOVE EPSPARM-VALIDATE-DATA(WS-IDX:1) TO + EPSPARM-NUMBER(WS-NUM-IDX:1) + SUBTRACT 1 FROM WS-IDX + WS-NUM-IDX + END-IF + END-IF + . + + A002-COMPUTE-DECIMAL. + IF WS-DECIMAL-SPACE = 0 + IF EPSPARM-VALIDATE-DATA(WS-IDX:1) = '.' + MOVE WS-IDX TO WS-DECIMAL-SPACE + END-IF + ELSE + IF EPSPARM-VALIDATE-DATA(WS-IDX:1) = '.' + MOVE STATIC-ERROR-TABLE(3) TO EPSPARM-RETURN-ERROR + MOVE WS-END-SPACE TO WS-IDX + MOVE 1 TO WS-DEC-IDX + ELSE + MOVE EPSPARM-VALIDATE-DATA(WS-IDX:1) TO + EPSPARM-DECIMAL(WS-DEC-IDX:1) + ADD 1 TO WS-DEC-IDX + END-IF + END-IF + ADD 1 TO WS-IDX + . diff --git a/test/applications/MortgageApplication/test.properties b/test/applications/MortgageApplication/test.properties index be0c4c14..caafda57 100644 --- a/test/applications/MortgageApplication/test.properties +++ b/test/applications/MortgageApplication/test.properties @@ -8,6 +8,7 @@ test_testOrder=resetBuild.groovy,\ mergeBuild.groovy,\ fullBuild.groovy,\ fullBuild_languageConfigurations.groovy,\ +fullBuild_fileTruncation.groovy,\ impactBuild.groovy,\ impactBuild_preview.groovy,\ impactBuild_properties.groovy,\ @@ -42,6 +43,18 @@ fullBuild_expectedFilesBuilt = epsnbrvl.cbl,epscsmrt.cbl,epsmort.bms,epsmlist.ln # list of source datasets (LLQ) that should be deleted during fullBuild.groovy cleanUp fullBuild_datasetsToCleanUp = BMS,COBOL,LINK +############################# +# fullBuild_fileTruncation.groovy properties +############################# +# +# list of changed source files to create test scenario with truncation error +fullBuild_truncation_changedFiles = cobol/epsnbrvl.cbl + +# list of changed source files to create test scenario with truncation error +fullBuild_truncation_errorMsg = CopyToPDS of buildFile MortgageApplication/cobol/epsnbrvl.cbl failed with an exception :: cobol/epsnbrvl.cbl + +# list of source datasets (LLQ) that should be deleted during cleanUp +fullBuild_datasetsToCleanUp = BMS,COBOL,OBJ,LINK ############################### # impactBuild.groovy properties diff --git a/test/testScripts/fullBuild_fileTruncation.groovy b/test/testScripts/fullBuild_fileTruncation.groovy new file mode 100644 index 00000000..89aecd4d --- /dev/null +++ b/test/testScripts/fullBuild_fileTruncation.groovy @@ -0,0 +1,113 @@ +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript +import groovy.transform.* +import com.ibm.dbb.* +import com.ibm.dbb.build.* + +@Field BuildProperties props = BuildProperties.getInstance() +@Field def testUtils = loadScript(new File("../utils/testUtilities.groovy")) + +println "\n**************************************************************" +println "** Executing test script ${this.class.getName()}.groovy" +println "**************************************************************" + +// Get the DBB_HOME location +def dbbHome = EnvVars.getHome() +if (props.verbose) println "** DBB_HOME = ${dbbHome}" + +// Create full build command +def fullBuildCommand = [] +fullBuildCommand << "${dbbHome}/bin/groovyz" +fullBuildCommand << "${props.zAppBuildDir}/build.groovy" +fullBuildCommand << "--workspace ${props.workspace}" +fullBuildCommand << "--application ${props.app}" +fullBuildCommand << (props.outDir ? "--outDir ${props.outDir}" : "--outDir ${props.zAppBuildDir}/out") +fullBuildCommand << "--hlq ${props.hlq}" +fullBuildCommand << "--logEncoding UTF-8" +fullBuildCommand << (props.url ? "--url ${props.url}" : "") +fullBuildCommand << (props.id ? "--id ${props.id}" : "") +fullBuildCommand << (props.pw ? "--pw ${props.pw}" : "") +fullBuildCommand << (props.pwFile ? "--pwFile ${props.pwFile}" : "") +fullBuildCommand << (props.verbose ? "--verbose" : "") +fullBuildCommand << (props.propFiles ? "--propFiles ${props.propFiles}" : "") +fullBuildCommand << "--fullBuild" + +@Field def assertionList = [] + +def changedFiles = props.fullBuild_truncation_changedFiles.split(',') +PropertyMappings fullBuild_truncation_errorMsgs = new PropertyMappings('fullBuild_truncation_errorMsg') + +try { + + // Create full build command to set baseline + testUtils.runBaselineBuild() + + // test setup + changedFiles.each { changedFile -> + + println "\n** Running build test for changed file $changedFile" + + // update changed file in Git repo test branch + testUtils.copyAndCommit(changedFile) + + // run build + println "** Executing ${fullBuildCommand.join(" ")}" + outputStream = new StringBuffer() + process = ['bash', '-c', fullBuildCommand.join(" ")].execute() + process.waitForProcessOutput(outputStream, System.err) + + // validate build results + validateBuild(changedFile, fullBuild_truncation_errorMsgs, outputStream) + } +} +catch(AssertionError e) { + def result = e.getMessage() + assertionList << result; + props.testsSucceeded = 'false' +} +finally { + // report failures + if (assertionList.size()>0) { + println "\n***" + println "**START OF FULL BUILD TRUNCATION ERROR TEST RESULTS**\n" + println "*FAILED FULL BUILD TRUNCATION ERROR TEST RESULTS*\n" + assertionList + println "\n**END OF FULL BUILD TRUNCATION ERROR TEST RESULTS**" + println "***" + } + + // reset test branch + testUtils.resetTestBranch() + + // cleanup datasets + testUtils.cleanUpDatasets(props.fullBuild_datasetsToCleanUp) +} + + +//************************************************************* +// Method Definitions +//************************************************************* + +def validateBuild(String changedFile, PropertyMappings fullBuild_truncation_errorMsgs, StringBuffer outputStream) { + + println "** Validating full build truncation results" + def expectedlogMsg = fullBuild_truncation_errorMsgs.getValue(changedFile) + + try{ + // Validate that error state build is written to log + assert outputStream.contains("Build State : ERROR") : "*! FULL BUILD TRUNCATION ERROR FOR $changedFile\nOUTPUT STREAM:\n$outputStream\n" + + // Validate that truncation message is written to log + assert outputStream.contains(expectedlogMsg) : "*! FULL BUILD TRUNCATION ERROR for $changedFile\nOUTPUT STREAM:\n$outputStream\n" + + // Validate that Finalization process is executed + assert outputStream.contains("********* Finalization of the build process *****") : "*! FULL BUILD TRUNCATION ERROR FOR $changedFile\nOUTPUT STREAM:\n$outputStream\n" + + println "**" + println "** FULL BUILD TRUNCATION ERROR TEST : PASSED FOR $changedFile **" + println "**" + } + catch(AssertionError e) { + def result = e.getMessage() + assertionList << result; + props.testsSucceeded = 'false' + } +} diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy index 46e2885c..6c0727f5 100644 --- a/utilities/BuildUtilities.groovy +++ b/utilities/BuildUtilities.groovy @@ -42,10 +42,10 @@ def assertBuildProperties(String requiredProps) { */ def createFullBuildList() { Set buildSet = new HashSet() - + // PropertyMappings PropertyMappings githashBuildableFilesMap = new PropertyMappings("githashBuildableFilesMap") - + // create the list of build directories List srcDirs = [] if (props.applicationSrcDirs) @@ -55,13 +55,13 @@ def createFullBuildList() { dir = getAbsolutePath(dir) Set fileSet =getFileSet(dir, true, '**/*.*', props.excludeFileList) buildSet.addAll(fileSet) - + // capture abbreviated gitHash for all buildable files String abbrevHash = gitUtils.getCurrentGitHash(dir, true) buildSet.forEach { buildableFile -> githashBuildableFilesMap.addFilePattern(abbrevHash, buildableFile) } - + } return buildSet @@ -100,19 +100,24 @@ def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMap // only copy the build file once if (!copiedFileCache.contains(buildFile)) { copiedFileCache.add(buildFile) - new CopyToPDS().file(new File(getAbsolutePath(buildFile))) - .dataset(srcPDS) - .member(CopyToPDS.createMemberName(buildFile)) - .execute() + try { + new CopyToPDS().file(new File(getAbsolutePath(buildFile))) + .dataset(srcPDS) + .member(CopyToPDS.createMemberName(buildFile)) + .execute() + } catch (BuildException e) { // Catch potential exceptions like file truncation + String errorMsg = "*! (BuildUtilities.copySourceFiles) CopyToPDS of buildFile ${buildFile} failed with an exception \n ${e.getMessage()}." + throw new BuildException(errorMsg) + } } - + if (dependencyDatasetMapping && props.userBuildDependencyFile && props.userBuild) { if (props.verbose) println "*** User Build Dependency File Detected. Skipping DBB Dependency Resolution." // userBuildDependencyFile present (passed from the IDE) // Skip dependency resolution, extract dependencies from userBuildDependencyFile, and copy directly dataset // Load property mapping containing the map of targetPDS and dependencyfile PropertyMappings dependenciesDatasetMapping = new PropertyMappings(dependencyDatasetMapping) - + // parse JSON and validate fields of userBuildDependencyFile def depFileData = validateDependencyFile(buildFile, props.userBuildDependencyFile) @@ -145,26 +150,31 @@ def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMap zunitFileExtension = (props.zunit_playbackFileExtension) ? props.zunit_playbackFileExtension : null // get index of last '.' in file path to extract the file extension def extIndex = dependencyLoc.lastIndexOf('.') - if( zunitFileExtension && !zunitFileExtension.isEmpty() && (dependencyLoc.substring(extIndex).contains(zunitFileExtension))){ - new CopyToPDS().file(new File(dependencyLoc)) - .copyMode(CopyMode.BINARY) - .dataset(dependencyPDS) - .member(memberName) - .execute() - } - else - { - new CopyToPDS().file(new File(dependencyLoc)) - .dataset(dependencyPDS) - .member(memberName) - .execute() + try { + if( zunitFileExtension && !zunitFileExtension.isEmpty() && (dependencyLoc.substring(extIndex).contains(zunitFileExtension))){ + new CopyToPDS().file(new File(dependencyLoc)) + .copyMode(CopyMode.BINARY) + .dataset(dependencyPDS) + .member(memberName) + .execute() + } + else + { + new CopyToPDS().file(new File(dependencyLoc)) + .dataset(dependencyPDS) + .member(memberName) + .execute() + } + } catch (BuildException e) { // Catch potential exceptions like file truncation + String errorMsg = "*! (BuildUtilities.copySourceFiles) CopyToPDS of dependency ${dependencyLoc} failed with an exception ${e.getMessage()}." + throw new BuildException(errorMsg) } } } } else if (dependencyDatasetMapping && dependencyResolver) { // resolve the logical dependencies to physical files to copy to data sets - + List physicalDependencies = resolveDependencies(dependencyResolver, buildFile) if (props.verbose) println "*** Physical dependencies for $buildFile:" @@ -177,18 +187,18 @@ def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMap printPhysicalDependencies(physicalDependencies) } } - + physicalDependencies.each { physicalDependency -> // Write Physical Dependency details to log on verbose, not on formatConsoleOutput if (props.verbose && !(props.formatConsoleOutput && props.formatConsoleOutput.toBoolean())) println physicalDependency - + if (physicalDependency.isResolved()) { // obtain target dataset based on Mappings // Order : // 1. langprefix_dependenciesAlternativeLibraryNameMapping based on the library setting recognized by DBB (COBOL and PLI) - // 2. langprefix_dependenciesDatasetMapping as a manual overwrite to determine an alternative library used in the default dd concatentation - String dependencyPDS + // 2. langprefix_dependenciesDatasetMapping as a manual overwrite to determine an alternative library used in the default dd concatentation + String dependencyPDS if (!physicalDependency.getLibrary().equals("SYSLIB") && dependenciesAlternativeLibraryNameMapping) { dependencyPDS = props.getProperty(parseJSONStringToMap(dependenciesAlternativeLibraryNameMapping).get(physicalDependency.getLibrary())) } @@ -207,19 +217,23 @@ def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMap String memberName = CopyToPDS.createMemberName(physicalDependency.getFile()) //retrieve zUnitFileExtension plbck zunitFileExtension = (props.zunit_playbackFileExtension) ? props.zunit_playbackFileExtension : null - - if( zunitFileExtension && !zunitFileExtension.isEmpty() && ((physicalDependency.getFile().substring(physicalDependency.getFile().indexOf("."))).contains(zunitFileExtension))){ - new CopyToPDS().file(new File(physicalDependencyLoc)) - .copyMode(CopyMode.BINARY) - .dataset(dependencyPDS) - .member(memberName) - .execute() - } else - { - new CopyToPDS().file(new File(physicalDependencyLoc)) - .dataset(dependencyPDS) - .member(memberName) - .execute() + try { + if( zunitFileExtension && !zunitFileExtension.isEmpty() && ((physicalDependency.getFile().substring(physicalDependency.getFile().indexOf("."))).contains(zunitFileExtension))){ + new CopyToPDS().file(new File(physicalDependencyLoc)) + .copyMode(CopyMode.BINARY) + .dataset(dependencyPDS) + .member(memberName) + .execute() + } else + { + new CopyToPDS().file(new File(physicalDependencyLoc)) + .dataset(dependencyPDS) + .member(memberName) + .execute() + } + } catch (BuildException e) { // Catch potential exceptions like file truncation + String errorMsg = "*! (BuildUtilities.copySourceFiles) CopyToPDS of dependency ${physicalDependencyLoc} failed with an exception \n ${e.getMessage()}." + throw new BuildException(errorMsg) } } } else { @@ -321,9 +335,9 @@ def updateBuildResult(Map args) { */ def createLogicalFile(SearchPathDependencyResolver spDependencyResolver, String buildFile) { - + LogicalFile logicalFile - + if (props.resolveSubsystems && props.resolveSubsystems.toBoolean()) { // include resolved dependencies to define file flags of logicalFile logicalFile = spDependencyResolver.resolveSubsystems(buildFile,props.workspace) @@ -416,7 +430,7 @@ def isMQ(LogicalFile logicalFile) { */ def getMqStubInstruction(LogicalFile logicalFile) { String mqStubInstruction - + if (isMQ(logicalFile)) { // https://www.ibm.com/docs/en/ibm-mq/9.3?topic=files-mq-zos-stub-programs if (isCICS(logicalFile)) { @@ -427,9 +441,9 @@ def getMqStubInstruction(LogicalFile logicalFile) { mqStubInstruction = " INCLUDE SYSLIB(CSQBSTUB)\n" } } else { - println("*! (BuildUtilities.getMqStubInstruction) MQ file attribute for ${logicalFile.getFile()} is false.") + println("*! (BuildUtilities.getMqStubInstruction) MQ file attribute for ${logicalFile.getFile()} is false.") } - + return mqStubInstruction } @@ -570,10 +584,10 @@ def getDeployType(String langQualifier, String buildFile, LogicalFile logicalFil * Creates a Generic PropertyRecord with the provided db2 information in bind.properties */ def generateDb2InfoRecord(String buildFile){ - + // New Generic Property Record PropertiesRecord db2BindInfo = new PropertiesRecord("db2BindInfo:${buildFile}") - + // Link to buildFile db2BindInfo.addProperty("file", buildFile) @@ -586,8 +600,8 @@ def generateDb2InfoRecord(String buildFile){ if (bindPropertyValue != null ) db2BindInfo.addProperty("${db2Prop}",bindPropertyValue) } } - - return db2BindInfo + + return db2BindInfo } /* @@ -596,12 +610,20 @@ def generateDb2InfoRecord(String buildFile){ */ def validateDependencyFile(String buildFile, String depFilePath) { String[] allowedEncodings = ["UTF-8", "IBM-1047"] - String[] reqDepFileProps = ["fileName", "isCICS", "isSQL", "isDLI", "isMQ", "dependencies", "schemaVersion"] + String[] reqDepFileProps = [ + "fileName", + "isCICS", + "isSQL", + "isDLI", + "isMQ", + "dependencies", + "schemaVersion" + ] depFilePath = getAbsolutePath(depFilePath) // Load dependency file and verify existance File depFile = new File(depFilePath) assert depFile.exists() : "*! Dependency file not found: ${depFile.getAbsolutePath()}" - + // Parse the JSON file String encoding = retrieveHFSFileEncoding(depFile) // Determine the encoding from filetag JsonSlurper slurper = new JsonSlurper().setType(JsonParserType.INDEX_OVERLAY) // Use INDEX_OVERLAY, fastest parser @@ -616,7 +638,7 @@ def validateDependencyFile(String buildFile, String depFilePath) { depFileData = slurper.parse(depFile) // Assume default encoding for system } if (props.verbose) println new JsonBuilder(depFileData).toPrettyString() // Pretty print if verbose - + // Validate JSON structure reqDepFileProps.each { depFileProp -> assert depFileData."${depFileProp}" != null : "*! Missing required dependency file field '$depFileProp'" @@ -650,8 +672,8 @@ def assertDbbBuildToolkitVersion(String currentVersion, String requiredVersion){ assert (label as int) >= ((requiredVersionList[i]) as int) : "Current DBB Toolkit Version $currentVersion does not meet the minimum required version $requiredVersion. EXIT." if (label > requiredVersionList[i]) foundValidVersion = true } - - } + + } } catch(AssertionError e) { println "Current DBB Toolkit Version $currentVersion does not meet the minimum required version $requiredVersion. EXIT." @@ -666,12 +688,12 @@ def assertDbbBuildToolkitVersion(String currentVersion, String requiredVersion){ */ def retrieveHFSFileEncoding(File file) { FileAttribute.Stat stat = FileAttribute.getStat(file.getAbsolutePath()) - FileAttribute.Tag tag = stat.getTag() + FileAttribute.Tag tag = stat.getTag() int i = 0 if (tag != null) { - char x = tag.getCodeCharacterSetID() - i = (int) x + char x = tag.getCodeCharacterSetID() + i = (int) x } switch(i) { @@ -679,7 +701,7 @@ def retrieveHFSFileEncoding(File file) { case 1208: return "UTF-8" default: return "IBM-${i}" } - + } /* @@ -689,7 +711,7 @@ def retrieveHFSFileEncoding(File file) { // def printResolutionRules(List rules) { // println("*** Configured resulution rules:") - + // // Print header of table // println(" " + "Library".padRight(10) + "Category".padRight(12) + "SourceDir/File".padRight(50) + "Directory".padRight(36) + "Collection".padRight(24) + "Archive".padRight(20)) // println(" " + " ".padLeft(10,"-") + " ".padLeft(12,"-") + " ".padLeft(50,"-") + " ".padLeft(36,"-") + " ".padLeft(24,"-") + " ".padLeft(20,"-")) @@ -812,38 +834,38 @@ def matches(String file, List pathMatchers) { */ def generateIdentifyStatement(String buildFile, String dsProperty) { - def String identifyStmt + def String identifyStmt int maxRecordLength = dsProperty.toLowerCase().contains("library") ? 80 : 40 - - if((props.mergeBuild || props.impactBuild || props.fullBuild) && MetadataStoreFactory.getMetadataStore() != null) { - - String member = CopyToPDS.createMemberName(buildFile) - String shortGitHash = getShortGitHash(buildFile) - - if (shortGitHash != null) { - - String identifyString = props.application + "/" + shortGitHash - // IDENTIFY EPSCSMRT('MortgageApplication/abcabcabc') - identifyStmt = " " + "IDENTIFY ${member}(\'$identifyString\')" - if (identifyString.length() > maxRecordLength) { - String errorMsg = "*!* BuildUtilities.generateIdentifyStatement() - Identify string exceeds $maxRecordLength chars: identifyStmt=$identifyStmt" - println(errorMsg) - props.error = "true" - updateBuildResult(errorMsg:errorMsg) - return null - } else { - return identifyStmt - } + + if((props.mergeBuild || props.impactBuild || props.fullBuild) && MetadataStoreFactory.getMetadataStore() != null) { + + String member = CopyToPDS.createMemberName(buildFile) + String shortGitHash = getShortGitHash(buildFile) + + if (shortGitHash != null) { + + String identifyString = props.application + "/" + shortGitHash + // IDENTIFY EPSCSMRT('MortgageApplication/abcabcabc') + identifyStmt = " " + "IDENTIFY ${member}(\'$identifyString\')" + if (identifyString.length() > maxRecordLength) { + String errorMsg = "*!* BuildUtilities.generateIdentifyStatement() - Identify string exceeds $maxRecordLength chars: identifyStmt=$identifyStmt" + println(errorMsg) + props.error = "true" + updateBuildResult(errorMsg:errorMsg) + return null } else { - println("*!* BuildUtilities.generateIdentifyStatement() - Could not obtain abbreviated git hash for $buildFile") - return null - } + return identifyStmt + } + } else { + println("*!* BuildUtilities.generateIdentifyStatement() - Could not obtain abbreviated git hash for $buildFile") + return null + } - } else { - return null - } - } + } else { + return null + } +} /** * method to print the logicalFile attributes (CICS, SQL, DLI, MQ) of a scanned file @@ -860,16 +882,16 @@ def generateIdentifyStatement(String buildFile, String dsProperty) { * This is implementing * https://github.com/IBM/dbb-zappbuild/issues/339 * -*/ + */ def printLogicalFileAttributes(LogicalFile logicalFile) { String cicsFlag = (logicalFile.isCICS() == isCICS(logicalFile)) ? "${logicalFile.isCICS()}" : "${isCICS(logicalFile)}*" String sqlFlag = (logicalFile.isSQL() == isSQL(logicalFile)) ? "${logicalFile.isSQL()}" : "${isSQL(logicalFile)}*" String dliFlag = (logicalFile.isDLI() == isDLI(logicalFile)) ? "${logicalFile.isDLI()}" : "${isDLI(logicalFile)}*" String mqFlag = (logicalFile.isMQ() == isMQ(logicalFile)) ? "${logicalFile.isMQ()}" : "${isMQ(logicalFile)}*" - + println "Program attributes: CICS=$cicsFlag, SQL=$sqlFlag, DLI=$dliFlag, MQ=$mqFlag" - + } /** @@ -902,3 +924,4 @@ def isGeneratedzUnitTestCaseProgram(String buildFile) { return false } + From f9257a10d765f1ac8ced1adb0a3d55e025327483 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 13 Dec 2023 23:57:46 +0100 Subject: [PATCH 4/7] Additional configuration options for IMS batch and online programs (#454) * Additional configuration options for IMS batch and online programs * PLI IMS compile parts Signed-off-by: Dennis Behm --- languages/Assembler.groovy | 3 ++ languages/Cobol.groovy | 3 ++ languages/LinkEdit.groovy | 9 ++-- languages/PLI.groovy | 7 +++ samples/application-conf/Assembler.properties | 4 ++ samples/application-conf/Cobol.properties | 4 ++ samples/application-conf/LinkEdit.properties | 7 +++ samples/application-conf/PLI.properties | 5 ++ samples/application-conf/README.md | 17 +++++-- samples/application-conf/file.properties | 46 +++++++++++++++++-- utilities/BuildUtilities.groovy | 20 +++++++- 11 files changed, 112 insertions(+), 13 deletions(-) diff --git a/languages/Assembler.groovy b/languages/Assembler.groovy index 0a22f10b..0f712532 100644 --- a/languages/Assembler.groovy +++ b/languages/Assembler.groovy @@ -458,6 +458,9 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb if (buildUtils.isCICS(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDFHLOAD).options("shr")) + if (buildUtils.isIMS(logicalFile)) + linkedit.dd(new DDStatement().dsn(props.SDFSRESL).options("shr")) + if (buildUtils.isSQL(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDSNLOAD).options("shr")) diff --git a/languages/Cobol.groovy b/languages/Cobol.groovy index 1a31f7c7..345b1526 100644 --- a/languages/Cobol.groovy +++ b/languages/Cobol.groovy @@ -384,6 +384,9 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb if (buildUtils.isCICS(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDFHLOAD).options("shr")) + if (buildUtils.isIMS(logicalFile)) + linkedit.dd(new DDStatement().dsn(props.SDFSRESL).options("shr")) + if (buildUtils.isSQL(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDSNLOAD).options("shr")) diff --git a/languages/LinkEdit.groovy b/languages/LinkEdit.groovy index 8b7c4b06..eb81c913 100644 --- a/languages/LinkEdit.groovy +++ b/languages/LinkEdit.groovy @@ -109,13 +109,16 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb linkedit.dd(new DDStatement().dsn(syslibDataset).options("shr")) } linkedit.dd(new DDStatement().dsn(props.SCEELKED).options("shr")) - + if (props.debug && props.SEQAMOD) linkedit.dd(new DDStatement().dsn(props.SEQAMOD).options("shr")) - if (props.SDFHLOAD) + if (buildUtils.isCICS(logicalFile) linkedit.dd(new DDStatement().dsn(props.SDFHLOAD).options("shr")) - + + if (buildUtils.isIMS(logicalFile)) + linkedit.dd(new DDStatement().dsn(props.SDFSRESL).options("shr")) + if (props.SDSNLOAD) linkedit.dd(new DDStatement().dsn(props.SDSNLOAD).options("shr")) diff --git a/languages/PLI.groovy b/languages/PLI.groovy index 34a90cf9..9264abde 100644 --- a/languages/PLI.groovy +++ b/languages/PLI.groovy @@ -129,6 +129,7 @@ def createPLIParms(String buildFile, LogicalFile logicalFile) { def parms = props.getFileProperty('pli_compileParms', buildFile) ?: "" def cics = props.getFileProperty('pli_compileCICSParms', buildFile) ?: "" def sql = props.getFileProperty('pli_compileSQLParms', buildFile) ?: "" + def ims = props.getFileProperty('pli_compileIMSParms', buildFile) ?: "" def errPrefixOptions = props.getFileProperty('pli_compileErrorPrefixParms', buildFile) ?: "" def compileDebugParms = props.getFileProperty('pli_compileDebugParms', buildFile) @@ -142,6 +143,9 @@ def createPLIParms(String buildFile, LogicalFile logicalFile) { if (props.errPrefix) parms = "$parms,$errPrefixOptions" + if (buildUtils.isIMS(logicalFile)) + parms = "$parms,$ims" + // add debug options if (props.debug) { parms = "$parms,$compileDebugParms" @@ -355,6 +359,9 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb if (buildUtils.isCICS(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDFHLOAD).options("shr")) + if (buildUtils.isIMS(logicalFile)) + linkedit.dd(new DDStatement().dsn(props.SDFSRESL).options("shr")) + if (buildUtils.isSQL(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDSNLOAD).options("shr")) diff --git a/samples/application-conf/Assembler.properties b/samples/application-conf/Assembler.properties index 84fcf7f3..b2849d92 100644 --- a/samples/application-conf/Assembler.properties +++ b/samples/application-conf/Assembler.properties @@ -78,6 +78,10 @@ assembler_deployTypeCICS=CICSLOAD # deployType for build files with isDLI=true assembler_deployTypeDLI=IMSLOAD +# +# deployType for build files with isIMS=true +assembler_deployTypeIMS=IMSLOAD + # # scan link edit load module for link dependencies # can be overridden by file properties diff --git a/samples/application-conf/Cobol.properties b/samples/application-conf/Cobol.properties index c39e7eb9..4046a14e 100644 --- a/samples/application-conf/Cobol.properties +++ b/samples/application-conf/Cobol.properties @@ -85,6 +85,10 @@ cobol_deployTypeCICS=CICSLOAD # deployType for build files with isDLI=true cobol_deployTypeDLI=IMSLOAD +# +# deployType for build files with isIMS=true +cobol_deployTypeIMS=IMSLOAD + # # scan link edit load module for link dependencies # can be overridden by file properties diff --git a/samples/application-conf/LinkEdit.properties b/samples/application-conf/LinkEdit.properties index abeaa1b2..b8cea7df 100644 --- a/samples/application-conf/LinkEdit.properties +++ b/samples/application-conf/LinkEdit.properties @@ -43,6 +43,13 @@ linkedit_deployTypeCICS=CICSLOAD # e.q isDLI = true :: **/link/epsmlist.lnk linkedit_deployTypeDLI=IMSLOAD +# +# deployType for build files with isIMS=true set in file properties +# DBB scanners cannot determine the file tags for linkcards automatically, +# requires the flag to be set via a file property +# e.q isIMS = true :: **/link/epsmlist.lnk +linkedit_deployTypeIMS=IMSLOAD + # # scan link edit load module for link dependencies # can be overridden by file properties diff --git a/samples/application-conf/PLI.properties b/samples/application-conf/PLI.properties index b91582e0..3eaad50b 100644 --- a/samples/application-conf/PLI.properties +++ b/samples/application-conf/PLI.properties @@ -35,6 +35,7 @@ pli_compileCICSParms=SYSTEM(CICS),PP(MACRO,CICS) pli_compileSQLParms=PP(SQL) pli_compileErrorPrefixParms=XINFO(XML) pli_compileDebugParms=TEST +pli_compileIMSParms=SYSTEM(IMS) # # default LinkEdit parameters @@ -83,6 +84,10 @@ pli_deployTypeCICS=CICSLOAD # deployType for build files with isDLI=true pli_deployTypeDLI=IMSLOAD +# +# deployType for build files with isIMS=true +pli_deployTypeIMS=IMSLOAD + # # scan link edit load module for link dependencies # can be overridden by file properties diff --git a/samples/application-conf/README.md b/samples/application-conf/README.md index 9606c442..d1c706a4 100644 --- a/samples/application-conf/README.md +++ b/samples/application-conf/README.md @@ -40,12 +40,20 @@ Property | Description --- | --- dbb.scriptMapping | DBB configuration file properties association build files to language scripts dbb.scannerMapping | zAppBuild configuration override/expansion to map files extensions to DBB dependency scanner configurations -isSQL | File property overwrite to indicate that a file requires to include SQL parameters -isCICS | File property overwrite to indicate that a file requires to include CICS parameters -isMQ | File property overwrite to indicate that a file requires to include MQ parameters -isDLI | File property overwrite to indicate that a file requires to include DLI parameters cobol_testcase | File property to indicate a generated zUnit cobol test case to use a different set of source and output libraries +General file level overwrites to control the allocations of system datasets for compile and link steps or activation of preprocessing + +Property | Description +--- | --- +isSQL | File property overwrite to indicate that a file requires to include SQL preprocessing, and allocation of Db2 libraries for compile and link phase. +isCICS | File property overwrite to indicate that a file requires to include CICS preprocessing, and allocation of CICS libraries for compile and link phase. Also used to indicate if a *batch* module is executed under CICS for pulling appropriate language interface modules for Db2 or MQ. +isMQ | File property overwrite to indicate that a file requires to include MQ libraries for compile and link phase. +isDLI | File property overwrite to indicate that a file requires to include DLI +isIMS | File property flag to indicate IMS batch and online programs to allocate the IMS RESLIB library during link phase (Compared to the other 4 above flags, the isIMS flag is a pure file property, and not computed by the DBB scanners). + +Please note that the above file property settings `isCICS` and `isIMS` are also used to control the allocations when processing link cards with `LinkEdit.groovy` to include the appropriate runtime specific language interfaces. + ### reports.properties Properties used by the build framework to generate reports. Sample properties file to all application-conf to overwrite central build-conf configuration. @@ -168,6 +176,7 @@ pli_compileParms | Default base compile parameters. | true pli_compileCICSParms | Default CICS compile parameters. Appended to base parameters if has value.| true pli_compileSQLParms | Default SQL compile parameters. Appended to base parameters if has value. | true pli_compileDebugParms | Default Debug compile parameters. Appended to base parameters if running with debug flag set. | true +pli_compileIMSParms | Default IMS compile parameters. Appended to parms for file with `isIMS` flag turned on. | true pli_compileErrorPrefixParms | IDz user build parameters. Appended to base parameters if has value. | true pli_impactPropertyList | List of build properties causing programs to rebuild when changed | false pli_impactPropertyListCICS | List of CICS build properties causing programs to rebuild when changed | false diff --git a/samples/application-conf/file.properties b/samples/application-conf/file.properties index fd035353..8db4eb69 100644 --- a/samples/application-conf/file.properties +++ b/samples/application-conf/file.properties @@ -37,13 +37,51 @@ dbb.scriptMapping = CRB.groovy :: **/crb/*.yaml # dbb.scannerMapping = "scannerClass":"DependencyScanner", "languageHint":"PLI" :: pli, inc # dbb.scannerMapping = "scannerClass":"ZUnitConfigScanner" :: bzucfg -# +#### # General file level overwrites through DBB Build Properties -# isCICS = true :: **/cobol/member.cbl -# isSQL = true :: **/cobol/member.cbl -# isMQ = true :: **/cobol/member.cbl +# to control the allocations of system datasets for compile and link steps +# or activation of preprocessing + +# isCICS - boolean flag indicating that the process for the module requires the +# CICS libraries, source code needs preprocessing +# +# flag is set by the DBB scanner if it detects EXEC CICS statements. +# +# Override the flag for source code or linkcards that are executed in a CICS environment +# to enable proprocessing and to resolve appropriate language interface module +# +isCICS = true :: **/cobol/member.cbl +# isSQL - boolean flag indicating that the process for the module requires the +# Db2 libraries, source code needs preprocessing +# +# flag is set by the DBB scanner if it detects SQL statements. +# +isSQL = true :: **/cobol/member.cbl + +# isMQ - boolean flag indicating that the process for the module requires the +# MQ libraries got compile and link steps and to generate the +# MQ stub instructions for the link phase based on the flags +# isCICS, isDLI and isIMS (except the LinkEdit.groovy) +# +# flag is set by the DBB scanner if it detects MQ calls. # +# Override the flag to force adding +# +isMQ = true :: **/cobol/member.cbl + +# isIMS - indicating that the process for the module requires the +# IMS libraries in the link phase. Applicable for +# DL/I batch programs +# IMS online programs +# +# flag is NOT set by the DBB scanner. +# +# Set the flag for source code or link cards +# to resolve appropriate IMS language interface module (DFSLI000) +# +isIMS = true :: **/cobol/DLIBATCH.cbl + # Please check for available file property overwrites within samples/application-conf/README.md # diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy index 6c0727f5..c0ed3090 100644 --- a/utilities/BuildUtilities.groovy +++ b/utilities/BuildUtilities.groovy @@ -424,6 +424,18 @@ def isMQ(LogicalFile logicalFile) { return isMQ } +/* + * isIMS - tests to see if the program is a DL/I program. If the logical file is false, then + * check to see if there is a file property. + */ +def isIMS(LogicalFile logicalFile) { + isIMS = false + String imsFlag = props.getFileProperty('isIMS', logicalFile.getFile()) + if (imsFlag) + isIMS = imsFlag.toBoolean() + return isIMS +} + /* * getMqStubInstruction - * returns include defintion for mq sub program for link edit @@ -435,7 +447,7 @@ def getMqStubInstruction(LogicalFile logicalFile) { // https://www.ibm.com/docs/en/ibm-mq/9.3?topic=files-mq-zos-stub-programs if (isCICS(logicalFile)) { mqStubInstruction = " INCLUDE SYSLIB(CSQCSTUB)\n" - } else if (isDLI(logicalFile)) { + } else if (isDLI(logicalFile) || isIMS(logicalFile)) { mqStubInstruction = " INCLUDE SYSLIB(CSQQSTUB)\n" } else { mqStubInstruction = " INCLUDE SYSLIB(CSQBSTUB)\n" @@ -556,7 +568,8 @@ def retrieveLastBuildResult(){ } /* - * returns the deployType for a logicalFile depending on the isCICS, isDLI setting + * returns the deployType for a logicalFile depending on the + * isCICS, isIMS and isDLI setting */ def getDeployType(String langQualifier, String buildFile, LogicalFile logicalFile){ // getDefault @@ -569,6 +582,9 @@ def getDeployType(String langQualifier, String buildFile, LogicalFile logicalFil if(isCICS(logicalFile)){ // if CICS String cicsDeployType = props.getFileProperty("${langQualifier}_deployTypeCICS", buildFile) if (cicsDeployType != null) deployType = cicsDeployType + } else if (isIMS(logicalFile)){ + String imsDeployType = props.getFileProperty("${langQualifier}_deployTypeIMS", buildFile) + if (imsDeployType != null) deployType = imsDeployType } else if (isDLI(logicalFile)){ String dliDeployType = props.getFileProperty("${langQualifier}_deployTypeDLI", buildFile) if (dliDeployType != null) deployType = dliDeployType From c8a9d08c4e569df9e3c6dfbae60890f62ca5ef59 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 14 Dec 2023 12:03:04 +0100 Subject: [PATCH 5/7] Corrections for zAppBuild 3.6.0 release (#458) * Fix broken char * Fix syntax error in LinkEdit.groovy Signed-off-by: Dennis Behm --- languages/LinkEdit.groovy | 6 +++--- utilities/BuildUtilities.groovy | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/languages/LinkEdit.groovy b/languages/LinkEdit.groovy index eb81c913..8c69ca2f 100644 --- a/languages/LinkEdit.groovy +++ b/languages/LinkEdit.groovy @@ -112,10 +112,10 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb if (props.debug && props.SEQAMOD) linkedit.dd(new DDStatement().dsn(props.SEQAMOD).options("shr")) - - if (buildUtils.isCICS(logicalFile) + + if (buildUtils.isCICS(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDFHLOAD).options("shr")) - + if (buildUtils.isIMS(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDFSRESL).options("shr")) diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy index c0ed3090..66f0b883 100644 --- a/utilities/BuildUtilities.groovy +++ b/utilities/BuildUtilities.groovy @@ -447,7 +447,7 @@ def getMqStubInstruction(LogicalFile logicalFile) { // https://www.ibm.com/docs/en/ibm-mq/9.3?topic=files-mq-zos-stub-programs if (isCICS(logicalFile)) { mqStubInstruction = " INCLUDE SYSLIB(CSQCSTUB)\n" - } else if (isDLI(logicalFile) || isIMS(logicalFile)) { + } else if (isDLI(logicalFile) || isIMS(logicalFile)) { mqStubInstruction = " INCLUDE SYSLIB(CSQQSTUB)\n" } else { mqStubInstruction = " INCLUDE SYSLIB(CSQBSTUB)\n" From b5d0038a1959deeb070a71822d65ae87082b7525 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 14 Dec 2023 15:57:35 +0100 Subject: [PATCH 6/7] Add validation of reporting prereqs (#459) * Add validation of reporting prereqs Signed-off-by: Dennis Behm --- build.groovy | 6 ++++-- utilities/ReportingUtilities.groovy | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/build.groovy b/build.groovy index d36952f3..d2bb12ad 100644 --- a/build.groovy +++ b/build.groovy @@ -662,7 +662,8 @@ def createBuildList() { } // Perform analysis and build report of external impacts - if (props.reportExternalImpacts && props.reportExternalImpacts.toBoolean()){ + // Prereq: Metadatastore Connection + if (metadataStore && props.reportExternalImpacts && props.reportExternalImpacts.toBoolean()){ if (buildSet && changedFiles) { println "** Perform analysis and reporting of external impacted files for the build list including changed files." reportingUtils.reportExternalImpacts(buildSet.plus(changedFiles)) @@ -674,7 +675,8 @@ def createBuildList() { } // Document and validate concurrent changes - if (props.reportConcurrentChanges && props.reportConcurrentChanges.toBoolean()){ + // Prereq: Workspace containing git repos. Skipped for --userBuild build type + if (!props.userBuild && props.reportConcurrentChanges && props.reportConcurrentChanges.toBoolean()){ println "** Calculate and document concurrent changes." reportingUtils.calculateConcurrentChanges(buildSet) } diff --git a/utilities/ReportingUtilities.groovy b/utilities/ReportingUtilities.groovy index 9aedde5f..08b972a7 100644 --- a/utilities/ReportingUtilities.groovy +++ b/utilities/ReportingUtilities.groovy @@ -194,13 +194,15 @@ def calculateConcurrentChanges(Set buildSet) { // initialize patterns List gitRefMatcherPatterns = createMatcherPatterns(props.reportConcurrentChangesGitBranchReferencePatterns) - + // obtain all current remote branches // TODO: Handle / Exclude branches from other repositories Set remoteBranches = new HashSet() props.applicationSrcDirs.split(",").each { dir -> dir = buildUtils.getAbsolutePath(dir) - remoteBranches.addAll(gitUtils.getRemoteGitBranches(dir)) + if (gitUtils.isGitDir(dir)) { + remoteBranches.addAll(gitUtils.getRemoteGitBranches(dir)) + } } // Run analysis for each remoteBranch, which matches the configured criteria From c05f9e3c3acde6464e1a539b8b9a33344f23d115 Mon Sep 17 00:00:00 2001 From: Mathieu Dalbin Date: Fri, 15 Dec 2023 13:27:55 +0100 Subject: [PATCH 7/7] zCEE2 language script (#434) * zCEE2 language script Signed-off-by: Mathieu Dalbin --- build-conf/README.md | 10 + build-conf/build.properties | 2 +- build-conf/zCEE2.properties | 28 +++ languages/zCEE2.groovy | 219 ++++++++++++++++++ .../application-conf/application.properties | 74 +++--- samples/application-conf/file.properties | 23 +- 6 files changed, 317 insertions(+), 39 deletions(-) create mode 100644 build-conf/zCEE2.properties create mode 100644 languages/zCEE2.groovy diff --git a/build-conf/README.md b/build-conf/README.md index 762c2425..67396051 100644 --- a/build-conf/README.md +++ b/build-conf/README.md @@ -301,6 +301,16 @@ zcee3_shellEnvironment | Shell environment used to run the gradle command zcee3_gradlePath | Path to gradle executable zcee3_gradle_JAVA_OPTS | JAVA Options used with gradle +### zCEE2.properties +Application properties used by zAppBuild/language/zCEE2.groovy + +Property | Description +--- | --- +zcee2_zconbtPath | Absolute path to zconbt executable on z/OS UNIX System Services +zcee2_JAVA_HOME | Java installation used by the zconbt utility +zcee2_inputType | Mapping of input files with types of files +zcee2_ARA_PackageArtifacts | Flag to indicate if artifacts produced for the ARA processing should be packaged + ### CRB.properties Application properties used by zAppBuild/language/CRB.groovy diff --git a/build-conf/build.properties b/build-conf/build.properties index 60d0d5c3..acff3409 100644 --- a/build-conf/build.properties +++ b/build-conf/build.properties @@ -19,7 +19,7 @@ buildPropFiles=datasets.properties,dependencyReport.properties,Assembler.properties,BMS.properties,\ MFS.properties,PSBgen.properties,DBDgen.properties,ACBgen.properties,Cobol.properties,\ LinkEdit.properties,PLI.properties,REXX.properties,ZunitConfig.properties,Transfer.properties,\ -CRB.properties,zCEE3.properties +CRB.properties,zCEE3.properties,zCEE2.properties # # Comma separated list of default application configuration property files to load diff --git a/build-conf/zCEE2.properties b/build-conf/zCEE2.properties new file mode 100644 index 00000000..f93d9c9f --- /dev/null +++ b/build-conf/zCEE2.properties @@ -0,0 +1,28 @@ +# Releng properties used by language/zCEE2.groovy + +# +# Comma separated list of required build properties for zCEE3.groovy +zcee2_requiredBuildProperties=zcee2_zconbtPath,zcee2_JAVA_HOME + +# +# Absolute path to zconbt executable on z/OS UNIX System Services +# for instance: /var/zosconnect/v359/bin/zconbt.zos +zcee2_zconbtPath= + +# +# Java installation used by the zconbt utility +# for instance: /usr/lpp/java/J8.0_64 +zcee2_JAVA_HOME= + +# +# Mapping of input files with types of files +# PROJECT can be used for SAR and AAR projects +# SAR and ARA can be used for SAR Properties files and ARA properties files +# Can be overridden by file-level properties +zcee2_inputType=PROJECT + +# +# Flag to indicate if artifacts produced for the ARA processing should be packaged (generated copybooks, API information copybook and logs) +# When set to true, the artifacts are located based on the dataStructuresLocation, apiInfoFileLocation and logFileDirectory properties of the ARA properties files +# When not defined, the default value is false and artifacts are packaged +zcee2_ARA_PackageArtifacts=true diff --git a/languages/zCEE2.groovy b/languages/zCEE2.groovy new file mode 100644 index 00000000..c77a21e7 --- /dev/null +++ b/languages/zCEE2.groovy @@ -0,0 +1,219 @@ +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript +import com.ibm.dbb.dependency.* +import com.ibm.dbb.build.* +import groovy.transform.* +import com.ibm.dbb.build.report.* +import com.ibm.dbb.build.report.records.* +import java.nio.file.*; + + +// define script properties +@Field BuildProperties props = BuildProperties.getInstance() +@Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) + +// verify required build properties +buildUtils.assertBuildProperties(props.zcee2_requiredBuildProperties) + +// create updated build map, removing duplicates in case of PROJECT input Type +HashMap updatedBuildMap = new HashMap() + +println("** Streamlining the build list to remove duplicates") +argMap.buildList.each { buildFile -> + PropertyMappings inputTypeMappings = new PropertyMappings("zcee2_inputType") + inputType = inputTypeMappings.getValue(buildFile) + + if (inputType) { + if (inputType == "PROJECT") { + File changedBuildFile = new File(buildFile); + File projectDir = changedBuildFile.getParentFile() + boolean projectDirFound = false + while (projectDir != null && !projectDirFound) { + File projectFile = new File(projectDir.getPath() + '/.project') + if (projectFile.exists()) { + projectDirFound = true + } else { + projectDir = projectDir.getParentFile() + } + } + if (projectDirFound) { + updatedBuildMap.put(projectDir.getPath(), "PROJECT") + } else { + if (props.verbose) println("!* No project directory found for file '${buildFile}'. Skipping...") + } + } else { + updatedBuildMap.put(buildFile, inputType); + } + } else { + println("!* No Input Type mapping for file ${buildFile}, skipping it...") + } +} + +println("** Building ${updatedBuildMap.size()} API ${updatedBuildMap.size() == 1 ? 'definition' : 'definitions'} mapped to ${this.class.getName()}.groovy script") +// sort the build list based on build file rank if provided +HashMap sortedMap = buildUtils.sortBuildMap(updatedBuildMap, 'zcee2_fileBuildRank') + +int currentBuildFileNumber = 1 + +// iterate through build list +sortedMap.each { buildFile, inputType -> + println "*** (${currentBuildFileNumber++}/${sortedMap.size()}) Building ${inputType == "PROJECT" ? 'project' : 'properties file'} $buildFile" + + String parameters = "" + String outputDir = "" + String outputFile = "" + if (inputType == "PROJECT") { + outputDir = "${props.buildOutDir}/zCEE2/$buildFile" + parameters = "-od $outputDir -pd $buildFile" + } else { + File changedBuildFile = new File(buildFile); + String outputFileName = changedBuildFile.getName().split("\\.")[0] + "." + inputType.toLowerCase() + File projectDir = changedBuildFile.getParentFile() + outputFile = "${props.buildOutDir}/zCEE2/${projectDir.getPath()}/${outputFileName}" + outputDir = "${props.buildOutDir}/zCEE2/${projectDir.getPath()}" + parameters = "-f $outputFile -p $buildFile" + } + File outputDirectory = new File(outputDir) + outputDirectory.mkdirs() + + + Properties ARAproperties = new Properties() + File dataStructuresLocation + File apiInfoFileLocation + File logFileDirectory + if (inputType == "ARA") { + File ARApropertiesFile = new File(buildFile) + ARApropertiesFile.withInputStream { + ARAproperties.load(it) + } + println("*** dataStructuresLocation: ${ARAproperties.dataStructuresLocation}") + println("*** apiInfoFileLocation: ${ARAproperties.apiInfoFileLocation}") + println("*** logFileDirectory: ${ARAproperties.logFileDirectory}") + dataStructuresLocation = new File(ARAproperties.dataStructuresLocation) + dataStructuresLocation.mkdirs() + apiInfoFileLocation = new File(ARAproperties.apiInfoFileLocation) + apiInfoFileLocation.mkdirs() + logFileDirectory = new File(ARAproperties.logFileDirectory) + logFileDirectory.mkdirs() + } + + + // log file - Changing slashes with dots to avoid conflicts + String logFilePath = buildFile.replace("/", ".") + File logFile = new File("${props.buildOutDir}/${logFilePath}.zCEE2.log") + if (logFile.exists()) + logFile.delete() + + String zconbtPath = props.getFileProperty('zcee2_zconbtPath', buildFile) + + File zconbt = new File(zconbtPath) + if (!zconbt.exists()) { + def errorMsg = "*! zconbt wasn't find at location '$zconbtPath'" + println(errorMsg) + props.error = "true" + buildUtils.updateBuildResult(errorMsg:errorMsg) + } else { + String[] command; + + command = [zconbtPath, parameters] + String commandString = command.join(" ") + if (props.verbose) + println("** Executing command '${commandString}'...") + + StringBuffer shellOutput = new StringBuffer() + StringBuffer shellError = new StringBuffer() + + String JAVA_HOME = props.getFileProperty('zcee2_JAVA_HOME', buildFile) + + ProcessBuilder cmd = new ProcessBuilder(zconbtPath, parameters); + Map env = cmd.environment(); + env.put("JAVA_HOME", JAVA_HOME); + env.put("PATH", JAVA_HOME + "/bin" + ";" + env.get("PATH")) + Process process = cmd.start() + process.consumeProcessOutput(shellOutput, shellError) + process.waitFor() + if (props.verbose) + println("** Exit value for the zconbt process: ${process.exitValue()}"); + + // write outputs to log file + String enc = props.logEncoding ?: 'IBM-1047' + logFile.withWriter(enc) { writer -> + writer.append(shellOutput) + writer.append(shellError) + } + + if (process.exitValue() != 0) { + def errorMsg = "*! Error during the zconbt process" + println(errorMsg) + if (props.verbose) + println("*! zconbt error message:\n${shellError}") + props.error = "true" + buildUtils.updateBuildResult(errorMsg:errorMsg) + } else { + if (props.verbose) + println("** zconbt output:\n${shellOutput}") + + ArrayList outputProperty = [] + Path outputDirectoryPath = Paths.get(props.buildOutDir) + if (inputType == "PROJECT") { + String[] outputFiles = outputDirectory.list() + for (int i=0; i