build.gradle

buildscript {
  repositories {
    maven {
      url 'https://artifactory.wetransform.to/artifactory/libs-release-local'
    }
    maven {
      url 'https://artifactory.wetransform.to/artifactory/libs-snapshot-local'
    }
  }
  dependencies {
    classpath 'to.wetransform.hale:gradle-hale-plugin:2.0.0'
  }
}

import java.text.*

plugins {
  id 'distribution'
}

apply plugin: 'to.wetransform.hale'

hale {
  cliArtifact = 'to.wetransform:hale-cli'
  cliVersion = '5.3.0'
}

repositories {
  // mavenLocal() // XXX for testing

  maven {
    url 'https://repo.osgeo.org/repository/release/'
  }
}

configurations.all {
  resolutionStrategy.cacheChangingModulesFor 180, 'minutes'
  //resolutionStrategy.cacheChangingModulesFor 0, 'seconds'
}

Calendar calendar = Calendar.getInstance();

// determine build ID
String buildID = project.findProperty('buildId') ?: System.getenv()['BUILD_ID'] // set externally
if (buildID == null){
  DateFormat dateFormat = new SimpleDateFormat('YYYY-MM-dd HH-mm-ss');
  buildID = dateFormat.format(calendar.getTime())
}
project.ext.buildId = buildID


/*
 * Migration tasks
 */

project.ext.advAlignmentsPath = project.hasProperty('advAlignments') ? project.getProperty('advAlignments') : 'adv-alignments'

task('dbMatching', type: hale.cli()) {
  args = [
    'project',
    'generate',
    'postnas-matching',
    '--reference-schema',
    'https://repository.gdi-de.org/schemas/adv/nas/7.1/aaa.xsd',
    '--target-schema',
    'database/schema.hsd',
    '--target',
    'database/matching.halex'
  ]

  description "Generate matching project between AAA XSD and PostNAS schemas"
  group 'Alignment migration'
}


def migrations = Config.load(file(project.hasProperty('migrationsFile') ? project.getProperty('migrationsFile') : 'migrations.yaml'))
def projectFiles = [:]

/*
task('migrate-all') {
  description "Runs all migration tasks (for all configured projects)."
  group 'Alignment migration'
}
*/

migrations.each { name, config ->
  if (config.enabled != null && !config.enabled) {
    return
  }

  def projectsDir = file('projects')
  if (!projectsDir.exists()) projectsDir.mkdirs()
  def autoProjectFile = new File(projectsDir, "${name}-auto.halex")
  def manualProjectFile = new File(projectsDir, "${name}-manual.halex")
  def dbProjectFile = new File(projectsDir, "${name}-db.halex")

  projectFiles[name] = dbProjectFile

  def diffFile = new File(projectsDir, "${name}-manual.diff")

  // create diff to previously created manual project
  task("diff-$name").doLast {
    if (manualProjectFile.exists()) {
      exec {
        commandLine'git', 'diff', '-p', '--no-index',
          (projectDir.toPath().relativize(autoProjectFile.toPath()).toString() + '.alignment.xml').replace('\\', '/'),
          (projectDir.toPath().relativize(manualProjectFile.toPath()).toString() + '.alignment.xml').replace('\\', '/')

        // git seems to always return with an exit code here
        ignoreExitValue = true

        standardOutput = diffFile.newOutputStream()
        // errorOutput = new TeeOutputStream(errFile.newOutputStream(), errorOutput)
      }

      if (!diffFile.length()) {
        diffFile.delete()
      }
    }

    description "Create a diff between the manually curated and the automatically generated migration project (${name})"
    group 'Alignment migration'
  }

  // add migration task for each project
  task("auto-$name", type: hale.cli(), dependsOn: "diff-$name") {
    args = [
      'project',
      'migrate',
      'matching',
      '--source-project',
      advAlignmentsPath + '/' + config.advProject,
      '--matching-project',
      'database/matching.halex', // PostNAS
      '--target',
      autoProjectFile as String
    ]

    description "Automatical migration of AdV project ${config.advProject} to PostNAS schema"
    group 'Alignment migration'
  }

  task("migrate-$name", dependsOn: "auto-$name") {
    description "Migrate AdV project ${config.advProject} to PostNAS schema, attempts to update manually curated project"
    group 'Alignment migration'
  }.doLast {
    // copy project
    copy {
      from projectsDir
      include projectsDir.toPath().relativize(autoProjectFile.toPath()).toString()
      include projectsDir.toPath().relativize(autoProjectFile.toPath()).toString() + '.alignment.xml'
      into projectsDir

      rename(/(.*)-auto(.*)/, '$1-manual$2')
    }
    // adapt project file to reference correct alignment
    def manualText = manualProjectFile.getText('UTF-8')
    manualText = manualText.replaceFirst(
      /-auto\.halex\.alignment\.xml/,
      '-manual.halex.alignment.xml')
    manualText = manualText.replaceFirst(
      /-auto\.halex/,
      '-manual.halex')
    manualProjectFile.write(manualText, 'UTF-8')

    if (diffFile.exists() && diffFile.length()) {
      // apply diff
      
      // first create a copy of the generated alignment (as git deletes the file)
      def tempFile = new File(getTemporaryDir(), autoProjectFile.name)
      copy {
        from autoProjectFile.toString() + '.alignment.xml'
        into tempFile.parentFile
        rename {
          tempFile.name
        }
      }
      // apply diff with git
      def error
      try {
        // try at least a partial apply
        // see also http://stackoverflow.com/questions/15796528/partial-git-apply
        exec {
          commandLine 'git', 'apply', '--reject', '--verbose', '--ignore-whitespace', projectDir.toPath().relativize(diffFile.toPath()).toString()
        }
      } catch (e) {
        error = e
      }
      // recreate generated alignment file
      copy {
        from tempFile
        into projectsDir
        rename {
          autoProjectFile.name + '.alignment.xml'
        }
      }

      // handle git apply problem
      if (error) {
        println "ERROR applying diff to update manually curated project."
        println "Please check if you can apply the changes manually."
        println "Please check any .rej files for rejected hunks."
        println "Be aware that the manually created project has been replaced by the generated version."
      }
    }
  }

  // tasks['migrate-all'].dependsOn("migrate-$name")

  // change source to database
  task("db-migrate-$name", type: hale.cli()) {
    args = [
      'project', 'migrate', 'source',
      '--schema',
      project.hasProperty('db.uri') ? project.getProperty('db.uri') : 'jdbc:postgresql://localhost:5432/testbed',
      '--schema-reader', 'eu.esdihumboldt.hale.io.jdbc.schema.reader',
      '--schema-setting',
      'jdbc.user=' + (project.hasProperty('db.user') ? project.getProperty('db.user') : 'postgres'),
      '--schema-setting',
      'jdbc.password=' + (project.hasProperty('db.password') ? project.getProperty('db.password') : 'postgres'),
      '--source-project',
      manualProjectFile as String,
      '--target',
      dbProjectFile as String
    ]

    def schemas = project.hasProperty('db.schemas') ? project.getProperty('db.schemas') : null
    if (schemas) {
      args << '--schema-setting'
      args << "schemas=$schemas"
    }

    description "Adapt the manually curated ${name}-project to a custom database connection"
    group 'Alignment migration'
  }

}

/*
 * Transformation
 */

enum State {
  start, transformed, validated, processed, uploaded
}
class StateManager {
  private final File stateFile

  StateManager(File stateFile) {
    this.stateFile = stateFile
  }

  void setCurrent(State state) {
    stateFile.text = (state ?: State.start).name()
  }

  State getCurrent() {
    if (stateFile.exists()) {
      State.valueOf(stateFile.text.trim())
    }
    else {
      State.start // assume start
    }
  }

  void reset() {
    setCurrent(State.start)
  }
}

def transformationFiles = [file('transformations.yaml')]
if (project.hasProperty('profile')) {
  transformationFiles.addAll(fileTree(dir: projectDir).matching {
      include("profiles/${project.getProperty('profile')}/transformations.yaml")
  } as List)
}
transformationFiles.addAll(fileTree(dir: projectDir).matching {
  // override configuration locally
  include('transformations-override.yaml')
} as List)

def transformations = Config.load(transformationFiles as File[])

def enabledTransformations = [:]

task('stateAll') {
  group 'Transformation status'
}

transformations.tasks.each { name, config ->
  // apply defaults
  if (transformations.defaults) {
    config = Config.mergeConfigs([transformations.defaults, config])
  }

  // check if enabled
  if (config.enabled != null && !config.enabled) {
    return
  }

  // requires a project to be set
  if (!config.project) {
    return
  }
  enabledTransformations[name] = true

  def projectId = config.project
  def projectFile = projectFiles[projectId]

  def baseFolder = new File(file('results'), name)
  baseFolder.mkdirs()
  def state = new StateManager(new File(baseFolder, 'state'))

  def targetFolder = new File(baseFolder, '01_transform')
  def targetFile = new File(targetFolder, 'result.gml')
  File statsFile = new File(targetFolder, 'statistics.json')

  task("state-${name}") {
    group 'Transformation status'
  }.doLast {
    println "Status for transformation ${name}:".padRight(40) + ' ' + state.current
  }
  tasks['stateAll'].dependsOn("state-${name}")

  // Transformation stage

  task("transform-${name}", type: hale.transform(), dependsOn: "db-migrate-$projectId") {
    // transformation project
    transformation = projectFile

    // database source
    source(project.findProperty('db.uri') ?: 'jdbc:postgresql://localhost:5432/testbed') {
      provider 'eu.esdihumboldt.hale.io.jdbc.instance.reader'

      setting 'jdbc.user', (project.findProperty('db.user') ?: 'postgres')
      setting 'jdbc.password', (project.findProperty('db.password') ?: 'postgres')
    }

    if (config.model) {
      // restrict to a specific model -> add generic filter
      filterArgs << '-filter'
      filterArgs << "CQL:advstandardmodell = '${config.model}'"

      if (config.additionalModels) {
        // add filters for additional models (also accepting those)
        config.additionalModels.each { modelName ->
          filterArgs << '-filter'
          filterArgs << "CQL:advstandardmodell = '${modelName}'"
        }
      }

      // add project variable
      projectVariables['ADV_MODELLART'] = config.model
    }

    // project variables from config
    if (config.variables) {
      projectVariables << config.variables
    }

    // transformation target
    target(targetFile) {
      // WFS 2 FC by default
      provider(config.writer?.provider ?: 'eu.esdihumboldt.hale.io.wfs.fc.write-2.0')
      
      // settings
      if (config.writer?.settings) {
        settings << config.writer.settings
      }
    }
    
    // XML schema validation
    if (config.validation.xmlSchema) {
      validate('eu.esdihumboldt.hale.io.xml.validator')
    }
    
    // folder for output and reports
    logFolder = targetFolder

    // general options
    printStacktrace = true
    trustGroovy = true

    // activate hale internal validation of transformed instances
    if (config.validation.haleInternal) {
      environment['HALE_TRANSFORMATION_INTERNAL_VALIDATION'] = 'true'
    }
    
    // statistics and ...
    additionalArgs << '-statisticsOut'
    additionalArgs << statsFile.absolutePath
    // ... success evaluation
    // additionalArgs << '-successEvaluation'
    // additionalArgs << project.file('success.groovy')

    // exit value is ignored -> we do a success evaluation in the validation stage
    ignoreExitValue = true

    description "Runs a transformation based on project ${projectId} for model ${config.model}."
    group 'Transformation'
  }.doFirst {
    targetFolder.deleteDir()
    targetFolder.mkdirs()

    // reset state -> we are starting anew
    state.reset()
  }.doLast {
    // stage completed
    state.current = State.transformed
    println 'Transformation stage completed'
  }

  // Validation stage
  
  def validateFolder = new File(baseFolder, '02_validate')
  
  task("validate-${name}") {
    description "Performs/evaluates validation on transformation results for project ${projectId} for model ${config.model}."
    group 'Transformation'
  }.doFirst {
    // verify state
    assert state.current != State.start : 'Validation requires transformation to have been run'
  }.doLast {
    // success evaluation

    validateFolder.mkdirs()
    def evaluationError = new File(validateFolder, 'evaluationError.txt')
    if (evaluationError.exists()) {
      evaluationError.delete()
    }

    // errors that require transformation to be re-run
    def stats
    def aggregated
    try {
      assert statsFile.exists() : 'Statistics file does not exist'

      // load stats
      stats = new groovy.json.JsonSlurper().parse(statsFile)
      aggregated = stats.aggregated

      assert aggregated['eu.esdihumboldt.hale.transform'].report.completed == true : 'Die Transformation wurde nicht abgeschlossen'

      assert aggregated['eu.esdihumboldt.hale.transform'].report.errors == 0 : 'Während der Transformation sind Fehler aufgetreten'

      assert aggregated['eu.esdihumboldt.hale.transform'].createdPerType.any { typeName, count -> count > 0 } : 'Aus der Transformation sind keine Objekte erzeugt worden sein'

      assert aggregated['eu.esdihumboldt.hale.io.instance.write.transformed'].report.completed == true : 'Das Schreiben der Ziel-Datei wurde nicht abgeschlossen'

      assert aggregated['eu.esdihumboldt.hale.io.instance.write.transformed'].report.errors == 0 : 'Während des Schreibens der Ziel-Datei sind Fehler aufgetreten'
    } catch (Throwable e) {
      // reset state
      state.reset()
      println 'Transformation needs to be re-run, resetting state'

      evaluationError.text = e.message
      throw e
    }

    // errors that are general validation errors
    try {
      // XML Schema
      if (config.validation.xmlSchema) {
        assert aggregated['eu.esdihumboldt.hale.io.xml.validator']?.report?.completed == true : 'Die XML Schema Validierung wurde nicht ausgeführt'

        assert aggregated['eu.esdihumboldt.hale.io.xml.validator'].report.errors == 0 : 'Die XML Schema Validierung ergab Fehler'

        assert aggregated['eu.esdihumboldt.hale.io.xml.validator'].report.warnings == 0 : 'Die XML Schema Validierung ergab Warnungen'
      }

      if (config.validation.haleInternal) {
        assert aggregated['eu.esdihumboldt.hale.instance.validation.internal']?.report?.completed == true : 'Die hale-interne Validierung wurde nicht ausgeführt'
        assert aggregated['eu.esdihumboldt.hale.instance.validation.internal'].report.errors == 0 : 'Die hale-interne Validierung ergab Fehler'
        assert aggregated['eu.esdihumboldt.hale.instance.validation.internal'].report.warnings == 0 : 'Die hale-interne Validierung ergab Warnungen'
      }
    } catch (Throwable e) {
      // reset state
      state.reset()
      println 'Validation failed and transformation needs to be re-run, resetting state'

      evaluationError.text = e.message
      throw e
    }

    //TODO other, external validations?

    // success -> mark stage as complete
    if (state.current == State.transformed) {
      // only override state for transformed (to not invalidate any processing)
      state.current = State.validated
    }
    println 'Validation stage completed'
  }

  // Processing stage
  
  def processFolder = new File(baseFolder, '03_process')
  def processResultFolder = new File(processFolder, 'result')

  // currently only splitting supported as processing
  boolean processingEnabled = config.split?.enabled
  
  if (processingEnabled) {
    def splitFolder = new File(processFolder, 'split')
    
    task("process-${name}") {
      description "Processes transformation result (Project ${projectId} for model ${config.model})."
      group 'Transformation'
    }.doFirst {
      // check preconditions
      assert state.current in [State.validated, State.processed, State.uploaded] : 'Processing requires a valid transformation result'

      processFolder.mkdirs()
      processResultFolder.deleteDir()
      processResultFolder.mkdirs()
    }.doLast {
      // actual processing
      
      // splitting
      if (config.split?.enabled) {
        splitFolder.deleteDir()
        splitFolder.mkdirs()
        def splitError = new File(splitFolder, 'splitError.txt')
        if (splitError.exists()) {
          splitError.delete()
        }

        try {
          // do actual splitting
          
          hale.cli {
            logFolder = splitFolder

            args = ['data', 'rewrite']

            // data source
            args << '--data'
            args << targetFile.absolutePath

            // data settings
            // -> don't change geometries
            args << '--data-setting'
            args << 'suppressParsingGeometry=true'

            // schema settings
            // -> all features types should be loaded
            args << '--schema-setting'
            args << 'relevantMode=featureTypes'

            // target
            args << '--target'
            args << new File(processResultFolder, 'part.gml')

            args << '--target-writer'
            args << (config.writer?.provider ?: 'eu.esdihumboldt.hale.io.wfs.fc.write-2.0')

            // split settings
            args << '--target-setting'
            args << "partition.mode=${config.split.mode}"
            args << '--target-setting'
            args << "instancesPerFile=${config.split.instances}"

            // writer settings
            if (config.writer?.settings) {
              config.writer?.settings.each { key, value ->
                args << '--target-setting'
                args << "${key}=$value"
              }
            }
          }

        } catch (Throwable e) {
          splitError.text = e.message
          throw e
        }
      }
      else {
        state.current = State.processed
        println 'Processing stage skipped (nothing to do)'
        return
      }

      // success -> mark stage as complete
      state.current = State.processed
      println 'Processing stage completed'
    }
  }

  // Upload stage
  
  boolean uploadEnabled = config.upload?.enabled
  
  if (uploadEnabled) {
    def uploadFolder = new File(baseFolder, '04_upload')

    task("upload-${name}") {
      description "Upload (processed) transformation result (Project ${projectId} for model ${config.model})."
      group 'Transformation'
    }.doFirst {
      // check preconditions
      assert state.current in [State.validated, State.processed, State.uploaded] : 'Uploading requires a valid (and optionally processed) transformation result'

      uploadFolder.deleteDir()
      uploadFolder.mkdirs()
    }.doLast {
      def uploadError = new File(uploadFolder, 'uploadError.txt')
      if (uploadError.exists()) {
        uploadError.delete()
      }

      // identify upload source(s)
      def sources = []
      try {
        if (state.current == State.validated || !processResultFolder.exists()) {
          // use transformation result
          sources << targetFile
        }
        else {
          // use processing result
          sources = fileTree(dir: processResultFolder).matching {
            include('*.gml')
          } as List
        }

        assert !sources.empty : 'No files could be identified for upload'

        assert config.upload.url : 'URL for upload is not configured'
      } catch (Throwable e) {
        uploadError.text = e.message
          throw e
      }

      for (int i = 0; i < sources.size(); i++) {
        def source = sources[i]
        def fileId = 'file_' + "$i".padLeft(5, '0')
        def reportFolder = new File(uploadFolder, fileId)
        reportFolder.mkdirs()

        // upload file
        try {
          hale.cli {
            logFolder = reportFolder

            args = ['data', 'rewrite']

            // data source
            args << '--data'
            args << (source as File).absolutePath

            // data settings
            // -> don't change geometries
            args << '--data-setting'
            args << 'suppressParsingGeometry=true'

            // schema settings
            // -> all features types should be loaded
            args << '--schema-setting'
            args << 'relevantMode=featureTypes'

            // target
            args << '--target'
            args << config.upload.url

            //TODO settings configurable?
            args << '--target-writer'
            args << 'eu.esdihumboldt.hale.io.wfs.write'
            args << '--target-setting'
            args << 'wfsVersion=2.0.0'
          }

        } catch (Throwable e) {
          uploadError.text = "Failed to upload file ${source} ($fileId): " + e.message
          throw e
        }
      }

      // success -> mark stage as complete
      state.current = State.uploaded
      println 'Upload stage completed'
    }
  }

  // Continue -> run pipeline based on current state
  
  boolean alreadyDone = state.current == State.uploaded
  
  task("continue-${name}") {
    description "Runs transformation pipeline from point of last failure or starts it anew (Project ${projectId} for model ${config.model})."
    group 'Transformation'

    /*
     * Important: This is based on the stage state at the time Gradle
     * is run!
     */
    switch(state.current) {
      case State.start:
        // run transformation
        dependsOn "transform-$name"
      case State.transformed:
        // run validation
        dependsOn "validate-$name"
        tasks.findByName("validate-$name").mustRunAfter "transform-$name"
      case State.validated:
        if (processingEnabled) {
          // run processing
          dependsOn "process-$name"
          tasks.findByName("process-$name").mustRunAfter "validate-$name"
        }
      case State.processed:
        if (uploadEnabled) {
          // run upload
          dependsOn "upload-$name"
          if (processingEnabled) {
            tasks.findByName("upload-$name").mustRunAfter "process-$name"
          }
          else {
            tasks.findByName("upload-$name").mustRunAfter "validate-$name"
          }
        }
      case State.uploaded:
        // nothing to do
      default:
        break
    }
  }.doLast {
    if (alreadyDone) {
      println 'Transformation pipeline was already complete'
    }
    else {
      println 'Transformation pipeline completed'
    }
  }

  // Run -> run complete pipeline

  task("run-${name}", type: GradleBuild) {
    description "Reruns the complete transformation pipeline (Project ${projectId} for model ${config.model})."
    group 'Transformation'

    // retain buildId (to have combined summary)
    startParameter.projectProperties.buildId = project.ext.buildId

    def taskList = ["transform-$name" as String, "validate-$name" as String]
    if (processingEnabled) {
      taskList << ("process-$name" as String)
    }
    if (uploadEnabled) {
      taskList << ("upload-$name" as String)
    }
    tasks = taskList
  }.doFirst {
    // reset any previous result
    baseFolder.deleteDir()
    baseFolder.mkdirs()
  }.doLast {
    println 'Transformation pipeline completed'
  }

}

// continue all

task("continueAll", type: GradleBuild) {
  description "Runs the transformation pipeline from point of last failure for all projects."
  group 'Transformation'

  // retain buildId (to have combined summary)
  startParameter.projectProperties.buildId = project.ext.buildId

  def taskList = []
  transformations.tasks.each { name, config ->
    if (enabledTransformations[name]) {
      taskList << ("continue-$name" as String)
    }
  }
  tasks = taskList
}

// run all

task("runAll", type: GradleBuild) {
  description "Reruns the complete transformation pipeline for all transformations."
  group 'Transformation'

  // retain buildId (to have combined summary)
  startParameter.projectProperties.buildId = project.ext.buildId

  def taskList = []
  transformations.tasks.each { name, config ->
    if (enabledTransformations[name]) {
      taskList << ("run-$name" as String)
    }
  }
  tasks = taskList

  // continue even if tasks failed -> runs all transformations even if one fails (same as --continue command line option)
  startParameter.continueOnFailure = true
}

// build report

class BuildReporter extends BuildAdapter implements TaskExecutionListener {

  final def taskRuns = [:]
  def project

  void beforeExecute(Task task) {
    taskRuns[task.name] = [startTime: java.time.LocalDateTime.now()]
  }

  @Override
  void afterExecute(Task task, TaskState taskState) {
    def t = taskRuns[task.name]
    t.endTime = java.time.LocalDateTime.now()
    
    long s = java.time.Duration.between(t.startTime, t.endTime).getSeconds()
    t.duration = "${(int) (s / 3600)}:${((int) ((s % 3600) / 60) as String).padLeft(2, '0')}:${((s % 60) as String).padLeft(2, '0')}" as String
    t.startTime = t.startTime as String
    t.endTime = t.endTime as String

    t.succeeded = taskState.failure == null
    if (taskState.failure) {
      def error = taskState.failure
      if (error.cause) {
        error = error.cause
      }
      t.error = error.message
    }
  }

  void buildFinished(BuildResult result) {
    if (result.action == 'Build') {
      def report = [
        failure: !!result.failure,
      ]
      if (result.failure) {
        def error = result.failure
        if (error.cause) {
          error = error.cause
        }
        report.error = error.message
      }
      report.tasks = taskRuns

      def summaryFile = project.file("logs/${project.ext.buildId}/summary.yaml")
      summaryFile.parentFile.mkdirs()
      Config.appendYaml(report, summaryFile)
    }
  }

}

def reporter = new BuildReporter(project: project)
gradle.addListener( (TaskExecutionListener) reporter)
gradle.addListener( (BuildListener) reporter)

distributions {
  main {
    distributionBaseName = 'postnas-inspire-alignments'
    contents {
      from('.') {
        include '*'
        include 'gradle/**'
        include 'buildSrc/**'
        include 'adv-alignments/**'
        include 'profiles/**'
        include 'projects/**'
        include 'database/**'
        exclude '.git'
        exclude '.github'
        exclude '.gitignore'
        exclude '.gradle'
        exclude '.releaserc.yml'
        exclude '.vscode'
        exclude '.wetf-repo.yml'
        exclude '*.log'
        exclude 'gradle.properties'
        exclude 'build/**'
        exclude 'workspace'
      }
    }
  }
}

wrapper {
  gradleVersion = '8.10.1'
}