azure-pipelines.yml

trigger:
- master

schedules:
- cron: "30 3 * * *"
  displayName: Daily early morning run
  branches:
    include:
    - master
  always: true

variables:
  vmImageName: 'ubuntu-18.04'

pool:
  vmImage: $(vmImageName)

#container:
#    image: ubuntu:latest
    # options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
# fix for sudo
#  steps:
#    - script: |
#        /tmp/docker exec -t -u 0 ci-container \
#        sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
#      displayName: Set up sudo

jobs:
### TEST JOB ###
  - job: Test
    displayName: Test job only
    condition: eq(variables['runTestJob'], 'true')
    variables:
      ramdisk: $(System.DefaultWorkingDirectory)/ramdisk
    steps: 
    # - checkout: none
    - script: |
        pwd
        mkdir $(ramdisk)
        sudo mount -t tmpfs -o rw,size=3G tmpfs $(ramdisk)
        ls -lah $(ramdisk)
        sudo chmod 777 $(ramdisk)
        touch $(ramdisk)/foobar
        ls -lah $(ramdisk)/foobar
        mount 
        sudo umount $(ramdisk)
        echo \"foo\" | tr -d '"'
        echo $(Agent.Name)
        echo $(Agent.HomeDirectory)
        echo $(Agent.BuildDirectory)
        ls -la /home/vsts
        env | grep PATH
        env | grep HOME
        bash scripts/addpeers_2.0.sh
        free -m
        mkdir ./ramdisk
        # sudo mount -t tmpfs -o rw,size=2G tmpfs ./ramdisk
        # sudo chmod 777 ./ramdisk
        # sudo find /lib/modules/$(uname -r) -type f -name '*.ko'
        # sudo find /lib/modules/$(uname -r) -type f -name '*.ko*'
        # sudo depmod -av|grep f2fs
        uname -r
      displayName: Test commands


### JOB ###
  - job: Build
    displayName: Build and publish tezos binaries
    # condition: and(always(), eq(variables['Build.Reason'], 'Schedule'))
    variables:
      tezosLastCommit: unknown
      tezosLastCommitStamp: unknown
      recentCommitUpdate: false
    steps:
    - checkout: none

    - task: PowerShell@2
      displayName: Set date time stamp variable
      inputs:
        targetType: 'inline'
        script: |
          $Time = Get-Date
          $Time = $Time.ToUniversalTime()
          $date = $Time.ToString("yyyy-MM-dd HH:mmK")
          Write-Host "##vso[task.setvariable variable=datetimestamp]$date"
          $UnixStamp = Get-Date -UFormat %s
          Write-Host "##vso[task.setvariable variable=unixdatetimestamp]$UnixStamp"

    - script: |
        git clone -b $(gitCheckoutBranch) --depth 1 $(gitCloneUrl)
        cd tezos
        commit_id=$(git log --format="%h" -n 1) && echo $commit_id
        stamp=$(git log -1 --format=%cd -n 1) && echo $stamp
        unix_git_stamp=$(git log -1 --format=%ct -n 1)
        echo "##vso[task.setvariable variable=tezosLastCommit]$commit_id"
        echo "##vso[task.setvariable variable=tezosLastCommitStamp]$stamp"
        current_unix_stamp=$(date +"%s")
        need_update=false
        if (( current_unix_stamp-unix_git_stamp < $(lookbackDurationCommitUpdate) )); then need_update=true && echo "found recent commit, shall build"; fi
        [[ $(forceBuild) == true ]] && need_update=true
        echo "##vso[task.setvariable variable=recentCommitUpdate]$need_update"
      displayName: Checkout tezos

# note: variables are not available immediately within the same script block after being set with task.setvariable
    - script: sudo add-apt-repository ppa:avsm/ppa && sudo apt-get update && sudo apt-get remove -y --purge man-db && sudo apt-get install -yy -qq opam rsync m4 build-essential patch unzip bubblewrap wget libev-dev libgmp-dev libhidapi-dev m4 perl pkg-config
      displayName: Install opam and dependencies
      condition: eq(variables['recentCommitUpdate'], true)

    - script: |
        opam init --bare -n
        eval $(opam env)
        cd tezos && ls -la && make build-deps && eval $(opam env) && make && ls -lah tezos-* && mkdir ../bin && mv -v tezos-* ../bin/
      displayName: Build tezos
      condition: eq(variables['recentCommitUpdate'], true)

    - publish: '$(System.DefaultWorkingDirectory)/bin/'
      artifact: tezos-binaries
      displayName: Publish tezos binaries
      condition: eq(variables['recentCommitUpdate'], true)


    - task: UniversalPackages@0
      displayName: Publish tezos binaries as universal package
      condition: eq(variables['recentCommitUpdate'], true)
      inputs:
        command: publish
        publishDirectory: '$(System.DefaultWorkingDirectory)/bin/'
        vstsFeedPublish: 'tezos-binaries'
        vstsFeedPackagePublish: 'tezos-binaries-ubuntu-azure'
        packagePublishDescription: 'Tezos binaries [$(tezosLastCommitStamp) / $(tezosLastCommit)] built on $(datetimestamp) within Azure Pipeline running on $(vmImageName).'

    - script: |
        docker pull tezos/tezos
        docker create -ti --name tz tezos/tezos:mainnet
        mkdir ./bin && pwd
        docker cp tz:/usr/local/bin/tezos-node ./bin/
        ls bin -la
        docker rm -f tz
      displayName: Get tezos-node binary from container image
      condition: false

### JOB ###
  - job: getUpdatedSnapshot
    timeoutInMinutes: 360
    displayName: Download, import snapshot, setup node and run
    variables: 
      snapshotFileName: snapshot.bootstrap
      ramdisk: $(System.DefaultWorkingDirectory)/ramdisk
      # experimental-rolling or full
    steps:
    - script: sudo apt-get update && sudo apt-get install -yy -qq jq dnsutils libev-dev libgmp-dev libhidapi-dev
      displayName: Install packages
    - script: |
        curl $(fullSnapshotUrl) -o $(snapshotFileName).xz
      displayName: Download snapshot
      condition: eq(variables['useExternalSnapshot'], true)

    - task: DownloadPipelineArtifact@2
      displayName: Download full snapshot from previous build
      condition: eq(variables['useExternalSnapshot'], false)
      inputs:
        buildType: 'specific'
        artifact: 'snapshots-for-pipeline'
        targetPath: $(Pipeline.Workspace)/snapshots-for-pipeline
        definition: $(System.DefinitionId)
        project: $(System.TeamProjectId)
        # pipelineId: 12 # the build id from which to download the artifacts. For example: 1764
        buildVersionToDownload: 'latest'
    - script: |
        ls -lah $(Pipeline.Workspace)/snapshots-for-pipeline
        mv -v $(Pipeline.Workspace)/snapshots-for-pipeline/snapshot-from-build.full.xz $(snapshotFileName).xz
        latest_block=$(cat $(Pipeline.Workspace)/snapshots-for-pipeline/snapshot-latest-block.info)
        echo "##vso[task.setvariable variable=importBlock]$latest_block"
      displayName: Prepare snapshot from artifact storage, previous build
      condition: eq(variables['useExternalSnapshot'], false)

    - script: |
        xz -l $(snapshotFileName).xz
        sb=$(xz --robot -l $(snapshotFileName).xz | grep file |  awk '{print $5}')
        let sb+=1024
        mkdir $(ramdisk)
        sudo mount -t tmpfs -o rw,size=$sb tmpfs $(ramdisk)
        sudo chmod 777 $(ramdisk)
        sudo mount | grep $(ramdisk)
        xz $(snapshotFileName).xz -d -c > $(ramdisk)/$(snapshotFileName) && rm $(snapshotFileName).xz && ls -lah $(ramdisk)/$(snapshotFileName)
      displayName: Extract snapshot
   
    - task: UniversalPackages@0
      displayName: 'Download latest tezos binaries'
      inputs:
        command: download
        vstsFeed: 'tezos-binaries'
        vstsFeedPackage: 'tezos-binaries-ubuntu-azure'
        vstsPackageVersion: '*' # = latest version
        downloadDirectory: '$(System.DefaultWorkingDirectory)/bin'

    - script: |
        chmod +x ./bin/*
        [[ ! -z "$(importBlock)" ]] && _block_arg="--block=$(importBlock)"
        echo "./bin/tezos-node snapshot import $(ramdisk)/$(snapshotFileName) $(tezosNodeImportOptions) ${_block_arg}"
        ./bin/tezos-node snapshot import $(ramdisk)/$(snapshotFileName) $(tezosNodeImportOptions) ${_block_arg}
      displayName: Import snapshot

    - script: free -m && sudo umount $(ramdisk) && mount | grep tmpfs && free -m
      displayName: Remove ramdisk

    - task: DownloadPipelineArtifact@2
      displayName: Download tezos config from previous build
      inputs:
        buildType: 'specific'
        artifact: 'node-config'
        targetPath: $(Pipeline.Workspace)/node-config
        definition: $(System.DefinitionId)
        project: $(System.TeamProjectId)
        # pipelineId: 12 # the build id from which to download the artifacts. For example: 1764
        buildVersionToDownload: 'latest'

    - script: | 
        ls -la $(Pipeline.Workspace)/snapshots-for-pipeline || true
        ls -la $(Pipeline.Workspace)/node-config || true
      displayName: '[INFO] List downloaded artifacts in workspace folder'

    # move existing config if available, then set condition for this task below
    - script: |
        ./bin/tezos-node identity generate
        ./bin/tezos-node config init
        # copy back stuff from previous build
        [[ -f $(Pipeline.Workspace)/node-config/peers.json ]] && mv -v $(Pipeline.Workspace)/node-config/peers.json /home/vsts/.tezos-node/
      displayName: Setup tezos node, restore config from artifact

    - script: |
        timeout 120 ./bin/tezos-node run $(tezosNodeOptions) || true
      displayName: Run tezos node in foreground for 2 minutes

    - script: |
        nohup ./bin/tezos-node run $(tezosNodeOptions) </dev/null >/dev/null 2>&1 &
      displayName: Run tezos node in background for a fixed time

    - script: |
        ps auxw | grep tezos-node
        du -hs /home/vsts/.tezos-node/ || true
        sleep 10
        bash scripts/addpeers_2.0.sh || true
        echo "Waiting for node to be bootstrapped"
        ./bin/tezos-client bootstrapped
        ./bin/tezos-client get timestamp
        du -hs /home/vsts/.tezos-node/ || true
      displayName: Run add peers script, wait for node to be bootstrapped

    - script: |
        sleep $(chainSyncDurationSeconds)
        _query_result=$(./bin/tezos-client rpc get /chains/main/blocks/head~10)
        # get information about the tip of the chain in one call, so it's consistent
        b=$(echo $_query_result | jq '.hash' | tr -d '"')
        l=$(echo $_query_result | jq '.header.level')
        t=$(echo $_query_result | jq '.header.timestamp' | tr -d '"' | sed 's/:/./g')
        echo "##vso[task.setvariable variable=block]$b"
        echo "##vso[task.setvariable variable=blockLevel]$l"
        echo "##vso[task.setvariable variable=blockTimeStamp]$t"
        pkill tezos-node
        sleep 10
        ps auxw | grep tezos-node
        pkill -9 tezos-node || true
      displayName: Wait to fully sync chain and terminate tezos-node
      name: endsync

    - task: CopyFiles@2
      displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)'
      inputs:
        SourceFolder: /home/vsts/.tezos-node/
        Contents: |
          **/*.json
          !/context
          !/store
        TargetFolder: '$(Build.ArtifactStagingDirectory)'

    - script: ls -la $(Build.ArtifactStagingDirectory)
      displayName: '[INFO] List contents of tezos config artifact directory.'

    - publish: '$(Build.ArtifactStagingDirectory)'
      displayName: 'Publish artifact: Node settings json'
      artifact: node-config

    - script: | 
        echo $(block) && echo $(blockLevel) && echo $(blockTimeStamp)
        mkdir ./snapshots
      displayName: Prepare for snapshot exports

    - task: PowerShell@2
      displayName: Set date time stamp variable
      inputs:
        targetType: 'inline'
        script: |
          $Time = Get-Date
          $Time = $Time.ToUniversalTime()
          $date = $Time.ToString("yyyy-MM-dd HH:mmK")
          Write-Host "##vso[task.setvariable variable=datetimestamp]$date"
        errorActionPreference: 'continue'
  
    - script: |
        echo "##vso[task.setvariable variable=rollingSnapshotFileName]./snapshots/mainnet.roll.$(date +%F_%H-%M).$(block).chain"
        echo "##vso[task.setvariable variable=fullSnapshotFileName]./snapshots/mainnet.full.$(date +%F_%H-%M)_$(block).chain"
      displayName: Set snapshot filenames

    - script: |
        echo $(rollingSnapshotFileName)
        ./bin/tezos-node snapshot export $(rollingSnapshotFileName) --rolling --block=$(block)
      displayName: Export rolling snapshot

    - script: |
        echo $(fullSnapshotFileName)
        ./bin/tezos-node snapshot export $(fullSnapshotFileName) --block=$(block)
      displayName: Export full snapshot

    - script: |
        jq -jr '.[] | select(.last_established_connection != null) | .last_established_connection | .[0] | "[",.addr,"]:",.port,"\n"' /home/vsts/.tezos-node/peers.json > ./snapshots/public_peers_tezos_mainnet.list
        tail ./snapshots/public_peers_tezos_mainnet.list || true
      displayName: Export a list of public peers from peers.json

    # TODO: Add checksum files
    - script: |
        xz -8 $(rollingSnapshotFileName)
        xz -l $(rollingSnapshotFileName).xz
        xz -8 $(fullSnapshotFileName)
        xz -l $(fullSnapshotFileName).xz
        sha256sum ./snapshots/*.xz > checksums.sha256
        cat ./snapshots/checksums.sha256 || true
      displayName: Compress snapshots

    - script: ls -la $(System.DefaultWorkingDirectory) && echo $(System.DefaultWorkingDirectory)
      displayName: '[INFO] List working directory'


    # Publish both snapshots
    - task: CopyFiles@2
      displayName: Prepare publish rolling and full snapshot to artifact staging
      inputs:
        sourceFolder: '$(System.DefaultWorkingDirectory)/snapshots'
        contents: '**' # recursive and everything, including the .list file for example
        targetFolder: $(Build.ArtifactStagingDirectory)
        cleanTargetFolder: true
        overWrite: true
        flattenFolders: false 
        preserveTimestamp: false 

    - script: echo $(Build.ArtifactStagingDirectory) && find $(Build.ArtifactStagingDirectory)
      displayName: '[INFO] List contents of artifact staging directory'

    # rename full snapshot for next build, after copying to staging directory above
    - script: |
        mv -v $(fullSnapshotFileName).xz snapshot-from-build.full.xz
        echo $(block) > snapshot-latest-block.info
      displayName: Rename current full snapshot to generic name for next build

    - publish: 'snapshot-from-build.full.xz'
      displayName: 'Save full snapshot for next sync (internal).'
      artifact: snapshots-for-pipeline


    - task: GithubRelease@0
      displayName: 'Create GitHub Release for snapshots'
      inputs:
        isPreRelease: false
        title: 'Tezos Mainnet Snapshots ($(datetimestamp))'
        gitHubConnection: phlogi-personal
        repositoryName: phlogi/tezos-snapshots
        tagSource: manual
        releaseNotesSource: input
        releaseNotes: Rolling and full snapshots up to level $(blockLevel) at $(blockTimeStamp) and block $(block). Current list of public tezos peers. 
        addChangeLog: false
        tag: $(Build.BuildNumber)
        assets: |
            $(Build.ArtifactStagingDirectory)/**