Merge pull request PowerDNS#12764 from fredmorcos/ci-fixes-cleanups

Ci fixes and cleanups

regression-tests.dnsdist/requirements.txt

@@ -3,6 +3,7 @@ nose>=1.3.7
 libnacl>=1.4.3,<1.7
 requests>=2.1.0
 protobuf>=3.0
+pyasn1==0.4.8
 pysnmp>=4.3.4
 future>=0.17.1
 pycurl>=7.43.0

regression-tests.recursor-dnssec/requirements.txt

@@ -2,6 +2,7 @@ dnspython>=1.11
 nose>=1.3.7
 protobuf>=2.5; sys_platform != 'darwin'
 protobuf>=3.0; sys_platform == 'darwin'
+pyasn1==0.4.8
 pysnmp>=4.3.4
 requests>=2.1.0
 Twisted>0.15.0

regression-tests/tests/.gitignore

@@ -3,3 +3,4 @@ real_result
 *.out
 start
 step.*
+verify-dnssec-zone/allow-missing

tasks.py

@@ -46,6 +46,7 @@
     'ruby-dev',
     'sqlite3',
     'unixodbc-dev',
+    'cmake',
 ]
 rec_build_deps = [
     'libcap-dev',
@@ -165,7 +166,8 @@ def install_libdecaf(c, product):
     c.run('git clone https://git.code.sf.net/p/ed448goldilocks/code /tmp/libdecaf')
     with c.cd('/tmp/libdecaf'):
         c.run('git checkout 41f349')
-        c.run('cmake -B build '
+        c.run('CC=clang-12 CXX=clang-12 '
+              'cmake -B build '
               '-DCMAKE_INSTALL_PREFIX=/usr/local '
               '-DCMAKE_INSTALL_LIBDIR=lib '
               '-DENABLE_STATIC=OFF '
@@ -302,9 +304,18 @@ def ci_docs_build_pdf(c):
 
 @task
 def ci_docs_upload_master(c, docs_host, pdf, username, product, directory=""):
-    c.run(f"rsync -crv --delete --no-p --chmod=g=rwX --exclude '*~' ./docs/_build/{product}-html-docs/ {username}@{docs_host}:{directory}")
-    c.run(f"rsync -crv --no-p --chmod=g=rwX --exclude '*~' ./docs/_build/{product}-html-docs.tar.bz2 {username}@{docs_host}:{directory}/html-docs.tar.bz2")
-    c.run(f"rsync -crv --no-p --chmod=g=rwX --exclude '*~' ./docs/_build/latex/{pdf} {username}@{docs_host}:{directory}")
+    rsync_cmd = " ".join([
+        "rsync",
+        "--checksum",
+        "--recursive",
+        "--verbose",
+        "--no-p",
+        "--chmod=g=rwX",
+        "--exclude '*~'",
+    ])
+    c.run(f"{rsync_cmd} --delete ./docs/_build/{product}-html-docs/ {username}@{docs_host}:{directory}")
+    c.run(f"{rsync_cmd} ./docs/_build/{product}-html-docs.tar.bz2 {username}@{docs_host}:{directory}/html-docs.tar.bz2")
+    c.run(f"{rsync_cmd} ./docs/_build/latex/{pdf} {username}@{docs_host}:{directory}")
 
 @task
 def ci_docs_add_ssh(c, ssh_key, host_key):
@@ -313,59 +324,122 @@ def ci_docs_add_ssh(c, ssh_key, host_key):
     c.run('chmod 600 ~/.ssh/id_ed25519')
     c.run(f'echo "{host_key}" > ~/.ssh/known_hosts')
 
+
+def get_sanitizers():
+    sanitizers = os.getenv('SANITIZERS')
+    if sanitizers != '':
+        sanitizers = sanitizers.split('+')
+        sanitizers = ['--enable-' + sanitizer for sanitizer in sanitizers]
+        sanitizers = ' '.join(sanitizers)
+    return sanitizers
+
+
+def get_cflags():
+    return " ".join([
+        "-O1",
+        "-Werror=vla",
+        "-Werror=shadow",
+        "-Wformat=2",
+        "-Werror=format-security",
+        "-Werror=string-plus-int",
+    ])
+
+
+def get_cxxflags():
+    return " ".join([
+        get_cflags(),
+        "-Wp,-D_GLIBCXX_ASSERTIONS",
+    ])
+
+
+def get_base_configure_cmd():
+    return " ".join([
+        f'CFLAGS="{get_cflags()}"',
+        f'CXXFLAGS="{get_cxxflags()}"',
+        './configure',
+        "CC='clang-12'",
+        "CXX='clang++-12'",
+        "--enable-option-checking=fatal",
+        "--enable-systemd",
+        "--with-libsodium",
+        "--enable-fortify-source=auto",
+        "--enable-auto-var-init=pattern",
+    ])
+
+
 @task
 def ci_auth_configure(c):
-    sanitizers = ' '.join('--enable-'+x for x in os.getenv('SANITIZERS').split('+')) if os.getenv('SANITIZERS') != '' else ''
-    unittests = ' --enable-unit-tests --enable-backend-unit-tests' if os.getenv('UNIT_TESTS') == 'yes' else ''
-    fuzzingtargets = ' --enable-fuzz-targets' if os.getenv('FUZZING_TARGETS') == 'yes' else ''
-    res = c.run('''CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \
-                   CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \
-                   ./configure \
-                      CC='clang-12' \
-                      CXX='clang++-12' \
-                      LDFLAGS='-L/usr/local/lib -Wl,-rpath,/usr/local/lib' \
-                      --enable-option-checking=fatal \
-                      --with-modules='bind geoip gmysql godbc gpgsql gsqlite3 ldap lmdb lua2 pipe remote tinydns' \
-                      --enable-systemd \
-                      --enable-tools \
-                      --enable-fuzz-targets \
-                      --enable-experimental-pkcs11 \
-                      --enable-experimental-gss-tsig \
-                      --enable-remotebackend-zeromq \
-                      --with-lmdb=/usr \
-                      --with-libsodium \
-                      --with-libdecaf \
-                      --prefix=/opt/pdns-auth \
-                      --enable-ixfrdist \
-                      --enable-fortify-source=auto \
-                      --enable-auto-var-init=pattern ''' + sanitizers + unittests + fuzzingtargets, warn=True)
+    sanitizers = get_sanitizers()
+
+    unittests = os.getenv('UNIT_TESTS')
+    if unittests == 'yes':
+        unittests = '--enable-unit-tests --enable-backend-unit-tests'
+    else:
+        unittests = ''
+
+    fuzz_targets = os.getenv('FUZZING_TARGETS')
+    fuzz_targets = '--enable-fuzz-targets' if fuzz_targets == 'yes' else ''
+
+    modules = " ".join([
+        "bind",
+        "geoip",
+        "gmysql",
+        "godbc",
+        "gpgsql",
+        "gsqlite3",
+        "ldap",
+        "lmdb",
+        "lua2",
+        "pipe",
+        "remote",
+        "tinydns",
+    ])
+    configure_cmd = " ".join([
+        get_base_configure_cmd(),
+        "LDFLAGS='-L/usr/local/lib -Wl,-rpath,/usr/local/lib'",
+        f"--with-modules='{modules}'",
+        "--enable-tools",
+        "--enable-experimental-pkcs11",
+        "--enable-experimental-gss-tsig",
+        "--enable-remotebackend-zeromq",
+        "--with-lmdb=/usr",
+        "--with-libdecaf",
+        "--prefix=/opt/pdns-auth",
+        "--enable-ixfrdist",
+        sanitizers,
+        unittests,
+        fuzz_targets,
+    ])
+    res = c.run(configure_cmd, warn=True)
     if res.exited != 0:
         c.run('cat config.log')
         raise UnexpectedExit(res)
+
+
 @task
 def ci_rec_configure(c):
-    sanitizers = ' '.join('--enable-'+x for x in os.getenv('SANITIZERS').split('+')) if os.getenv('SANITIZERS') != '' else ''
-    unittests = ' --enable-unit-tests' if os.getenv('UNIT_TESTS') == 'yes' else ''
-    res = c.run('''            CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \
-            CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \
-            ./configure \
-              CC='clang-12' \
-              CXX='clang++-12' \
-              --enable-option-checking=fatal \
-              --enable-nod \
-              --enable-systemd \
-              --prefix=/opt/pdns-recursor \
-              --with-libsodium \
-              --with-lua=luajit \
-              --with-libcap \
-              --with-net-snmp \
-              --enable-fortify-source=auto \
-              --enable-auto-var-init=pattern \
-              --enable-dns-over-tls ''' + sanitizers + unittests, warn=True)
+    sanitizers = get_sanitizers()
+
+    unittests = os.getenv('UNIT_TESTS')
+    unittests = '--enable-unit-tests' if unittests == 'yes' else ''
+
+    configure_cmd = " ".join([
+        get_base_configure_cmd(),
+        "--enable-nod",
+        "--prefix=/opt/pdns-recursor",
+        "--with-lua=luajit",
+        "--with-libcap",
+        "--with-net-snmp",
+        "--enable-dns-over-tls",
+        sanitizers,
+        unittests,
+    ])
+    res = c.run(configure_cmd, warn=True)
     if res.exited != 0:
         c.run('cat config.log')
         raise UnexpectedExit(res)
 
+
 @task
 def ci_dnsdist_configure(c, features):
     additional_flags = ''