upgrade to tar 1.29

.gitignore

@@ -1,2 +1,4 @@
+aci-builder/files/dgr/usr/bin/tar*
+
 dist/
 target/

.travis.yml

@@ -10,7 +10,8 @@ install:
   - sudo rm -rf /var/lib/apt/lists/*
   - sudo add-apt-repository ppa:duggan/bats --yes
   - sudo apt-get update -qq
-  - sudo apt-get install -qq upx
+  - sudo apt-get install -qq upx coreutils
+  - sudo apt-get install -qq realpath || true
   - sudo ./scripts/travis-install-rkt.sh
   - sudo apt-get install -qq bats
 

aci-builder/bin-run/builder.go

@@ -136,39 +136,36 @@ func (b *Builder) tarAci() error {
 	}
 
 	upperPath := b.pod.Root + PATH_OVERLAY + "/" + upperId + PATH_UPPER
-	upperNamedRootfs := upperPath + "/" + manifestApp(b.pod).Name.String()
-	upperRootfs := upperPath + common.PathRootfs
-
-	if err := os.RemoveAll(upperNamedRootfs + PATH_TMP); err != nil {
-		logs.WithEF(err, b.fields.WithField("path", upperNamedRootfs+PATH_TMP)).Warn("Failed to clean tmp directory")
-	}
-
-	if err := os.Rename(upperNamedRootfs, upperRootfs); err != nil { // TODO this is dirty and can probably be renamed during tar
-		return errs.WithEF(err, b.fields.WithField("path", upperNamedRootfs), "Failed to rename rootfs")
-	}
-	defer os.Rename(upperRootfs, upperNamedRootfs)
-
-	dir, err := os.Getwd()
-	if err != nil {
-		return errs.WithEF(err, b.fields, "Failed to get current working directory")
-	}
-	defer func() {
-		if err := os.Chdir(dir); err != nil {
-			logs.WithEF(err, b.fields.WithField("path", dir)).Warn("Failed to chdir back")
+	rootfsAlias := manifestApp(b.pod).Name.String()
+	destination := b.aciTargetPath + common.PathImageAci // absolute dir, outside upperPath (think: /tmp/…)
+
+	params := []string{"--sort=name", "--numeric-owner", "--exclude", rootfsAlias + PATH_TMP + "/*"}
+	params = append(params, "-C", upperPath, "--transform", "s@^"+rootfsAlias+"@rootfs@")
+	params = append(params, "-cf", destination, common.PathManifest[1:], rootfsAlias)
+
+	logs.WithF(b.fields).Debug("Calling tar to collect all files")
+	if err := common.ExecCmd("tar", params...); err != nil {
+		// In case the host's tar is too old, try the builder's if it exists.
+		stage1Tar := rktcommon.Stage1RootfsPath(b.pod.Root)
+		var buildersTar string
+		for _, t := range []string{"/dgr/usr/bin/tar", "/bin/tar", "/usr/bin/tar"} {
+			buildersTar = filepath.Join(stage1Tar, t)
+			if _, err := os.Stat(buildersTar); err == nil {
+				break
+			}
 		}
-	}()
 
-	if err := os.Chdir(upperPath); err != nil {
-		return errs.WithEF(err, b.fields.WithField("path", upperPath), "Failed to chdir to upper base path")
-	}
-	if err := common.Tar(b.aciTargetPath+common.PathImageAci, common.PathManifest[1:], common.PathRootfs[1:]+"/"); err != nil {
-		return errs.WithEF(err, b.fields, "Failed to tar aci")
+		if err2 := common.ExecCmd(buildersTar, params...); err2 != nil {
+			// If that failed, output the original error nevertheless.
+			logs.WithFields(b.fields).WithField("params", params).Debug("Parameters to 'tar' within the builder")
+			return errs.WithEF(err, b.fields, "Failed to tar aci")
+		}
+		logs.WithF(b.fields).Debug("Had to resort to 'tar' from the builder to create the aci file")
 	}
 	// common.ExecCmd sometimes silently fails, hence the redundant check.
-	if _, err := os.Stat(b.aciTargetPath + common.PathImageAci); os.IsNotExist(err) {
+	if _, err := os.Stat(destination); os.IsNotExist(err) {
 		return errs.WithEF(err, b.fields, "Expected aci has not been created")
 	}
-	logs.WithField("path", dir).Debug("chdir")
 	return nil
 }
 

aci-builder/build-tar.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -euo pipefail
+if [[ ! -z ${debug+x} ]]; then
+  set -x
+fi
+
+start=$(date +%s)
+dir="$(dirname $0)"
+mkdir -p "${dir}/files/dgr/usr/bin"
+: ${tar:="$(realpath "${dir}/files/dgr/usr/bin/tar")"}
+
+echo -e "\033[0;32mBuilding tar\033[0m\n"
+
+# 'core2' is very old, but without setting an option some GCC versions
+# will pick built-in defaults – which could be as "recent" as 'silvermont',
+# thus excluding CPUs without SSE4 (like some pre-2007 AMDs still in the wild).
+: ${CPU_SETTING:="-march=core2 -mtune=intel"}
+if [[ "$(uname -m)" != "x86_64" ]]; then
+  CPU_SETTING=""
+fi
+
+WORKDIR="$(mktemp -d -t aci-builder-tar.XXXXXX)"
+pushd . &>/dev/null
+cd ${WORKDIR}
+curl --silent --show-error --fail --location \
+  -H "accept: application/x-xz, application/x-tar, application/tar+xz" \
+  https://ftp.gnu.org/gnu/tar/tar-1.29.tar.xz \
+| tar --strip-components=1 --xz -x
+
+if ! ./configure --prefix=/usr --libexecdir=/libexec --disable-rpath \
+  CFLAGS="-Os ${CPU_SETTING} -ffunction-sections -fdata-sections -fstack-protector-strong -fpie -fpic" \
+  LDFLAGS="-Wl,-O1 -Wl,-z,relro,-z,now -Wl,--as-needed -Wl,--strip-all -Wl,--gc-sections" >/dev/null; then
+
+  # Old compiler. Most probably in a Travis-CI VM. Resort to conservative settings.
+  echo "== First run of configure failed. Trying fallback options:"
+  ./configure --prefix=/usr --libexecdir=/libexec --disable-rpath \
+    CFLAGS="-Os ${CPU_SETTING%% *} -ffunction-sections -fdata-sections" \
+    LDFLAGS="-Wl,-O1 -Wl,-z,relro,-z,now -Wl,--strip-all -Wl,--gc-sections" >/dev/null
+fi
+make -j$(nproc) >/dev/null
+
+popd &>/dev/null
+mv ${WORKDIR}/src/tar "${dir}/files/dgr/usr/bin/tar"
+rm -r ${WORKDIR}
+
+echo -e "\033[0;32mBuilding tar took: $[ $(date +%s) - ${start} ]s\033[0m\n"
+
+exit 0

aci-builder/build.sh

@@ -16,26 +16,28 @@ mkdir -p ${rootfs}/dgr ${rootfs}/usr/bin
 GOOS=linux GOARCH=amd64 go build --ldflags '-s -w -extldflags "-static"' -o ${rootfs}/dgr/builder/stage1/run ${dir}/bin-run
 upx ${rootfs}/dgr/builder/stage1/run
 
-sudo tar xf ${dir}/rootfs.tar.xz -C ${rootfs}/dgr/
+: ${tar:="$(realpath "${dir}/files/dgr/usr/bin/tar")"}
+
+sudo "${tar}" \
+  --transform "s:usr/sbin/haveged:usr/bin/haveged:" \
+  --exclude "./etc/udev" \
+  --exclude "./usr/share/locale" \
+  --exclude "./usr/libexec" \
+  --exclude "./usr/lib/systemd" \
+  --exclude "./usr/lib/udev" \
+  --exclude "./usr/sbin" \
+  -C ${rootfs}/dgr/ \
+  -xf ${dir}/rootfs.tar.xz
 sudo cp -R ${dir}/files/. ${rootfs}
 sudo chown root: ${rootfs}
 cp ${dir}/manifest.json ${target}/manifest
 sudo cp --no-preserve=ownership ${dist}/templater ${rootfs}/dgr/usr/bin/
 
-# some cleanup
-sudo rm -Rf ${rootfs}/dgr/etc/udev
-sudo rm -Rf ${rootfs}/dgr/usr/share/locale
-sudo rm -Rf ${rootfs}/dgr/usr/libexec
-sudo rm -Rf ${rootfs}/dgr/usr/lib/systemd
-sudo rm -Rf ${rootfs}/dgr/usr/lib/udev
-
-
-sudo mv ${rootfs}/dgr/usr/sbin/haveged ${rootfs}/dgr/usr/bin/haveged
-sudo rm -Rf ${rootfs}/dgr/usr/sbin/
 sudo bash -c "cd ${rootfs}/dgr/usr && ln -s bin sbin && cd -"
 
 cd ${target}
-sudo tar cpfz ../bindata/aci-builder.aci rootfs manifest
+sudo "${tar}" --sort=name --numeric-owner \
+  -cpzf ../bindata/aci-builder.aci manifest rootfs
 sudo chown ${USER}: ../bindata/aci-builder.aci
 sudo rm -Rf rootfs/
 cd -

aci-tester/build.sh

@@ -30,6 +30,9 @@ curl --fail --silent --show-error --location --remote-time --compressed --create
 
 chmod +x ${rootfs}/dgr/usr/bin/*
 
+: ${tar:="$(realpath "${dir}/../aci-builder/files/dgr/usr/bin/tar")"}
 cd ${target}
-tar cpfz ../bindata/aci-tester.aci rootfs manifest
+"${tar}" --sort=name --numeric-owner \
+  --owner=0 --group=0 \
+  -cpzf ../bindata/aci-tester.aci manifest rootfs
 cd -

gomake.cfg

@@ -4,6 +4,10 @@ release_osarchi=linux-amd64
 upx=true
 
 pre-build() {
+    local tar="$(realpath "$(dirname $0)/aci-builder/files/dgr/usr/bin/tar")"
+    if [[ ! -x "${tar}" ]] || ! "${tar}" --help | grep -q -F sort; then
+        ${work_path}/aci-builder/build-tar.sh
+    fi
     [ -f ${work_path}/${target_name}/templater ] || ${work_path}/bin-templater/build.sh
     [ -f ${work_path}/${target_name}/bindata/aci-tester.aci ] || ${work_path}/aci-tester/build.sh
     [ -f ${work_path}/${target_name}/bindata/aci-builder.aci ] || ${work_path}/aci-builder/build.sh