From 7a70482e1fb5e2255b2a034b35f59c37b479443e Mon Sep 17 00:00:00 2001 From: W-Mark Kubacki Date: Sun, 5 Mar 2017 17:28:16 +0100 Subject: [PATCH 1/3] CI: install realpath (or update coreutils) Travis, the current CI system, uses a very old Ubuntu distribution whose package 'coreutils' does not include 'realpath'. Or, 'coreutils' has had it but the CI image maintainer has stripped it from the final image for consistency's sake. Anyway, this gets us realpath in one way or another. --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3a262c00..bec76df2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,8 @@ install: - sudo rm -rf /var/lib/apt/lists/* - sudo add-apt-repository ppa:duggan/bats --yes - sudo apt-get update -qq - - sudo apt-get install -qq upx + - sudo apt-get install -qq upx coreutils + - sudo apt-get install -qq realpath || true - sudo ./scripts/travis-install-rkt.sh - sudo apt-get install -qq bats From 498b3b3e9539b75aa59f5830c31e809df5301fa9 Mon Sep 17 00:00:00 2001 From: W-Mark Kubacki Date: Mon, 28 Nov 2016 16:24:25 +0100 Subject: [PATCH 2/3] aci-builder: use the most recent version of 'tar' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to have the most recent 'tar' – which is necessary for reproducible builds – we build it ourselves. Future commits will use the henceforth available options `--sort`, `--clamp-mtime`, `--exclude`, as well as `--transform`. --- .gitignore | 2 ++ aci-builder/build-tar.sh | 49 ++++++++++++++++++++++++++++++++++++++++ aci-builder/build.sh | 26 +++++++++++---------- aci-tester/build.sh | 5 +++- gomake.cfg | 4 ++++ 5 files changed, 73 insertions(+), 13 deletions(-) create mode 100755 aci-builder/build-tar.sh diff --git a/.gitignore b/.gitignore index 92d6b778..e370ceda 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ +aci-builder/files/dgr/usr/bin/tar* + dist/ target/ \ No newline at end of file diff --git a/aci-builder/build-tar.sh b/aci-builder/build-tar.sh new file mode 100755 index 00000000..c39eafc1 --- /dev/null +++ b/aci-builder/build-tar.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -euo pipefail +if [[ ! -z ${debug+x} ]]; then + set -x +fi + +start=$(date +%s) +dir="$(dirname $0)" +mkdir -p "${dir}/files/dgr/usr/bin" +: ${tar:="$(realpath "${dir}/files/dgr/usr/bin/tar")"} + +echo -e "\033[0;32mBuilding tar\033[0m\n" + +# 'core2' is very old, but without setting an option some GCC versions +# will pick built-in defaults – which could be as "recent" as 'silvermont', +# thus excluding CPUs without SSE4 (like some pre-2007 AMDs still in the wild). +: ${CPU_SETTING:="-march=core2 -mtune=intel"} +if [[ "$(uname -m)" != "x86_64" ]]; then + CPU_SETTING="" +fi + +WORKDIR="$(mktemp -d -t aci-builder-tar.XXXXXX)" +pushd . &>/dev/null +cd ${WORKDIR} +curl --silent --show-error --fail --location \ + -H "accept: application/x-xz, application/x-tar, application/tar+xz" \ + https://ftp.gnu.org/gnu/tar/tar-1.29.tar.xz \ +| tar --strip-components=1 --xz -x + +if ! ./configure --prefix=/usr --libexecdir=/libexec --disable-rpath \ + CFLAGS="-Os ${CPU_SETTING} -ffunction-sections -fdata-sections -fstack-protector-strong -fpie -fpic" \ + LDFLAGS="-Wl,-O1 -Wl,-z,relro,-z,now -Wl,--as-needed -Wl,--strip-all -Wl,--gc-sections" >/dev/null; then + + # Old compiler. Most probably in a Travis-CI VM. Resort to conservative settings. + echo "== First run of configure failed. Trying fallback options:" + ./configure --prefix=/usr --libexecdir=/libexec --disable-rpath \ + CFLAGS="-Os ${CPU_SETTING%% *} -ffunction-sections -fdata-sections" \ + LDFLAGS="-Wl,-O1 -Wl,-z,relro,-z,now -Wl,--strip-all -Wl,--gc-sections" >/dev/null +fi +make -j$(nproc) >/dev/null + +popd &>/dev/null +mv ${WORKDIR}/src/tar "${dir}/files/dgr/usr/bin/tar" +rm -r ${WORKDIR} + +echo -e "\033[0;32mBuilding tar took: $[ $(date +%s) - ${start} ]s\033[0m\n" + +exit 0 diff --git a/aci-builder/build.sh b/aci-builder/build.sh index 5e3e5c9b..0c980608 100755 --- a/aci-builder/build.sh +++ b/aci-builder/build.sh @@ -16,26 +16,28 @@ mkdir -p ${rootfs}/dgr ${rootfs}/usr/bin GOOS=linux GOARCH=amd64 go build --ldflags '-s -w -extldflags "-static"' -o ${rootfs}/dgr/builder/stage1/run ${dir}/bin-run upx ${rootfs}/dgr/builder/stage1/run -sudo tar xf ${dir}/rootfs.tar.xz -C ${rootfs}/dgr/ +: ${tar:="$(realpath "${dir}/files/dgr/usr/bin/tar")"} + +sudo "${tar}" \ + --transform "s:usr/sbin/haveged:usr/bin/haveged:" \ + --exclude "./etc/udev" \ + --exclude "./usr/share/locale" \ + --exclude "./usr/libexec" \ + --exclude "./usr/lib/systemd" \ + --exclude "./usr/lib/udev" \ + --exclude "./usr/sbin" \ + -C ${rootfs}/dgr/ \ + -xf ${dir}/rootfs.tar.xz sudo cp -R ${dir}/files/. ${rootfs} sudo chown root: ${rootfs} cp ${dir}/manifest.json ${target}/manifest sudo cp --no-preserve=ownership ${dist}/templater ${rootfs}/dgr/usr/bin/ -# some cleanup -sudo rm -Rf ${rootfs}/dgr/etc/udev -sudo rm -Rf ${rootfs}/dgr/usr/share/locale -sudo rm -Rf ${rootfs}/dgr/usr/libexec -sudo rm -Rf ${rootfs}/dgr/usr/lib/systemd -sudo rm -Rf ${rootfs}/dgr/usr/lib/udev - - -sudo mv ${rootfs}/dgr/usr/sbin/haveged ${rootfs}/dgr/usr/bin/haveged -sudo rm -Rf ${rootfs}/dgr/usr/sbin/ sudo bash -c "cd ${rootfs}/dgr/usr && ln -s bin sbin && cd -" cd ${target} -sudo tar cpfz ../bindata/aci-builder.aci rootfs manifest +sudo "${tar}" --sort=name --numeric-owner \ + -cpzf ../bindata/aci-builder.aci manifest rootfs sudo chown ${USER}: ../bindata/aci-builder.aci sudo rm -Rf rootfs/ cd - diff --git a/aci-tester/build.sh b/aci-tester/build.sh index c779d028..29bfe4ed 100755 --- a/aci-tester/build.sh +++ b/aci-tester/build.sh @@ -30,6 +30,9 @@ curl --fail --silent --show-error --location --remote-time --compressed --create chmod +x ${rootfs}/dgr/usr/bin/* +: ${tar:="$(realpath "${dir}/../aci-builder/files/dgr/usr/bin/tar")"} cd ${target} -tar cpfz ../bindata/aci-tester.aci rootfs manifest +"${tar}" --sort=name --numeric-owner \ + --owner=0 --group=0 \ + -cpzf ../bindata/aci-tester.aci manifest rootfs cd - diff --git a/gomake.cfg b/gomake.cfg index d81d3150..03fb9576 100644 --- a/gomake.cfg +++ b/gomake.cfg @@ -4,6 +4,10 @@ release_osarchi=linux-amd64 upx=true pre-build() { + local tar="$(realpath "$(dirname $0)/aci-builder/files/dgr/usr/bin/tar")" + if [[ ! -x "${tar}" ]] || ! "${tar}" --help | grep -q -F sort; then + ${work_path}/aci-builder/build-tar.sh + fi [ -f ${work_path}/${target_name}/templater ] || ${work_path}/bin-templater/build.sh [ -f ${work_path}/${target_name}/bindata/aci-tester.aci ] || ${work_path}/aci-tester/build.sh [ -f ${work_path}/${target_name}/bindata/aci-builder.aci ] || ${work_path}/aci-builder/build.sh From f753652c8e3c64a59a99590afb90952f6572c6f7 Mon Sep 17 00:00:00 2001 From: W-Mark Kubacki Date: Mon, 28 Nov 2016 16:18:31 +0100 Subject: [PATCH 3/3] aci, builder: utilize the new tar for ACI files Enables dgr to work on hosts that don't have any tar. (See also #217.) Sorts the contents to have a reproducible order, but pulls the manifest file to the front for fast access. Sorted contents of ACIs allow for easier comparison, and usage of tools such as zsync, and deduplication on the server. The price, sorting by 'tar', is cheap. zap the "chdir-acrobatique", use `tar -C`: dgr changes paths and performs needless renames, which result in mayhem if the process quits prematurely or the timing were off. The solution is to use tar's `-C` param and transform the filenames. closes #210 --- aci-builder/bin-run/builder.go | 51 ++++++++++++++++------------------ dgr/common/tar.go | 11 -------- 2 files changed, 24 insertions(+), 38 deletions(-) delete mode 100644 dgr/common/tar.go diff --git a/aci-builder/bin-run/builder.go b/aci-builder/bin-run/builder.go index 56455238..4740e2d1 100644 --- a/aci-builder/bin-run/builder.go +++ b/aci-builder/bin-run/builder.go @@ -136,39 +136,36 @@ func (b *Builder) tarAci() error { } upperPath := b.pod.Root + PATH_OVERLAY + "/" + upperId + PATH_UPPER - upperNamedRootfs := upperPath + "/" + manifestApp(b.pod).Name.String() - upperRootfs := upperPath + common.PathRootfs - - if err := os.RemoveAll(upperNamedRootfs + PATH_TMP); err != nil { - logs.WithEF(err, b.fields.WithField("path", upperNamedRootfs+PATH_TMP)).Warn("Failed to clean tmp directory") - } - - if err := os.Rename(upperNamedRootfs, upperRootfs); err != nil { // TODO this is dirty and can probably be renamed during tar - return errs.WithEF(err, b.fields.WithField("path", upperNamedRootfs), "Failed to rename rootfs") - } - defer os.Rename(upperRootfs, upperNamedRootfs) - - dir, err := os.Getwd() - if err != nil { - return errs.WithEF(err, b.fields, "Failed to get current working directory") - } - defer func() { - if err := os.Chdir(dir); err != nil { - logs.WithEF(err, b.fields.WithField("path", dir)).Warn("Failed to chdir back") + rootfsAlias := manifestApp(b.pod).Name.String() + destination := b.aciTargetPath + common.PathImageAci // absolute dir, outside upperPath (think: /tmp/…) + + params := []string{"--sort=name", "--numeric-owner", "--exclude", rootfsAlias + PATH_TMP + "/*"} + params = append(params, "-C", upperPath, "--transform", "s@^"+rootfsAlias+"@rootfs@") + params = append(params, "-cf", destination, common.PathManifest[1:], rootfsAlias) + + logs.WithF(b.fields).Debug("Calling tar to collect all files") + if err := common.ExecCmd("tar", params...); err != nil { + // In case the host's tar is too old, try the builder's if it exists. + stage1Tar := rktcommon.Stage1RootfsPath(b.pod.Root) + var buildersTar string + for _, t := range []string{"/dgr/usr/bin/tar", "/bin/tar", "/usr/bin/tar"} { + buildersTar = filepath.Join(stage1Tar, t) + if _, err := os.Stat(buildersTar); err == nil { + break + } } - }() - if err := os.Chdir(upperPath); err != nil { - return errs.WithEF(err, b.fields.WithField("path", upperPath), "Failed to chdir to upper base path") - } - if err := common.Tar(b.aciTargetPath+common.PathImageAci, common.PathManifest[1:], common.PathRootfs[1:]+"/"); err != nil { - return errs.WithEF(err, b.fields, "Failed to tar aci") + if err2 := common.ExecCmd(buildersTar, params...); err2 != nil { + // If that failed, output the original error nevertheless. + logs.WithFields(b.fields).WithField("params", params).Debug("Parameters to 'tar' within the builder") + return errs.WithEF(err, b.fields, "Failed to tar aci") + } + logs.WithF(b.fields).Debug("Had to resort to 'tar' from the builder to create the aci file") } // common.ExecCmd sometimes silently fails, hence the redundant check. - if _, err := os.Stat(b.aciTargetPath + common.PathImageAci); os.IsNotExist(err) { + if _, err := os.Stat(destination); os.IsNotExist(err) { return errs.WithEF(err, b.fields, "Expected aci has not been created") } - logs.WithField("path", dir).Debug("chdir") return nil } diff --git a/dgr/common/tar.go b/dgr/common/tar.go deleted file mode 100644 index 1da0af8a..00000000 --- a/dgr/common/tar.go +++ /dev/null @@ -1,11 +0,0 @@ -package common - -func Tar(destination string, source ...string) error { // remove zip - source = append(source, "") - source = append(source, "") - copy(source[2:], source[0:]) - source[0] = "cpf" - source[1] = destination - - return ExecCmd("tar", source...) -}