New rootfs built from clout apt packages
- Remove debootstrap entirely, use multistrap instead as it supports
packages from more than one source
- All multistrapping packages come from cloud APT, and sources.list in
the final image point to cloud.
- The package install/update step at the end is slightly different, as
keys need to be installed first, due to all packages from cloud being
signed
Change-Id: I18188ff557f1043e8786dd4bb9a9439dc8d42472
diff --git a/Makefile b/Makefile
index 4e0ed5f..da8a7da 100644
--- a/Makefile
+++ b/Makefile
@@ -33,7 +33,6 @@
@echo
include $(ROOTDIR)/build/boot.mk
-include $(ROOTDIR)/build/debootstrap.mk
include $(ROOTDIR)/build/img2simg.mk
include $(ROOTDIR)/build/kernel.mk
include $(ROOTDIR)/build/kernel-modules.mk
diff --git a/debootstrap.mk b/debootstrap.mk
deleted file mode 100644
index 3d9b3d7..0000000
--- a/debootstrap.mk
+++ /dev/null
@@ -1,44 +0,0 @@
-ifeq ($(ROOTDIR),)
-$(error $$ROOTDIR IS NOT DEFINED -- don\'t forget to source setup.sh)
-endif
-
-include $(ROOTDIR)/build/preamble.mk
-
-DEBOOTSTRAP_TARBALL_REVISION ?= latest
-DEBOOTSTRAP_FETCH_TARBALL ?= $(IS_GLINUX)
-
-validate-bootstrap-tarball: $(DEBOOTSTRAP_TARBALL)
- cd $(ROOTDIR)/cache && \
- sha256sum -c $(DEBOOTSTRAP_TARBALL_SHA256)
-
-ifeq ($(DEBOOTSTRAP_FETCH_TARBALL),true)
-$(DEBOOTSTRAP_TARBALL): $(TARBALL_FETCH_ROOT_DIRECTORY)/$(DEBOOTSTRAP_TARBALL_REVISION)/debootstrap.tgz
- mkdir -p $(ROOTDIR)/cache
- cp $< $<.sha256sum $(ROOTDIR)/cache
-else
-$(DEBOOTSTRAP_TARBALL): $(ROOTDIR)/build/debootstrap.mk $(ROOTDIR)/build/preamble.mk
- mkdir -p $(PRODUCT_OUT)/obj/DEBOOTSTRAP
- mkdir -p $(ROOTDIR)/cache
- /usr/sbin/debootstrap \
- --foreign \
- $(DEBOOTSTRAP_ARGS) \
- --make-tarball=$(DEBOOTSTRAP_TARBALL) \
- stretch $(PRODUCT_OUT)/obj/DEBOOTSTRAP
- +make -f $(ROOTDIR)/build/debootstrap.mk bootstrap-sha256sum
-endif
-
-bootstrap-tarball: $(DEBOOTSTRAP_TARBALL)
-
-bootstrap-sha256sum: $(DEBOOTSTRAP_TARBALL)
- cd $(ROOTDIR)/cache && \
- sha256sum $(notdir $(DEBOOTSTRAP_TARBALL)) > $(DEBOOTSTRAP_TARBALL_SHA256)
-
-targets::
- @echo "validate-bootstrap-tarball - validates the bootstrap tarball matches the SHA-256 sums"
- @echo "bootstrap-tarball - makes the debootstrap tarball for faster rootfs building"
- @echo "bootstrap-sha256sum - calculates the SHA-256 sums of the debootstrap tarball"
-
-clean::
- sudo rm -rf $(PRODUCT_OUT)/obj/DEBOOTSTRAP
-
-.PHONY:: validate-bootstrap-tarball bootstrap-sha256sum bootstrap-tarball
diff --git a/docker.mk b/docker.mk
index d434e78..3985c00 100644
--- a/docker.mk
+++ b/docker.mk
@@ -48,14 +48,12 @@
-v $(FETCH_PBUILDER_DIRECTORY)\:/pbuilder \
-v $(PACKAGES_FETCH_ROOT_DIRECTORY)\:/packages \
-w /rootdir \
- -e "DEBOOTSTRAP_FETCH_TARBALL=$(DEBOOTSTRAP_FETCH_TARBALL)" \
-e "ROOTFS_FETCH_TARBALL=$(ROOTFS_FETCH_TARBALL)" \
-e "ARM64_BUILDER_FETCH_TARBALL=$(ARM64_BUILDER_FETCH_TARBALL)" \
-e "FETCH_PBUILDER_BASE=$(FETCH_PBUILDER_BASE)" \
-e "TARBALL_FETCH_ROOT_DIRECTORY=/tarballs" \
-e "PREBUILT_DOCKER_ROOT=/docker" \
-e "ROOTFS_REVISION=$(ROOTFS_REVISION)" \
- -e "DEBOOTSTRAP_TARBALL_REVISION=$(DEBOOTSTRAP_TARBALL_REVISION)" \
-e "PREBUILT_MODULES_ROOT=/modules" \
-e "FETCH_PBUILDER_DIRECTORY=/pbuilder" \
-e "PACKAGES_FETCH_ROOT_DIRECTORY=/packages" \
@@ -83,13 +81,11 @@
-v $(PREBUILT_DOCKER_ROOT)\:/docker \
-v $(PREBUILT_MODULES_ROOT)\:/modules \
-w /build \
- -e "DEBOOTSTRAP_FETCH_TARBALL=$(DEBOOTSTRAP_FETCH_TARBALL)" \
-e "ROOTFS_FETCH_TARBALL=$(ROOTFS_FETCH_TARBALL)" \
-e "ARM64_BUILDER_FETCH_TARBALL=$(ARM64_BUILDER_FETCH_TARBALL)" \
-e "TARBALL_FETCH_ROOT_DIRECTORY=/tarballs" \
-e "PREBUILT_DOCKER_ROOT=/docker" \
-e "ROOTFS_REVISION=$(ROOTFS_REVISION)" \
- -e "DEBOOTSTRAP_TARBALL_REVISION=$(DEBOOTSTRAP_TARBALL_REVISION)" \
-e "PREBUILT_MODULES_ROOT=/modules" \
arm64-builder \
/bin/bash -c \
diff --git a/kokoro/continuous.cfg b/kokoro/continuous.cfg
index 6aca927..5655eb7 100644
--- a/kokoro/continuous.cfg
+++ b/kokoro/continuous.cfg
@@ -1,6 +1,4 @@
build_file: "continuous-build/build/kokoro/continuous.sh"
-gfile_resources: "/x20/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/rootfs/latest/debootstrap.tgz"
-gfile_resources: "/x20/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/rootfs/latest/debootstrap.tgz.sha256sum"
gfile_resources: "/x20/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/rootfs/latest/rootfs.raw.img"
gfile_resources: "/x20/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/rootfs/latest/rootfs.raw.img.sha256sum"
gfile_resources: "/x20/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/debs/apex.deb"
diff --git a/kokoro/continuous.sh b/kokoro/continuous.sh
index 1d7340d..ecc1e45 100644
--- a/kokoro/continuous.sh
+++ b/kokoro/continuous.sh
@@ -18,7 +18,6 @@
export FETCH_PBUILDER_DIRECTORY=${KOKORO_GFILE_DIR}
export PACKAGES_FETCH_ROOT_DIRECTORY=${KOKORO_GFILE_DIR}
export PACKAGES_REVISION=.
-export DEBOOTSTRAP_TARBALL_REVISION=.
export ROOTFS_REVISION=.
export FETCH_PACKAGES=false
diff --git a/kokoro/release.cfg b/kokoro/release.cfg
index bf2ba30..15518ce 100644
--- a/kokoro/release.cfg
+++ b/kokoro/release.cfg
@@ -1,6 +1,4 @@
build_file: "continuous-build/build/kokoro/release.sh"
-gfile_resources: "/x20/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/rootfs/latest/debootstrap.tgz"
-gfile_resources: "/x20/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/rootfs/latest/debootstrap.tgz.sha256sum"
gfile_resources: "/x20/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/rootfs/latest/rootfs.raw.img"
gfile_resources: "/x20/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/rootfs/latest/rootfs.raw.img.sha256sum"
gfile_resources: "/x20/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/debs/apex.deb"
diff --git a/kokoro/release.sh b/kokoro/release.sh
index d8f26b3..8a78217 100644
--- a/kokoro/release.sh
+++ b/kokoro/release.sh
@@ -17,7 +17,6 @@
export FETCH_PBUILDER_DIRECTORY=${KOKORO_GFILE_DIR}
export PACKAGES_FETCH_ROOT_DIRECTORY=${KOKORO_GFILE_DIR}
export PACKAGES_REVISION=.
-export DEBOOTSTRAP_TARBALL_REVISION=.
export ROOTFS_REVISION=.
export FETCH_PACKAGES=false
diff --git a/kokoro/rootfs.cfg b/kokoro/rootfs.cfg
index f285cd0..482a906 100644
--- a/kokoro/rootfs.cfg
+++ b/kokoro/rootfs.cfg
@@ -4,8 +4,6 @@
action {
define_artifacts {
- regex: "debootstrap.tgz"
- regex: "debootstrap.tgz.sha256sum"
regex: "rootfs.raw.img"
regex: "rootfs.raw.img.sha256sum"
}
diff --git a/kokoro/rootfs.sh b/kokoro/rootfs.sh
index 459b656..745927f 100644
--- a/kokoro/rootfs.sh
+++ b/kokoro/rootfs.sh
@@ -7,15 +7,11 @@
source build/setup.sh
popd
-export DEBOOTSTRAP_FETCH_TARBALL=false
export ROOTFS_FETCH_TARBALL=false
export FETCH_PACKAGES=false
export PREBUILT_DOCKER_ROOT=$KOKORO_GFILE_DIR
-m docker-bootstrap-tarball
m docker-rootfs_raw
-cp git/cache/debootstrap.tgz $KOKORO_ARTIFACTS_DIR
-cp git/cache/debootstrap.tgz.sha256sum $KOKORO_ARTIFACTS_DIR
cp git/out/target/product/imx8m_phanbell/obj/ROOTFS/rootfs.raw.img $KOKORO_ARTIFACTS_DIR
cp git/out/target/product/imx8m_phanbell/obj/ROOTFS/rootfs.raw.img.sha256sum $KOKORO_ARTIFACTS_DIR
diff --git a/multistrap.conf b/multistrap.conf
new file mode 100644
index 0000000..e39c8e3
--- /dev/null
+++ b/multistrap.conf
@@ -0,0 +1,22 @@
+[General]
+arch=arm64
+cleanup=true
+unpack=true
+bootstrap=main non-free contrib
+aptsources=main non-free contrib
+noauth=true
+
+[main]
+packages=MAIN_PACKAGES
+source=https://packages.cloud.google.com/apt
+suite=aiy-debian-stretch-main
+
+[non-free]
+packages=
+source=https://packages.cloud.google.com/apt
+suite=aiy-debian-stretch-non-free
+
+[contrib]
+packages=
+source=https://packages.cloud.google.com/apt
+suite=aiy-debian-stretch-contrib
diff --git a/preamble.mk b/preamble.mk
index e3469e9..3f6812a 100644
--- a/preamble.mk
+++ b/preamble.mk
@@ -14,15 +14,12 @@
KERNEL_OUT_DIR := $(PRODUCT_OUT)/obj/KERNEL_OBJ
KERNEL_OPTIONS := ARCH=arm64 CROSS_COMPILE=$(TOOLCHAIN) LOCALVERSION=-aiy
-# Used by debootstrap and rootfs both
-DEBOOTSTRAP_TARBALL := $(ROOTDIR)/cache/debootstrap.tgz
-DEBOOTSTRAP_TARBALL_SHA256 := $(ROOTDIR)/cache/debootstrap.tgz.sha256sum
-
FETCH_PBUILDER_DIRECTORY ?= /google/data/ro/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/pbuilder
FETCH_PBUILDER_BASE ?= $(IS_GLINUX)
-DEBOOTSTRAP_EXTRA := \
+PACKAGES_EXTRA := \
alsa-utils \
+ apt-transport-https \
apt-listchanges \
apt-utils \
aptitude \
@@ -118,14 +115,6 @@
xdg-user-dirs \
xwayland
-DEBOOTSTRAP_ARGS := \
- --arch=arm64 \
- --keyring /usr/share/keyrings/debian-archive-keyring.gpg \
- --variant=buildd \
- --components=main,non-free \
- --exclude=debfoster \
- --include=$$(echo $(DEBOOTSTRAP_EXTRA) |tr ' ' ',') \
-
TARBALL_FETCH_ROOT_DIRECTORY ?= \
/google/data/ro/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/rootfs
diff --git a/prereqs.mk b/prereqs.mk
index 0e498a3..a619f1a 100644
--- a/prereqs.mk
+++ b/prereqs.mk
@@ -14,7 +14,6 @@
coreutils \
debhelper \
debian-archive-keyring \
- debootstrap \
device-tree-compiler \
fakeroot \
genext2fs \
@@ -24,6 +23,7 @@
libcap-dev \
libwayland-dev \
mtools \
+ multistrap \
parted \
pbuilder \
pkg-config \
diff --git a/rootfs.mk b/rootfs.mk
index 48ba58b..98ed474 100644
--- a/rootfs.mk
+++ b/rootfs.mk
@@ -57,19 +57,23 @@
mkdir -p $(dir $(ROOTFS_RAW_IMG))
cp $< $<.sha256sum $(dir $(ROOTFS_RAW_IMG))
else
-$(ROOTFS_RAW_IMG): $(ROOTDIR)/build/debootstrap.mk $(ROOTDIR)/build/preamble.mk $(ROOTDIR)/build/rootfs.mk $(DEBOOTSTRAP_TARBALL)
- +make -f $(ROOTDIR)/build/debootstrap.mk validate-bootstrap-tarball
+$(ROOTFS_RAW_IMG): $(ROOTDIR)/build/preamble.mk $(ROOTDIR)/build/rootfs.mk
mkdir -p $(ROOTFS_DIR)
rm -f $(ROOTFS_RAW_IMG)
fallocate -l 4G $(ROOTFS_RAW_IMG)
mkfs.ext4 -F -j $(ROOTFS_RAW_IMG)
tune2fs -o discard $(ROOTFS_RAW_IMG)
+ -sudo umount $(ROOTFS_DIR)/dev
-sudo umount $(ROOTFS_DIR)
sudo mount -o loop $(ROOTFS_RAW_IMG) $(ROOTFS_DIR)
- sudo qemu-debootstrap \
- $(DEBOOTSTRAP_ARGS) \
- --unpack-tarball=$(DEBOOTSTRAP_TARBALL) \
- stretch $(ROOTFS_DIR)
+ cp $(ROOTDIR)/build/multistrap.conf $(PRODUCT_OUT)
+ sed -i -e 's/MAIN_PACKAGES/$(PACKAGES_EXTRA)/g' $(PRODUCT_OUT)/multistrap.conf
+ sudo multistrap -f $(PRODUCT_OUT)/multistrap.conf -d $(ROOTFS_DIR)
+
+ sudo mount -o bind /dev $(ROOTFS_DIR)/dev
+ sudo chroot $(ROOTFS_DIR) /var/lib/dpkg/info/dash.preinst install
+ sudo DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true LC_ALL=C LANGUAGE=C LANG=C chroot $(ROOTFS_DIR) dpkg --configure -a
+ sudo umount $(ROOTFS_DIR)/dev
sudo umount $(ROOTFS_DIR)
sudo rmdir $(ROOTFS_DIR)
sudo sync $(ROOTFS_RAW_IMG)
@@ -80,23 +84,26 @@
$(ROOTFS_PATCHED_IMG): $(ROOTFS_RAW_IMG) \
$(ROOTDIR)/board/fstab.emmc \
$(ROOTDIR)/build/boot.mk \
- kernel-deb \
$(ROOTDIR)/cache/packages.tgz \
- | $(PRODUCT_OUT)/boot.img \
- modules
+ | $(PRODUCT_OUT)/boot.img
cp $(ROOTFS_RAW_IMG) $(ROOTFS_PATCHED_IMG).wip
mkdir -p $(ROOTFS_DIR)
-sudo umount $(ROOTFS_DIR)/boot
-sudo umount $(ROOTFS_DIR)
sudo mount -o loop $(ROOTFS_PATCHED_IMG).wip $(ROOTFS_DIR)
sudo mount -o loop $(PRODUCT_OUT)/boot.img $(ROOTFS_DIR)/boot
+ sudo mount -o bind /dev $(ROOTFS_DIR)/dev
sudo cp $(ROOTDIR)/board/fstab.emmc $(ROOTFS_DIR)/etc/fstab
- sudo sed -i '1 i\deb [trusted=yes] file:///opt/aiy/packages ./' $(ROOTFS_DIR)/etc/apt/sources.list
+ echo 'nameserver 8.8.8.8' | sudo tee $(ROOTFS_DIR)/etc/resolv.conf
+ echo 'deb [trusted=yes] file:///opt/aiy/packages ./' | sudo tee $(ROOTFS_DIR)/etc/apt/sources.list.d/local.list
sudo mkdir -p $(ROOTFS_DIR)/opt/aiy
sudo tar -xvf $(ROOTDIR)/cache/packages.tgz -C $(ROOTFS_DIR)/opt/aiy/
- sudo chroot $(ROOTFS_DIR) bash -c 'apt-get update && apt-get install --allow-downgrades --no-install-recommends -y $(PRE_INSTALL_PACKAGES)'
+ sudo chroot $(ROOTFS_DIR) bash -c 'apt-get update'
+ sudo chroot $(ROOTFS_DIR) bash -c 'apt-get install aiy-board-keyring'
+ sudo chroot $(ROOTFS_DIR) bash -c 'apt-get update'
+ sudo chroot $(ROOTFS_DIR) bash -c 'apt-get install --allow-downgrades --no-install-recommends -y $(PRE_INSTALL_PACKAGES)'
sudo mount -t tmpfs none $(ROOTFS_DIR)/tmp
sudo cp $(PRODUCT_OUT)/packages/linux-headers-*-aiy_*_arm64.deb \
@@ -106,6 +113,7 @@
+make -f $(ROOTDIR)/build/rootfs.mk adjustments
+ sudo umount $(ROOTFS_DIR)/dev
sudo umount $(ROOTFS_DIR)/boot
sudo umount $(ROOTFS_DIR)
sudo rmdir $(ROOTFS_DIR)
@@ -122,6 +130,6 @@
rm -f $(ROOTFS_PATCHED_IMG) $(ROOTFS_RAW_IMG) $(PRODUCT_OUT)/rootfs.img
targets::
- @echo "rootfs - runs debootstrap to build the rootfs tree"
+ @echo "rootfs - runs multistrap to build the rootfs tree"
.PHONY:: rootfs rootfs_raw adjustments fetch_debs push_debs
