Rework packages caching mechanism
Packages are now built by kokoro and pulled in to a local build
unless FETCH_PACKAGES=false . All packages are copied into
rootfs.img in the form of a local debian repo so a user can install
e.g. -dev packages later.
To build packages locally:
rm -rf cache/packages.tgz out
FETCH_PACKAGES=false m
Note:
In this change FETCH_PACKAGES actually defaults to false. This will be
changed when the first kokoro artifact is available.
PACKAGES_FETCH_ROOT_DIRECTORY is primed with a packages.tgz for testing
purposes.
Change-Id: I626587fda2b963183066c2a81223b047afa514eb
diff --git a/Makefile b/Makefile
index ac4170e..fff7a6b 100644
--- a/Makefile
+++ b/Makefile
@@ -10,8 +10,10 @@
+make -f $(ROOTDIR)/build/Makefile validate-bootstrap-tarball
+make -f $(ROOTDIR)/build/Makefile all
-$(PRODUCT_OUT):
- mkdir -p $(PRODUCT_OUT)
+out-dirs:
+ mkdir -p $(PRODUCT_OUT)/packages
+ mkdir -p $(PRODUCT_OUT)/obj
+ mkdir -p $(ROOTDIR)/cache
all: boot-targets
diff --git a/docker.mk b/docker.mk
index a7a1b19..433fc25 100644
--- a/docker.mk
+++ b/docker.mk
@@ -46,7 +46,7 @@
-v $(PREBUILT_DOCKER_ROOT)\:/docker \
-v $(PREBUILT_MODULES_ROOT)\:/modules \
-v $(FETCH_PBUILDER_DIRECTORY)\:/pbuilder \
- -v /var/cache/pbuilder\:/var/cache/pbuilder \
+ -v $(PACKAGES_FETCH_ROOT_DIRECTORY)\:/packages \
-w /rootdir \
-e "DEBOOTSTRAP_FETCH_TARBALL=$(DEBOOTSTRAP_FETCH_TARBALL)" \
-e "ROOTFS_FETCH_TARBALL=$(ROOTFS_FETCH_TARBALL)" \
@@ -58,6 +58,7 @@
-e "DEBOOTSTRAP_TARBALL_REVISION=$(DEBOOTSTRAP_TARBALL_REVISION)" \
-e "PREBUILT_MODULES_ROOT=/modules" \
-e "FETCH_PBUILDER_DIRECTORY=/pbuilder" \
+ -e "PACKAGES_FETCH_ROOT_DIRECTORY=/packages" \
aiy-board-builder \
/bin/bash -c \
'groupadd --gid $(shell id -g) $(shell id -g -n); \
@@ -81,6 +82,7 @@
$(call docker-run,bootstrap,make-bootstrap-tarball)
$(call docker-run,rootfs,rootfs_raw)
+$(call docker-run,rootfs-final,rootfs)
$(call docker-run,boot,boot)
$(call docker-run,all,boot-targets)
$(call docker-run,sdcard,sdcard)
diff --git a/kernel.mk b/kernel.mk
index 7528c57..d94bea4 100644
--- a/kernel.mk
+++ b/kernel.mk
@@ -15,15 +15,15 @@
$(KERNEL_OUT_DIR)/arch/arm64/boot/Image: $(KERNEL_OUT_DIR)/.config
+make -C $(KERNEL_SRC_DIR) O=$(KERNEL_OUT_DIR) $(KERNEL_OPTIONS) Image modules dtbs
-kernel-deb: $(PRODUCT_OUT)/linux-image-4.9.51-aiy_1_arm64.deb
+kernel-deb: $(PRODUCT_OUT)/packages/linux-image-4.9.51-aiy_1_arm64.deb
kernel: $(KERNEL_OUT_DIR)/arch/arm64/boot/Image $(KERNEL_OUT_DIR)/arch/arm64/boot/dts/freescale/fsl-imx8mq-phanbell.dtb
-$(PRODUCT_OUT)/linux-image-4.9.51-aiy_1_arm64.deb: $(KERNEL_OUT_DIR)/.config
+$(PRODUCT_OUT)/packages/linux-image-4.9.51-aiy_1_arm64.deb: $(KERNEL_OUT_DIR)/.config | out-dirs
+make -C $(KERNEL_SRC_DIR) O=$(KERNEL_OUT_DIR) $(KERNEL_OPTIONS) \
KDEB_PKGVERSION=1 KBUILD_IMAGE=Image deb-pkg
- mv $(KERNEL_OUT_DIR)/../linux-image-4.9.51-aiy_1_arm64.deb $(PRODUCT_OUT)
- mv $(KERNEL_OUT_DIR)/../linux-headers-4.9.51-aiy_1_arm64.deb $(PRODUCT_OUT)
+ mv $(KERNEL_OUT_DIR)/../linux-image-4.9.51-aiy_1_arm64.deb $(PRODUCT_OUT)/packages
+ mv $(KERNEL_OUT_DIR)/../linux-headers-4.9.51-aiy_1_arm64.deb $(PRODUCT_OUT)/packages
targets::
@echo "kernel - builds the kernel and boot partition"
diff --git a/kokoro/continuous.cfg b/kokoro/continuous.cfg
index 8b398b5..877b2d4 100644
--- a/kokoro/continuous.cfg
+++ b/kokoro/continuous.cfg
@@ -16,5 +16,6 @@
regex: "partition-table-*.img"
regex: "rootfs.img"
regex: "u-boot.imx"
+ regex: "packages.tgz"
}
}
diff --git a/kokoro/continuous.sh b/kokoro/continuous.sh
index 9fa5bbe..f14533d 100644
--- a/kokoro/continuous.sh
+++ b/kokoro/continuous.sh
@@ -18,6 +18,7 @@
export FETCH_PBUILDER_DIRECTORY=${KOKORO_GFILE_DIR}
export DEBOOTSTRAP_TARBALL_REVISION=.
export ROOTFS_REVISION=.
+export FETCH_PACKAGES=false
m docker-all
m docker-sdcard
@@ -35,3 +36,4 @@
cp ${PRODUCT_OUT}/partition-table-*.img ${KOKORO_ARTIFACTS_DIR}
cp ${PRODUCT_OUT}/rootfs.img ${KOKORO_ARTIFACTS_DIR}
cp ${PRODUCT_OUT}/sdcard.img ${KOKORO_ARTIFACTS_DIR}
+cp ${ROOTDIR}/cache/packages.tgz $KOKORO_ARTIFACTS_DIR
diff --git a/kokoro/release.sh b/kokoro/release.sh
index 11fb227..581ab66 100644
--- a/kokoro/release.sh
+++ b/kokoro/release.sh
@@ -17,6 +17,7 @@
export FETCH_PBUILDER_DIRECTORY=${KOKORO_GFILE_DIR}
export DEBOOTSTRAP_TARBALL_REVISION=.
export ROOTFS_REVISION=.
+export FETCH_PACKAGES=false
m docker-make-repo
diff --git a/kokoro/rootfs.sh b/kokoro/rootfs.sh
index 5d4936a..d215363 100644
--- a/kokoro/rootfs.sh
+++ b/kokoro/rootfs.sh
@@ -9,6 +9,7 @@
export DEBOOTSTRAP_FETCH_TARBALL=false
export ROOTFS_FETCH_TARBALL=false
+export FETCH_PACKAGES=false
export PREBUILT_DOCKER_ROOT=$KOKORO_GFILE_DIR
m docker-bootstrap
diff --git a/packages.mk b/packages.mk
index b198da3..9aea15e 100644
--- a/packages.mk
+++ b/packages.mk
@@ -11,14 +11,13 @@
BUILDPACKAGE_CMD := dpkg-buildpackage -b -rfakeroot -us -uc -tc
define make-equivs-package-target
-$(PRODUCT_OUT)/.$1: $(ROOTDIR)/packages/equivs/$1
- cd $(PRODUCT_OUT); equivs-build $$<
+$(PRODUCT_OUT)/.$1: $(ROOTDIR)/packages/equivs/$1 | out-dirs
+ cd $(PRODUCT_OUT)/packages; equivs-build $$<
touch $$@
endef
ifeq ($(FETCH_PBUILDER_BASE),true)
-$(ROOTDIR)/cache/base.tgz: $(FETCH_PBUILDER_DIRECTORY)/base.tgz
- mkdir -p $(ROOTDIR)/cache
+$(ROOTDIR)/cache/base.tgz: $(FETCH_PBUILDER_DIRECTORY)/base.tgz | out-dirs
cp $< $(ROOTDIR)/cache
else
$(ROOTDIR)/cache/base.tgz: /usr/bin/qemu-aarch64-static
@@ -53,12 +52,12 @@
# $4: space separated list of external dependencies (may be empty)
define make-pbuilder-package-target
$1: $(PRODUCT_OUT)/.$1-pbuilder
-PBUILDER_TARGETS += $1
+PBUILDER_TARGETS += $(PRODUCT_OUT)/.$1-pbuilder
$(PRODUCT_OUT)/.$1-pbuilder: \
$(foreach package,$3,$(PRODUCT_OUT)/.$(package)-pbuilder) \
$(shell find $(ROOTDIR)/packages/$1 -type f) \
$(shell find $(ROOTDIR)/$2 -type f | sed -e 's/ /\\ /g') \
- | $(ROOTDIR)/cache/base.tgz \
+ | out-dirs $(ROOTDIR)/cache/base.tgz \
$4
cd $(ROOTDIR)/$2; git submodule init; git submodule update;
@@ -73,7 +72,7 @@
debian
cd $(PRODUCT_OUT)/obj/$1; pdebuild \
- --buildresult $(PRODUCT_OUT) -- \
+ --buildresult $(PRODUCT_OUT)/packages -- \
--basetgz $(ROOTDIR)/cache/base.tgz \
--configfile $(ROOTDIR)/build/pbuilderrc \
--hookdir $(ROOTDIR)/build/pbuilder-hooks \
@@ -111,6 +110,18 @@
$(eval $(call make-pbuilder-package-target,aiy-board-tools,packages/aiy-board-tools))
$(eval $(call make-pbuilder-package-target,aiy-board-wlan,packages/aiy-board-wlan))
-packages:: $(foreach package,$(ALL_PACKAGE_NAMES),$(PRODUCT_OUT)/.$(package)) $(PBUILDER_TARGETS)
+ALL_PACKAGE_TARGETS := $(foreach package,$(ALL_PACKAGE_NAMES),$(PRODUCT_OUT)/.$(package)) $(PBUILDER_TARGETS)
+$(warning ALL_PACKAGE_TARGETS $(ALL_PACKAGE_TARGETS))
+packages-tarball: $(ROOTDIR)/cache/packages.tgz
+ifeq ($(FETCH_PACKAGES),true)
+$(ROOTDIR)/cache/packages.tgz: $(PACKAGES_FETCH_ROOT_DIRECTORY)/packages.tgz | out-dirs
+ cp $< $(ROOTDIR)/cache
+else
+$(ROOTDIR)/cache/packages.tgz: $(ALL_PACKAGE_TARGETS) | out-dirs
+ $(ROOTDIR)/build/update_packages.sh
+ tar -C $(PRODUCT_OUT) -czf $@ packages
+endif
+
+packages:: $(ALL_PACKAGE_TARGETS)
.PHONY:: packages
diff --git a/pbuilder-hooks/D05deps b/pbuilder-hooks/D05deps
index f8feb1b..70833e4 100755
--- a/pbuilder-hooks/D05deps
+++ b/pbuilder-hooks/D05deps
@@ -1,8 +1,3 @@
#!/bin/bash -xe
-apt-get install -y apt-utils
-(
- flock -e 200
- ( cd "$DEPSBASE"; apt-ftparchive packages . > Packages )
- echo "deb [trusted=yes] file://$DEPSBASE ./" >> /etc/apt/sources.list
- apt-get update
-) 200>/tmp/pbuilder-apt-lock
+echo "deb [trusted=yes] file://$DEPSBASE ./" >> /etc/apt/sources.list
+apt-get update
diff --git a/pbuilderrc b/pbuilderrc
index 1709fd9..847a975 100644
--- a/pbuilderrc
+++ b/pbuilderrc
@@ -6,7 +6,7 @@
# read pbuilderrc.5 document for notes on specific options.
MIRRORSITE=http://http.us.debian.org/debian
PBUILDERSATISFYDEPENDSCMD=/usr/lib/pbuilder/pbuilder-satisfydepends-apt
-export DEPSBASE=$PRODUCT_OUT
+export DEPSBASE=$PRODUCT_OUT/packages
BINDMOUNTS="$DEPSBASE $ROOTDIR"
APTCACHE=$ROOTDIR/cache/pbuilder_apt
APTCACHEHARDLINK=no
diff --git a/preamble.mk b/preamble.mk
index 4f4f12f..171b0e3 100644
--- a/preamble.mk
+++ b/preamble.mk
@@ -118,4 +118,5 @@
PREBUILT_DOCKER_ROOT ?= /google/data/ro/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/docker
-DEBCACHE_ROOT ?= /google/data/rw/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/debcache
+FETCH_PACKAGES ?= false
+PACKAGES_FETCH_ROOT_DIRECTORY ?= /google/data/ro/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/debcache
diff --git a/prereqs.mk b/prereqs.mk
index 492a79e..7dc16b1 100644
--- a/prereqs.mk
+++ b/prereqs.mk
@@ -5,6 +5,7 @@
include $(ROOTDIR)/build/preamble.mk
REQUIRED_PACKAGES := \
+ apt-utils \
bc \
binutils-aarch64-linux-gnu \
build-essential \
diff --git a/rootfs.mk b/rootfs.mk
index 8cbbae8..867c68a 100644
--- a/rootfs.mk
+++ b/rootfs.mk
@@ -11,8 +11,6 @@
ROOTFS_FETCH_TARBALL ?= true
ROOTFS_REVISION ?= latest
-ROOTFS_PUSH_DEBS ?= false
-
USER_GROUPS := \
adm \
audio \
@@ -26,6 +24,44 @@
users \
video
+PRE_INSTALL_PACKAGES := \
+ aiy-board-audio \
+ aiy-board-gadget \
+ aiy-board-keyring \
+ aiy-board-tools \
+ aiy-board-wlan \
+ gstreamer1.0-alsa \
+ gstreamer1.0-plugins-bad \
+ gstreamer1.0-plugins-base \
+ gstreamer1.0-plugins-base-apps \
+ gstreamer1.0-plugins-good \
+ gstreamer1.0-tools \
+ imx-atf \
+ imx-firmware \
+ imx-gpu-viv \
+ imx-gst1.0-plugin \
+ imx-mkimage \
+ imx-vpu-hantro \
+ imx-vpuwrap \
+ libdrm2 \
+ libdrm-vivante \
+ libegl1-mesa \
+ libegl1-mesa-dev \
+ libgbm1 \
+ libgbm-dev \
+ libgl1-mesa-glx \
+ libgles1-mesa \
+ libgles2-mesa \
+ libgstreamer1.0-0 \
+ libgstreamer-plugins-bad1.0-0 \
+ libgstreamer-plugins-base1.0-0 \
+ linux-headers-4.9.51-aiy \
+ linux-image-4.9.51-aiy \
+ mesa-common-dev \
+ uboot-imx \
+ wayland-protocols-imx \
+ weston-imx \
+
rootfs: $(PRODUCT_OUT)/rootfs.img
rootfs_raw: $(ROOTFS_RAW_IMG)
@@ -86,10 +122,10 @@
$(ROOTFS_PATCHED_IMG): $(ROOTFS_RAW_IMG) \
$(ROOTDIR)/board/fstab.emmc \
$(ROOTDIR)/build/boot.mk \
- $(PRODUCT_OUT)/linux-image-4.9.51-aiy_1_arm64.deb \
+ kernel-deb \
+ $(ROOTDIR)/cache/packages.tgz \
| $(PRODUCT_OUT)/boot.img \
- modules \
- packages
+ modules
cp -r $(ROOTFS_RAW_IMG) $(ROOTFS_PATCHED_IMG)
mkdir -p $(ROOTFS_DIR)
-sudo umount $(ROOTFS_DIR)/boot
@@ -102,10 +138,10 @@
sudo cp $(ROOTDIR)/board/fstab.emmc $(ROOTFS_DIR)/etc/fstab
- sudo mount -t tmpfs none $(ROOTFS_DIR)/tmp
- sudo rsync -avm --exclude="*-dbgsym_*.deb" --exclude="*-dev_*.deb" --include="*.deb" --exclude="*" $(PRODUCT_OUT)/ $(ROOTFS_DIR)/tmp/
- sudo chroot $(ROOTFS_DIR) bash -c 'apt-get install --allow-downgrades --no-install-recommends -y /tmp/*.deb'
- sudo umount $(ROOTFS_DIR)/tmp
+ sudo sed -i '1 i\deb [trusted=yes] file:///opt/aiy/packages ./' $(ROOTFS_DIR)/etc/apt/sources.list
+ sudo mkdir -p $(ROOTFS_DIR)/opt/aiy
+ sudo tar -xvf $(ROOTDIR)/cache/packages.tgz -C $(ROOTFS_DIR)/opt/aiy/
+ sudo chroot $(ROOTFS_DIR) bash -c 'apt-get update && apt-get install --allow-downgrades --no-install-recommends -y $(PRE_INSTALL_PACKAGES)'
sudo umount $(ROOTFS_DIR)/boot
sudo umount $(ROOTFS_DIR)
@@ -116,21 +152,6 @@
$(PRODUCT_OUT)/rootfs.img: $(HOST_OUT)/bin/img2simg $(ROOTFS_PATCHED_IMG)
$(HOST_OUT)/bin/img2simg $(ROOTFS_PATCHED_IMG) $(PRODUCT_OUT)/rootfs.img
-fetch_debs:
- $(info Fetching debs from cache...)
- mkdir -p $(PRODUCT_OUT)
- rsync -rv $(DEBCACHE_ROOT)/ $(PRODUCT_OUT)/
- find $(PRODUCT_OUT) -name *.deb | xargs touch -d "-1337 days ago"
-
-ifeq ($(ROOTFS_PUSH_DEBS),true)
-push_debs:
- $(info Pushing debs to cache...)
- rsync -rvm --exclude="aiy*.deb" --include="*.deb" --include="*/" --exclude="*" $(PRODUCT_OUT)/ $(DEBCACHE_ROOT)/
-else
-push_debs:
- $(error Pushing debs to cache disabled)
-endif
-
clean::
if mount |grep -q $(ROOTFS_DIR); then sudo umount -R $(ROOTFS_DIR); fi
if [[ -d $(ROOTFS_DIR) ]]; then rmdir $(ROOTFS_DIR); fi
diff --git a/signing.mk b/signing.mk
index 0d68ce5..0be4cbc 100644
--- a/signing.mk
+++ b/signing.mk
@@ -5,15 +5,16 @@
include $(ROOTDIR)/build/preamble.mk
make-repo: $(PRODUCT_OUT)/repo/debian_repo/dists/stable/Release
-$(PRODUCT_OUT)/repo/debian_repo/dists/stable/Release: $(ROOTDIR)/build/distributions | kernel-deb modules packages
+$(PRODUCT_OUT)/repo/debian_repo/dists/stable/Release: $(ROOTDIR)/build/distributions | kernel-deb modules packages-tarball
mkdir -p $(PRODUCT_OUT)/repo
mkdir -p $(PRODUCT_OUT)/repo/debian_repo
mkdir -p $(PRODUCT_OUT)/repo/deb_repo_config
+ tar -xvf $(ROOTDIR)/cache/packages.tgz -C $(PRODUCT_OUT)/repo
cp $(ROOTDIR)/build/distributions $(PRODUCT_OUT)/repo/deb_repo_config/
reprepro --basedir $(PRODUCT_OUT)/repo \
--outdir $(PRODUCT_OUT)/repo/debian_repo \
--confdir $(PRODUCT_OUT)/repo/deb_repo_config \
- includedeb stable $(PRODUCT_OUT)/*.deb
+ includedeb stable $(PRODUCT_OUT)/repo/packages/*.deb
find $(PRODUCT_OUT)/repo/debian_repo -type d | xargs chmod 777
find $(PRODUCT_OUT)/repo/debian_repo -type f | xargs chmod 666
diff --git a/update_packages.sh b/update_packages.sh
new file mode 100755
index 0000000..fb1988f
--- /dev/null
+++ b/update_packages.sh
@@ -0,0 +1,5 @@
+#!/bin/bash -xe
+(
+ flock -e 200
+ ( cd $PRODUCT_OUT/packages; apt-ftparchive packages . > Packages )
+) 200>$PRODUCT_OUT/obj/packages-lock