Rework packages caching mechanism

Packages are now built by kokoro and pulled in to a local build
unless FETCH_PACKAGES=false . All packages are copied into
rootfs.img in the form of a local debian repo so a user can install
e.g. -dev packages later.

To build packages locally:
rm -rf cache/packages.tgz out
FETCH_PACKAGES=false m

Note:
In this change FETCH_PACKAGES actually defaults to false. This will be
changed when the first kokoro artifact is available.
PACKAGES_FETCH_ROOT_DIRECTORY is primed with a packages.tgz for testing
purposes.

Change-Id: I626587fda2b963183066c2a81223b047afa514eb
diff --git a/Makefile b/Makefile
index ac4170e..fff7a6b 100644
--- a/Makefile
+++ b/Makefile
@@ -10,8 +10,10 @@
 	+make -f $(ROOTDIR)/build/Makefile validate-bootstrap-tarball
 	+make -f $(ROOTDIR)/build/Makefile all
 
-$(PRODUCT_OUT):
-	mkdir -p $(PRODUCT_OUT)
+out-dirs:
+	mkdir -p $(PRODUCT_OUT)/packages
+	mkdir -p $(PRODUCT_OUT)/obj
+	mkdir -p $(ROOTDIR)/cache
 
 all: boot-targets
 
diff --git a/docker.mk b/docker.mk
index a7a1b19..433fc25 100644
--- a/docker.mk
+++ b/docker.mk
@@ -46,7 +46,7 @@
 		-v $(PREBUILT_DOCKER_ROOT)\:/docker \
 		-v $(PREBUILT_MODULES_ROOT)\:/modules \
 		-v $(FETCH_PBUILDER_DIRECTORY)\:/pbuilder \
-		-v /var/cache/pbuilder\:/var/cache/pbuilder \
+		-v $(PACKAGES_FETCH_ROOT_DIRECTORY)\:/packages \
 		-w /rootdir \
 		-e "DEBOOTSTRAP_FETCH_TARBALL=$(DEBOOTSTRAP_FETCH_TARBALL)" \
 		-e "ROOTFS_FETCH_TARBALL=$(ROOTFS_FETCH_TARBALL)" \
@@ -58,6 +58,7 @@
 		-e "DEBOOTSTRAP_TARBALL_REVISION=$(DEBOOTSTRAP_TARBALL_REVISION)" \
 		-e "PREBUILT_MODULES_ROOT=/modules" \
 		-e "FETCH_PBUILDER_DIRECTORY=/pbuilder" \
+		-e "PACKAGES_FETCH_ROOT_DIRECTORY=/packages" \
 		aiy-board-builder \
 		/bin/bash -c \
 			'groupadd --gid $(shell id -g) $(shell id -g -n); \
@@ -81,6 +82,7 @@
 
 $(call docker-run,bootstrap,make-bootstrap-tarball)
 $(call docker-run,rootfs,rootfs_raw)
+$(call docker-run,rootfs-final,rootfs)
 $(call docker-run,boot,boot)
 $(call docker-run,all,boot-targets)
 $(call docker-run,sdcard,sdcard)
diff --git a/kernel.mk b/kernel.mk
index 7528c57..d94bea4 100644
--- a/kernel.mk
+++ b/kernel.mk
@@ -15,15 +15,15 @@
 $(KERNEL_OUT_DIR)/arch/arm64/boot/Image: $(KERNEL_OUT_DIR)/.config
 		+make -C $(KERNEL_SRC_DIR) O=$(KERNEL_OUT_DIR) $(KERNEL_OPTIONS) Image modules dtbs
 
-kernel-deb: $(PRODUCT_OUT)/linux-image-4.9.51-aiy_1_arm64.deb
+kernel-deb: $(PRODUCT_OUT)/packages/linux-image-4.9.51-aiy_1_arm64.deb
 
 kernel: $(KERNEL_OUT_DIR)/arch/arm64/boot/Image $(KERNEL_OUT_DIR)/arch/arm64/boot/dts/freescale/fsl-imx8mq-phanbell.dtb
 
-$(PRODUCT_OUT)/linux-image-4.9.51-aiy_1_arm64.deb: $(KERNEL_OUT_DIR)/.config
+$(PRODUCT_OUT)/packages/linux-image-4.9.51-aiy_1_arm64.deb: $(KERNEL_OUT_DIR)/.config | out-dirs
 	+make -C $(KERNEL_SRC_DIR) O=$(KERNEL_OUT_DIR) $(KERNEL_OPTIONS) \
 		KDEB_PKGVERSION=1 KBUILD_IMAGE=Image deb-pkg
-	mv $(KERNEL_OUT_DIR)/../linux-image-4.9.51-aiy_1_arm64.deb $(PRODUCT_OUT)
-	mv $(KERNEL_OUT_DIR)/../linux-headers-4.9.51-aiy_1_arm64.deb $(PRODUCT_OUT)
+	mv $(KERNEL_OUT_DIR)/../linux-image-4.9.51-aiy_1_arm64.deb $(PRODUCT_OUT)/packages
+	mv $(KERNEL_OUT_DIR)/../linux-headers-4.9.51-aiy_1_arm64.deb $(PRODUCT_OUT)/packages
 
 targets::
 	@echo "kernel - builds the kernel and boot partition"
diff --git a/kokoro/continuous.cfg b/kokoro/continuous.cfg
index 8b398b5..877b2d4 100644
--- a/kokoro/continuous.cfg
+++ b/kokoro/continuous.cfg
@@ -16,5 +16,6 @@
     regex: "partition-table-*.img"
     regex: "rootfs.img"
     regex: "u-boot.imx"
+    regex: "packages.tgz"
   }
 }
diff --git a/kokoro/continuous.sh b/kokoro/continuous.sh
index 9fa5bbe..f14533d 100644
--- a/kokoro/continuous.sh
+++ b/kokoro/continuous.sh
@@ -18,6 +18,7 @@
 export FETCH_PBUILDER_DIRECTORY=${KOKORO_GFILE_DIR}
 export DEBOOTSTRAP_TARBALL_REVISION=.
 export ROOTFS_REVISION=.
+export FETCH_PACKAGES=false
 
 m docker-all
 m docker-sdcard
@@ -35,3 +36,4 @@
 cp ${PRODUCT_OUT}/partition-table-*.img ${KOKORO_ARTIFACTS_DIR}
 cp ${PRODUCT_OUT}/rootfs.img ${KOKORO_ARTIFACTS_DIR}
 cp ${PRODUCT_OUT}/sdcard.img ${KOKORO_ARTIFACTS_DIR}
+cp ${ROOTDIR}/cache/packages.tgz $KOKORO_ARTIFACTS_DIR
diff --git a/kokoro/release.sh b/kokoro/release.sh
index 11fb227..581ab66 100644
--- a/kokoro/release.sh
+++ b/kokoro/release.sh
@@ -17,6 +17,7 @@
 export FETCH_PBUILDER_DIRECTORY=${KOKORO_GFILE_DIR}
 export DEBOOTSTRAP_TARBALL_REVISION=.
 export ROOTFS_REVISION=.
+export FETCH_PACKAGES=false
 
 m docker-make-repo
 
diff --git a/kokoro/rootfs.sh b/kokoro/rootfs.sh
index 5d4936a..d215363 100644
--- a/kokoro/rootfs.sh
+++ b/kokoro/rootfs.sh
@@ -9,6 +9,7 @@
 
 export DEBOOTSTRAP_FETCH_TARBALL=false
 export ROOTFS_FETCH_TARBALL=false
+export FETCH_PACKAGES=false
 export PREBUILT_DOCKER_ROOT=$KOKORO_GFILE_DIR
 
 m docker-bootstrap
diff --git a/packages.mk b/packages.mk
index b198da3..9aea15e 100644
--- a/packages.mk
+++ b/packages.mk
@@ -11,14 +11,13 @@
 BUILDPACKAGE_CMD := dpkg-buildpackage -b -rfakeroot -us -uc -tc
 
 define make-equivs-package-target
-$(PRODUCT_OUT)/.$1: $(ROOTDIR)/packages/equivs/$1
-	cd $(PRODUCT_OUT); equivs-build $$<
+$(PRODUCT_OUT)/.$1: $(ROOTDIR)/packages/equivs/$1 | out-dirs
+	cd $(PRODUCT_OUT)/packages; equivs-build $$<
 	touch $$@
 endef
 
 ifeq ($(FETCH_PBUILDER_BASE),true)
-$(ROOTDIR)/cache/base.tgz: $(FETCH_PBUILDER_DIRECTORY)/base.tgz
-	mkdir -p $(ROOTDIR)/cache
+$(ROOTDIR)/cache/base.tgz: $(FETCH_PBUILDER_DIRECTORY)/base.tgz | out-dirs
 	cp $< $(ROOTDIR)/cache
 else
 $(ROOTDIR)/cache/base.tgz: /usr/bin/qemu-aarch64-static
@@ -53,12 +52,12 @@
 # $4: space separated list of external dependencies (may be empty)
 define make-pbuilder-package-target
 $1: $(PRODUCT_OUT)/.$1-pbuilder
-PBUILDER_TARGETS += $1
+PBUILDER_TARGETS += $(PRODUCT_OUT)/.$1-pbuilder
 $(PRODUCT_OUT)/.$1-pbuilder: \
 	$(foreach package,$3,$(PRODUCT_OUT)/.$(package)-pbuilder) \
 	$(shell find $(ROOTDIR)/packages/$1 -type f) \
 	$(shell find $(ROOTDIR)/$2 -type f | sed -e 's/ /\\ /g') \
-	| $(ROOTDIR)/cache/base.tgz \
+	| out-dirs $(ROOTDIR)/cache/base.tgz \
 	$4
 
 	cd $(ROOTDIR)/$2; git submodule init; git submodule update;
@@ -73,7 +72,7 @@
 		debian
 
 	cd $(PRODUCT_OUT)/obj/$1; pdebuild \
-		--buildresult $(PRODUCT_OUT) -- \
+		--buildresult $(PRODUCT_OUT)/packages -- \
 		--basetgz $(ROOTDIR)/cache/base.tgz \
 		--configfile $(ROOTDIR)/build/pbuilderrc \
 		--hookdir $(ROOTDIR)/build/pbuilder-hooks \
@@ -111,6 +110,18 @@
 $(eval $(call make-pbuilder-package-target,aiy-board-tools,packages/aiy-board-tools))
 $(eval $(call make-pbuilder-package-target,aiy-board-wlan,packages/aiy-board-wlan))
 
-packages:: $(foreach package,$(ALL_PACKAGE_NAMES),$(PRODUCT_OUT)/.$(package)) $(PBUILDER_TARGETS)
+ALL_PACKAGE_TARGETS := $(foreach package,$(ALL_PACKAGE_NAMES),$(PRODUCT_OUT)/.$(package)) $(PBUILDER_TARGETS)
+$(warning ALL_PACKAGE_TARGETS $(ALL_PACKAGE_TARGETS))
+packages-tarball: $(ROOTDIR)/cache/packages.tgz
+ifeq ($(FETCH_PACKAGES),true)
+$(ROOTDIR)/cache/packages.tgz: $(PACKAGES_FETCH_ROOT_DIRECTORY)/packages.tgz | out-dirs
+	cp $< $(ROOTDIR)/cache
+else
+$(ROOTDIR)/cache/packages.tgz: $(ALL_PACKAGE_TARGETS) | out-dirs
+	$(ROOTDIR)/build/update_packages.sh
+	tar -C $(PRODUCT_OUT) -czf $@ packages
+endif
+
+packages:: $(ALL_PACKAGE_TARGETS)
 
 .PHONY:: packages
diff --git a/pbuilder-hooks/D05deps b/pbuilder-hooks/D05deps
index f8feb1b..70833e4 100755
--- a/pbuilder-hooks/D05deps
+++ b/pbuilder-hooks/D05deps
@@ -1,8 +1,3 @@
 #!/bin/bash -xe
-apt-get install -y apt-utils
-(
-	flock -e 200
-	( cd "$DEPSBASE"; apt-ftparchive packages . > Packages )
-	echo "deb [trusted=yes] file://$DEPSBASE ./" >> /etc/apt/sources.list
-	apt-get update
-) 200>/tmp/pbuilder-apt-lock
+echo "deb [trusted=yes] file://$DEPSBASE ./" >> /etc/apt/sources.list
+apt-get update
diff --git a/pbuilderrc b/pbuilderrc
index 1709fd9..847a975 100644
--- a/pbuilderrc
+++ b/pbuilderrc
@@ -6,7 +6,7 @@
 # read pbuilderrc.5 document for notes on specific options.
 MIRRORSITE=http://http.us.debian.org/debian
 PBUILDERSATISFYDEPENDSCMD=/usr/lib/pbuilder/pbuilder-satisfydepends-apt
-export DEPSBASE=$PRODUCT_OUT
+export DEPSBASE=$PRODUCT_OUT/packages
 BINDMOUNTS="$DEPSBASE $ROOTDIR"
 APTCACHE=$ROOTDIR/cache/pbuilder_apt
 APTCACHEHARDLINK=no
diff --git a/preamble.mk b/preamble.mk
index 4f4f12f..171b0e3 100644
--- a/preamble.mk
+++ b/preamble.mk
@@ -118,4 +118,5 @@
 
 PREBUILT_DOCKER_ROOT ?= /google/data/ro/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/docker
 
-DEBCACHE_ROOT ?= /google/data/rw/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/debcache
+FETCH_PACKAGES ?= false
+PACKAGES_FETCH_ROOT_DIRECTORY ?= /google/data/ro/teams/spacepark/enterprise/kokoro/prod/spacepark/enterprise/debcache
diff --git a/prereqs.mk b/prereqs.mk
index 492a79e..7dc16b1 100644
--- a/prereqs.mk
+++ b/prereqs.mk
@@ -5,6 +5,7 @@
 include $(ROOTDIR)/build/preamble.mk
 
 REQUIRED_PACKAGES := \
+	apt-utils \
 	bc \
 	binutils-aarch64-linux-gnu \
 	build-essential \
diff --git a/rootfs.mk b/rootfs.mk
index 8cbbae8..867c68a 100644
--- a/rootfs.mk
+++ b/rootfs.mk
@@ -11,8 +11,6 @@
 ROOTFS_FETCH_TARBALL ?= true
 ROOTFS_REVISION ?= latest
 
-ROOTFS_PUSH_DEBS ?= false
-
 USER_GROUPS := \
 	adm \
 	audio \
@@ -26,6 +24,44 @@
 	users \
 	video
 
+PRE_INSTALL_PACKAGES := \
+	aiy-board-audio \
+	aiy-board-gadget \
+	aiy-board-keyring \
+	aiy-board-tools \
+	aiy-board-wlan \
+	gstreamer1.0-alsa \
+	gstreamer1.0-plugins-bad \
+	gstreamer1.0-plugins-base \
+	gstreamer1.0-plugins-base-apps \
+	gstreamer1.0-plugins-good \
+	gstreamer1.0-tools \
+	imx-atf \
+	imx-firmware \
+	imx-gpu-viv \
+	imx-gst1.0-plugin \
+	imx-mkimage \
+	imx-vpu-hantro \
+	imx-vpuwrap \
+	libdrm2 \
+	libdrm-vivante \
+	libegl1-mesa \
+	libegl1-mesa-dev \
+	libgbm1 \
+	libgbm-dev \
+	libgl1-mesa-glx \
+	libgles1-mesa \
+	libgles2-mesa \
+	libgstreamer1.0-0 \
+	libgstreamer-plugins-bad1.0-0 \
+	libgstreamer-plugins-base1.0-0 \
+	linux-headers-4.9.51-aiy \
+	linux-image-4.9.51-aiy \
+	mesa-common-dev \
+	uboot-imx \
+	wayland-protocols-imx \
+	weston-imx \
+
 rootfs: $(PRODUCT_OUT)/rootfs.img
 rootfs_raw: $(ROOTFS_RAW_IMG)
 
@@ -86,10 +122,10 @@
 $(ROOTFS_PATCHED_IMG): $(ROOTFS_RAW_IMG) \
                        $(ROOTDIR)/board/fstab.emmc \
                        $(ROOTDIR)/build/boot.mk \
-                       $(PRODUCT_OUT)/linux-image-4.9.51-aiy_1_arm64.deb \
+                       kernel-deb \
+                       $(ROOTDIR)/cache/packages.tgz \
                        | $(PRODUCT_OUT)/boot.img \
-                         modules \
-                         packages
+                         modules
 	cp -r $(ROOTFS_RAW_IMG) $(ROOTFS_PATCHED_IMG)
 	mkdir -p $(ROOTFS_DIR)
 	-sudo umount $(ROOTFS_DIR)/boot
@@ -102,10 +138,10 @@
 
 	sudo cp $(ROOTDIR)/board/fstab.emmc $(ROOTFS_DIR)/etc/fstab
 
-	sudo mount -t tmpfs none $(ROOTFS_DIR)/tmp
-	sudo rsync -avm --exclude="*-dbgsym_*.deb" --exclude="*-dev_*.deb" --include="*.deb" --exclude="*" $(PRODUCT_OUT)/ $(ROOTFS_DIR)/tmp/
-	sudo chroot $(ROOTFS_DIR) bash -c 'apt-get install --allow-downgrades --no-install-recommends -y /tmp/*.deb'
-	sudo umount $(ROOTFS_DIR)/tmp
+	sudo sed -i '1 i\deb [trusted=yes] file:///opt/aiy/packages ./' $(ROOTFS_DIR)/etc/apt/sources.list
+	sudo mkdir -p $(ROOTFS_DIR)/opt/aiy
+	sudo tar -xvf $(ROOTDIR)/cache/packages.tgz -C $(ROOTFS_DIR)/opt/aiy/
+	sudo chroot $(ROOTFS_DIR) bash -c 'apt-get update && apt-get install --allow-downgrades --no-install-recommends -y $(PRE_INSTALL_PACKAGES)'
 
 	sudo umount $(ROOTFS_DIR)/boot
 	sudo umount $(ROOTFS_DIR)
@@ -116,21 +152,6 @@
 $(PRODUCT_OUT)/rootfs.img: $(HOST_OUT)/bin/img2simg $(ROOTFS_PATCHED_IMG)
 	$(HOST_OUT)/bin/img2simg $(ROOTFS_PATCHED_IMG) $(PRODUCT_OUT)/rootfs.img
 
-fetch_debs:
-	$(info Fetching debs from cache...)
-	mkdir -p $(PRODUCT_OUT)
-	rsync -rv $(DEBCACHE_ROOT)/ $(PRODUCT_OUT)/
-	find $(PRODUCT_OUT) -name *.deb | xargs touch -d "-1337 days ago"
-
-ifeq ($(ROOTFS_PUSH_DEBS),true)
-push_debs:
-	$(info Pushing debs to cache...)
-	rsync -rvm --exclude="aiy*.deb" --include="*.deb" --include="*/" --exclude="*"  $(PRODUCT_OUT)/ $(DEBCACHE_ROOT)/
-else
-push_debs:
-	$(error Pushing debs to cache disabled)
-endif
-
 clean::
 	if mount |grep -q $(ROOTFS_DIR); then sudo umount -R $(ROOTFS_DIR); fi
 	if [[ -d $(ROOTFS_DIR) ]]; then rmdir $(ROOTFS_DIR); fi
diff --git a/signing.mk b/signing.mk
index 0d68ce5..0be4cbc 100644
--- a/signing.mk
+++ b/signing.mk
@@ -5,15 +5,16 @@
 include $(ROOTDIR)/build/preamble.mk
 
 make-repo: $(PRODUCT_OUT)/repo/debian_repo/dists/stable/Release
-$(PRODUCT_OUT)/repo/debian_repo/dists/stable/Release: $(ROOTDIR)/build/distributions | kernel-deb modules packages
+$(PRODUCT_OUT)/repo/debian_repo/dists/stable/Release: $(ROOTDIR)/build/distributions | kernel-deb modules packages-tarball
 	mkdir -p $(PRODUCT_OUT)/repo
 	mkdir -p $(PRODUCT_OUT)/repo/debian_repo
 	mkdir -p $(PRODUCT_OUT)/repo/deb_repo_config
+	tar -xvf $(ROOTDIR)/cache/packages.tgz -C $(PRODUCT_OUT)/repo
 	cp $(ROOTDIR)/build/distributions $(PRODUCT_OUT)/repo/deb_repo_config/
 	reprepro --basedir $(PRODUCT_OUT)/repo \
 	         --outdir $(PRODUCT_OUT)/repo/debian_repo \
 	         --confdir $(PRODUCT_OUT)/repo/deb_repo_config \
-	         includedeb stable $(PRODUCT_OUT)/*.deb
+	         includedeb stable $(PRODUCT_OUT)/repo/packages/*.deb
 	find $(PRODUCT_OUT)/repo/debian_repo -type d | xargs chmod 777
 	find $(PRODUCT_OUT)/repo/debian_repo -type f | xargs chmod 666
 
diff --git a/update_packages.sh b/update_packages.sh
new file mode 100755
index 0000000..fb1988f
--- /dev/null
+++ b/update_packages.sh
@@ -0,0 +1,5 @@
+#!/bin/bash -xe
+(
+	flock -e 200
+	( cd $PRODUCT_OUT/packages; apt-ftparchive packages . > Packages )
+) 200>$PRODUCT_OUT/obj/packages-lock