publish-unstable: Make publishing unstable also update
This eliminates the need for a separate, somewhat confusing job. Essentially,
this makes the publishing process idempotent: it will always either create the
publish if one doesn't exist, or it will update the existing ones.
This change also migrates a bunch of utility functions into the library, so we
can better standardize on how we handle things with aptly.
Change-Id: If6daadb1411c0af85dfbd6b82edbb0c1ed6c91b6
diff --git a/cicd/jobs/task_update_unstable.jenkins b/cicd/jobs/task_update_unstable.jenkins
deleted file mode 100644
index 92dfe9d..0000000
--- a/cicd/jobs/task_update_unstable.jenkins
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env groovy
-
-pipelineJob("task.update.unstable") {
- description("Update apt packages repository for unstable \"release\"")
-
- definition {
- cpsScm {
- scm {
- git {
- remote {
- url('https://coral.googlesource.com/gke-jenkins')
- }
- branches('*/master')
- }
- }
- scriptPath("cicd/pipelines/tasks/task_update_unstable.jenkins")
- }
- }
-}
diff --git a/cicd/pipelines/tasks/task_publish_unstable.jenkins b/cicd/pipelines/tasks/task_publish_unstable.jenkins
index 600be6b..baae01d 100644
--- a/cicd/pipelines/tasks/task_publish_unstable.jenkins
+++ b/cicd/pipelines/tasks/task_publish_unstable.jenkins
@@ -1,23 +1,31 @@
#!/usr/bin/env groovy
-String getLatestSnapshot(repository_stem) {
- def script = """
- aptly snapshot list --sort=time --raw \
- | grep -E '^${repository_stem}-' \
- | tail -n1
- """
+def updateBspPublish(String boardName) {
+ def bspSnapshotName = functions.getLatestSnapshot("unstable-bsp-${boardName}")
- return sh(returnStdout: true, script: script).trim()
+ if (functions.bspIsPublished(releaseName, boardName)) {
+ sh "aptly publish switch --batch --force-overwrite --passphrase-file=/var/lib/aptly/keyring/passphrase.txt unstable filesystem:public:unstable-bsp-${boardName} ${bspSnapshotName}"
+ } else {
+ sh "aptly publish snapshot --batch --force-overwrite --passphrase-file=/var/lib/aptly/keyring/passphrase.txt --architectures=source,amd64,arm64,armhf --distribution=unstable ${bspSnapshotName} filesystem:public:unstable-bsp-${boardName}"
+ }
}
-def installGpgKeyring() {
+def updateCorePublish() {
+ def debianMainMirrorSnapshotName = functions.getLatestSnapshot('debian-buster')
+ def debianContribMirrorSnapshotName = functions.getLatestSnapshot('debian-buster-contrib')
+ def debianNonfreeMirrorSnapshotName = functions.getLatestSnapshot('debian-buster-nonfree')
+ def coreSnapshotName = functions.getLatestSnapshot('unstable-core')
+ String stamp = functions.generateDateStamp()
+
sh """
- install -d -m 700 -o root -g root /var/lib/aptly/.gnupg
- tar -C /var/lib/aptly/.gnupg -zxf /var/lib/aptly/keyring/release-keyring.tar.gz
- chown -R root:root /var/lib/aptly/.gnupg
- find /var/lib/aptly/.gnupg -type d -exec chmod 700 '{}' ';'
- find /var/lib/aptly/.gnupg -type f -exec chmod 600 '{}' ';'
+ aptly snapshot merge core-full-unstable-${stamp} ${debianMainMirrorSnapshotName} ${debianContribMirrorSnapshotName} ${debianNonfreeMirrorSnapshotName} ${coreSnapshotName}
"""
+
+ if (functions.coreIsPublished(releaseName)) {
+ sh "aptly publish switch --batch --force-overwrite --passphrase-file=/var/lib/aptly/keyring/passphrase.txt unstable filesystem:public:unstable core-full-unstable-${stamp}"
+ } else {
+ sh "aptly publish snapshot --batch --force-overwrite --passphrase-file=/var/lib/aptly/keyring/passphrase.txt --architectures=source,amd64,arm64,armhf --distribution=unstable core-full-unstable-${stamp} filesystem:public:unstable"
+ }
}
def workspacePath = "/home/jenkins/workspace"
@@ -37,34 +45,18 @@
def aptlyVolume = persistentVolumeClaim(claimName: 'aptly-state', mountPath: '/var/lib/aptly')
def gpgVolume = secretVolume(secretName: 'mendel-release-credentials', mountPath: '/var/lib/aptly/keyring')
-def boards = [
- "enterprise",
- "excelsior"
-]
-
podTemplate(label: buildLabel, containers: [jnlpContainer, debianContainer], volumes: [aptlyVolume, gpgVolume], envVars: []) {
node(buildLabel) {
dir(sourcePath) {
container('debian') {
sh "cp /etc/aptly.conf ~/.aptly.conf"
withEnv(['GNUPGHOME=/var/lib/aptly/.gnupg']) {
- def debianMainMirrorSnapshotName = getLatestSnapshot('debian-buster')
- def debianContribMirrorSnapshotName = getLatestSnapshot('debian-buster-contrib')
- def debianNonfreeMirrorSnapshotName = getLatestSnapshot('debian-buster-nonfree')
- def coreSnapshotName = getLatestSnapshot('unstable-core')
- def date = new Date()
- String stamp = date.format("yyyyMMdd-HHmmss")
+ functions.installGpgKeyring()
- installGpgKeyring()
+ updateCorePublish();
- sh """
- aptly snapshot merge core-full-unstable-${stamp} ${debianMainMirrorSnapshotName} ${debianContribMirrorSnapshotName} ${debianNonfreeMirrorSnapshotName} ${coreSnapshotName}
- aptly publish snapshot --batch --force-overwrite --passphrase-file=/var/lib/aptly/keyring/passphrase.txt --architectures=source,amd64,arm64,armhf --distribution=unstable core-full-unstable-${stamp} filesystem:public:unstable
- """
-
- for (board in boards) {
- def bspSnapshotName = getLatestSnapshot("unstable-bsp-${board}")
- sh "aptly publish snapshot --batch --force-overwrite --passphrase-file=/var/lib/aptly/keyring/passphrase.txt --architectures=source,amd64,arm64,armhf --distribution=unstable ${bspSnapshotName} filesystem:public:unstable-bsp-${board}"
+ for (board in functions.SupportedBoards) {
+ updateBspPublish(board);
}
}
}
diff --git a/cicd/pipelines/tasks/task_update_unstable.jenkins b/cicd/pipelines/tasks/task_update_unstable.jenkins
deleted file mode 100644
index eb1c43f..0000000
--- a/cicd/pipelines/tasks/task_update_unstable.jenkins
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env groovy
-
-String getLatestSnapshot(repository_stem) {
- def script = """
- aptly snapshot list --sort=time --raw \
- | grep -E '^${repository_stem}-[0-9]' \
- | tail -n1
- """
-
- return sh(returnStdout: true, script: script).trim()
-}
-
-def installGpgKeyring() {
- sh """
- install -d -m 700 -o root -g root /var/lib/aptly/.gnupg
- tar -C /var/lib/aptly/.gnupg -zxf /var/lib/aptly/keyring/release-keyring.tar.gz
- chown -R root:root /var/lib/aptly/.gnupg
- find /var/lib/aptly/.gnupg -type d -exec chmod 700 '{}' ';'
- find /var/lib/aptly/.gnupg -type f -exec chmod 600 '{}' ';'
- """
-}
-
-def workspacePath = "/home/jenkins/workspace"
-def buildLabel = "task.publish.unstable-${UUID.randomUUID().toString()}"
-def sourcePath = "${workspacePath}/src"
-
-// FIXME(jtgans): Get rid of privileged! This is a security risk!
-def jnlpContainer = containerTemplate(name: 'jnlp',
- image: 'jenkins/jnlp-slave:alpine')
-def debianContainer = containerTemplate(name: 'debian',
- image: 'gcr.io/mendel-linux-cloud-infra/mendel-builder:latest',
- command: 'cat',
- args: '',
- ttyEnabled: true,
- privileged: true,
- alwaysPullImage: true)
-def aptlyVolume = persistentVolumeClaim(claimName: 'aptly-state', mountPath: '/var/lib/aptly')
-def gpgVolume = secretVolume(secretName: 'mendel-release-credentials', mountPath: '/var/lib/aptly/keyring')
-
-podTemplate(label: buildLabel, containers: [jnlpContainer, debianContainer], volumes: [aptlyVolume, gpgVolume], envVars: []) {
- node(buildLabel) {
- dir(sourcePath) {
- container('debian') {
- sh "cp /etc/aptly.conf ~/.aptly.conf"
- withEnv(['GNUPGHOME=/var/lib/aptly/.gnupg']) {
- def debianMainMirrorSnapshotName = getLatestSnapshot('debian-buster')
- def debianContribMirrorSnapshotName = getLatestSnapshot('debian-buster-contrib')
- def debianNonfreeMirrorSnapshotName = getLatestSnapshot('debian-buster-non-free')
- def coreSnapshotName = getLatestSnapshot('unstable-core')
- def bspSnapshotName = getLatestSnapshot('unstable-bsp-enterprise')
- def date = new Date()
- String stamp = date.format("yyyyMMdd-HHmmss")
-
- installGpgKeyring()
-
- sh """
- aptly snapshot merge core-full-unstable-${stamp} ${debianMainMirrorSnapshotName} ${debianContribMirrorSnapshotName} ${debianNonfreeMirrorSnapshotName} ${coreSnapshotName}
- aptly publish switch --batch --force-overwrite --passphrase-file=/var/lib/aptly/keyring/passphrase.txt unstable filesystem:public:unstable core-full-unstable-${stamp}
- aptly publish switch --batch --force-overwrite --passphrase-file=/var/lib/aptly/keyring/passphrase.txt unstable filesystem:public:unstable-bsp-enterprise ${bspSnapshotName}
- """
- }
- }
- }
- }
-}
diff --git a/vars/functions.groovy b/vars/functions.groovy
index 9e7ae84..8f2ce4e 100644
--- a/vars/functions.groovy
+++ b/vars/functions.groovy
@@ -1,5 +1,35 @@
#!/usr/bin/env groovy
+def SupportedBoards = [
+ 'enterprise',
+ 'excelsior'
+]
+
+String generateDateStamp() {
+ def date = new Date()
+ return date.format("yyyyMMdd-HHmmss")
+}
+
+String getLatestSnapshot(repository_stem) {
+ def script = """
+ aptly snapshot list --sort=time --raw \
+ | grep -E '^${repository_stem}-' \
+ | tail -n1
+ """
+
+ return sh(returnStdout: true, script: script).trim()
+}
+
+def installGpgKeyring() {
+ sh """
+ install -d -m 700 -o root -g root /var/lib/aptly/.gnupg
+ tar -C /var/lib/aptly/.gnupg -zxf /var/lib/aptly/keyring/release-keyring.tar.gz
+ chown -R root:root /var/lib/aptly/.gnupg
+ find /var/lib/aptly/.gnupg -type d -exec chmod 700 '{}' ';'
+ find /var/lib/aptly/.gnupg -type f -exec chmod 600 '{}' ';'
+ """
+}
+
def initSourceTree(boardName, needsNative = false) {
sh """
/opt/gcompute-tools/git-cookie-authdaemon
@@ -134,3 +164,15 @@
}
}
}
+
+String coreIsPublished(String releaseName) {
+ def script = "aptly publish list --raw |grep -E '^filesystem:public:${releaseName} ${releaseName}'; true"
+ String publishName = sh(returnStdout: true, script: script).trim()
+ return (publishName != "")
+}
+
+String bspIsPublished(String releaseName, String boardName) {
+ def script = "aptly publish list --raw |grep -E '^filesystem:public:${releaseName}-bsp-${boardName} ${releaseName}'; true"
+ String publishName = sh(returnStdout: true, script: script).trim()
+ return (publishName != "")
+}
