blob: 4a38b8b74c683d542ac517b7eba59fb9f922d643 [file] [log] [blame]
#!/usr/bin/python3
import argparse
import glob
import os
import re
import shutil
import subprocess
import sys
import tempfile
from apt import cache
from debian import changelog
from debian import deb822
from git import Repo
VERSION_MAP = {
'any': 'armhf arm64',
'linux-any': 'armhf arm64',
}
ARCHES = [
'arm64',
'armhf',
]
def GetDebianDirectories(rootdir):
packages_dir = os.path.join(rootdir, 'packages')
if not os.path.exists(packages_dir):
print('No packages directory found!')
sys.exit(-1)
debian_dirs = []
for package in os.scandir(packages_dir):
debian_dir = os.path.join(package.path, 'debian')
if os.path.exists(debian_dir):
debian_dirs.append(debian_dir)
return debian_dirs
def GeneratePackageList(directory):
package_name = os.path.split(os.path.split(directory)[0])[1]
package_tuples = []
with open(os.path.join(directory, 'changelog'), 'rb') as changelog_file, \
open(os.path.join(directory, 'control')) as control_file:
cl = changelog.Changelog(file=changelog_file)
version = str(cl.get_version())
packages = deb822.Packages.iter_paragraphs(control_file)
for p in packages:
if 'Package' in p and 'Architecture' in p:
arches = p['Architecture']
if arches in VERSION_MAP:
arches = VERSION_MAP[arches]
arches = arches.split(' ')
for arch in arches:
package_tuples.append((package_name, '%s:%s' % (p['Package'], arch), version))
return package_tuples
def UpdateNeeded(package_name, package_version, apt_cache):
package_name = package_name
package = apt_cache.get(package_name)
if package:
return package_version > package.versions[0].version
else:
return True
def GetSourceDirectory(package):
proc = subprocess.run(['make', package + '-source-directory'], stdout=subprocess.PIPE, universal_newlines=True)
proc.check_returncode()
for line in proc.stdout.split(os.linesep):
if line.startswith('Source directory: '):
return line.split(': ')[1]
def CheckVersionTags(rootdir, package, version):
source_dir = os.path.join(rootdir, GetSourceDirectory(package))
debian_dir = os.path.join(rootdir, 'packages', package)
source_repo = Repo(source_dir)
debian_repo = Repo(debian_dir)
source_tags = source_repo.git.tag('-l')
debian_tags = debian_repo.git.tag('-l')
if version in source_tags and version in debian_tags:
print('Found tags for %s of %s. Checking out tags.' % (version, package))
source_repo.git.checkout('tags/' + version)
debian_repo.git.checkout('tags/' + version)
return True
else:
print('Did not find tags for %s of %s.' % (version, package))
return False
def main():
parser = argparse.ArgumentParser(description='Find which packages are newer in the local repository than Apt')
parser.add_argument('-rootdir', type=str, required=True)
parser.add_argument('-sources_list', type=str, required=True)
parser.add_argument('-package_dir', type=str, required=True)
parser.add_argument('-output_tarball', type=str, required=True)
args = parser.parse_args()
# Find directories of Debian package data.
debian_directories = GetDebianDirectories(args.rootdir)
packages = []
for directory in debian_directories:
packages += GeneratePackageList(directory)
# Check the versions of packages in the local repository,
# and compare against the versions of packages in the apt cache.
# If a source version that is newer than upstream is present in
# the local source repository, check whether git tags exist for that version.
packages_to_update = dict()
debs_to_update = set()
with tempfile.TemporaryDirectory() as tempdir:
apt_dir = os.path.join(tempdir, 'etc', 'apt')
os.makedirs(apt_dir)
shutil.copyfile(args.sources_list, os.path.join(apt_dir, 'sources.list'))
apt_cache = cache.Cache(rootdir=tempdir, memonly=True)
apt_cache.update()
apt_cache.open()
for (package_group, package_name, version) in packages:
if UpdateNeeded(package_name, version, apt_cache):
if CheckVersionTags(args.rootdir, package_group, version):
packages_to_update[package_group] = version
debs_to_update.add(package_name)
apt_cache.close()
# Compile appropriately tagged packages for all arches.
for package in packages_to_update:
print('make ' + package + '...')
for arch in ARCHES:
proc = subprocess.run(["make", "USERSPACE_ARCH="+arch, package], stdout=sys.stdout, stderr=sys.stderr)
proc.check_returncode()
# Find the set of output files corresponding to the packages we are going to upload.
output_files = set()
for deb in debs_to_update:
for filename in glob.glob('%s/**/*%s*' % (args.package_dir, deb.split(':')[0])):
output_files.add(filename)
# Generate a tarball appropriate for uploading containing the new packages.
with tempfile.TemporaryDirectory() as tempdir:
bsp_dir = os.path.join(tempdir, 'packages', 'bsp')
core_dir = os.path.join(tempdir, 'packages', 'core')
os.makedirs(bsp_dir)
os.makedirs(core_dir)
for filename in output_files:
(path, deb_name) = os.path.split(filename)
(_, repository) = os.path.split(path)
if repository == 'bsp':
shutil.copy(filename, bsp_dir)
if repository == 'core':
shutil.copy(filename, core_dir)
tar_command = "tar -C %s --overwrite -czf %s packages" % (tempdir, args.output_tarball)
proc = subprocess.run(tar_command.split(' '), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc.check_returncode()
if __name__ == '__main__':
main()