From 9274060aa9e57b5af515d531559d8638503ea317 Mon Sep 17 00:00:00 2001 From: Pekka Ristola Date: Tue, 30 May 2023 23:34:30 +0300 Subject: [PATCH 01/63] Improve archfiles nvchecker source - Replace the JSON network api call with `pacman -Fl`, which doesn't require multiple requests to Arch servers. - Change the `update_pacmandb` function to also update the files database with `pacman -Fy`. - Insert the correct `dbpath` automatically when parsing `lilac.yaml` files. - Improve the interface of the archfiles source. - Deprecate the `pkgpart` option. Backwards compatibility is maintained. - Document the options for `archfiles`. --- lilac2/lilacyaml.py | 3 ++- lilac2/pkgbuild.py | 26 ++++++++++----------- nvchecker_source/README.rst | 26 +++++++++++++++++++++ nvchecker_source/archfiles.py | 44 +++++++++++++++++++++++++++++------ 4 files changed, 77 insertions(+), 22 deletions(-) diff --git a/lilac2/lilacyaml.py b/lilac2/lilacyaml.py index 5100610e..cbff225e 100644 --- a/lilac2/lilacyaml.py +++ b/lilac2/lilacyaml.py @@ -136,7 +136,8 @@ def parse_update_on( entry.setdefault(k, v) # fill our dbpath if not provided - if entry.get('source') == 'alpm': + source = entry.get('source') + if source == 'alpm' or source == 'archfiles': entry.setdefault('dbpath', str(PACMAN_DB_DIR)) ret_update.append(entry) diff --git a/lilac2/pkgbuild.py b/lilac2/pkgbuild.py index 3c4518cb..e76062d3 100644 --- a/lilac2/pkgbuild.py +++ b/lilac2/pkgbuild.py @@ -52,20 +52,18 @@ def _save_timed_dict( safe_overwrite(str(path), data_str, mode='w') def update_pacmandb(dbpath: Path, *, quiet: bool = False) -> None: - if quiet: - kwargs = {'stdout': subprocess.DEVNULL} - else: - kwargs = {} - - for _ in range(3): - p = subprocess.run( # type: ignore # what a mess... - ['fakeroot', 'pacman', '-Sy', '--dbpath', dbpath], - **kwargs, - ) - if p.returncode == 0: - break - else: - p.check_returncode() + stdout = subprocess.DEVNULL if quiet else None + + for update_arg in ['-Sy', '-Fy']: + for _ in range(3): + p = subprocess.run( + ['fakeroot', 'pacman', update_arg, '--dbpath', dbpath], + stdout = stdout, + ) + if p.returncode == 0: + break + else: + p.check_returncode() def update_data(dbpath: Path, *, quiet: bool = False) -> None: update_pacmandb(dbpath, quiet=quiet) diff --git a/nvchecker_source/README.rst b/nvchecker_source/README.rst index 0271b908..6f46c174 100644 --- a/nvchecker_source/README.rst +++ b/nvchecker_source/README.rst @@ -16,3 +16,29 @@ use_max_tag This source supports `list options`_ when ``use_max_tag`` is set. .. _list options: https://github.com/lilydjwg/nvchecker#list-options + +ALPM files database +------------------- +:: + + source = "archfiles" + +Search package files in a local ALPM files database. The package does not need to be installed. + +archfiles + Name of the package. + +filename + Regular expression for the file name. If it contains a matching group, the first group is returned. Otherwise return the whole file name. + +repo + Name of the package repository in which the package resides. If not provided, search all repositories. + +strip_dir + Strip directory from the path before matching. Defaults to ``true``. + +dbpath + Path to the ALPM database directory. Lilac sets this automatically. + +pkgpart + Deprecated, use ``archfiles`` and ``repo`` instead. Has the form ``//``. diff --git a/nvchecker_source/archfiles.py b/nvchecker_source/archfiles.py index c1b4a032..fc019bbf 100644 --- a/nvchecker_source/archfiles.py +++ b/nvchecker_source/archfiles.py @@ -1,17 +1,47 @@ +from asyncio import create_subprocess_exec +from asyncio.subprocess import PIPE import re from nvchecker.api import GetVersionError -PKG_URL = 'https://archlinux.org/packages/%s/files/json/' +async def get_files(info: tuple) -> list: + dbpath, pkg = info + # there's no pyalpm bindings for the file databases + cmd = ['pacman', '-Flq', '--dbpath', dbpath, pkg] + + p = await create_subprocess_exec(*cmd, stdout = PIPE, stderr = PIPE) + stdout, stderr = await p.communicate() + + if p.returncode == 0: + return stdout.decode().splitlines() + else: + raise GetVersionError( + 'pacman failed to get file list', + pkg = pkg, + cmd = cmd, + stdout = stdout.decode(errors='replace'), + stderr = stderr.decode(errors='replace'), + returncode = p.returncode, + ) async def get_version(name, conf, *, cache, **kwargs): - key = conf['pkgpart'] + pkg = conf.get('archfiles') + repo = conf.get('repo') + if pkg is None: + repo, _, pkg = conf['pkgpart'].split('/') + if repo is not None: + pkg = f'{repo}/{pkg}' + dbpath = conf.get('dbpath', '/var/lib/pacman') regex = re.compile(conf['filename']) - j = await cache.get_json(PKG_URL % key) + strip_dir = conf.get('strip_dir', True) + + files = await cache.get((dbpath, pkg), get_files) - for f in j['files']: - fn = f.rsplit('/', 1)[-1] - if regex.fullmatch(fn): - return fn + for f in files: + fn = f.rsplit('/', 1)[-1] if strip_dir else f + match = regex.fullmatch(fn) + if match: + groups = match.groups() + return groups[0] if len(groups) > 0 else fn raise GetVersionError('no file matches specified regex') From f1a513fb9d6819db0cd33f599b70dc356f8a2183 Mon Sep 17 00:00:00 2001 From: Pekka Ristola Date: Wed, 31 May 2023 21:57:18 +0300 Subject: [PATCH 02/63] Revert changes to archfiles source in favor of alpmfiles source in nvchecker --- lilac2/lilacyaml.py | 2 +- nvchecker_source/README.rst | 26 --------------------- nvchecker_source/archfiles.py | 44 ++++++----------------------------- 3 files changed, 8 insertions(+), 64 deletions(-) diff --git a/lilac2/lilacyaml.py b/lilac2/lilacyaml.py index cbff225e..49d18095 100644 --- a/lilac2/lilacyaml.py +++ b/lilac2/lilacyaml.py @@ -137,7 +137,7 @@ def parse_update_on( # fill our dbpath if not provided source = entry.get('source') - if source == 'alpm' or source == 'archfiles': + if source == 'alpm' or source == 'alpmfiles': entry.setdefault('dbpath', str(PACMAN_DB_DIR)) ret_update.append(entry) diff --git a/nvchecker_source/README.rst b/nvchecker_source/README.rst index 6f46c174..0271b908 100644 --- a/nvchecker_source/README.rst +++ b/nvchecker_source/README.rst @@ -16,29 +16,3 @@ use_max_tag This source supports `list options`_ when ``use_max_tag`` is set. .. _list options: https://github.com/lilydjwg/nvchecker#list-options - -ALPM files database -------------------- -:: - - source = "archfiles" - -Search package files in a local ALPM files database. The package does not need to be installed. - -archfiles - Name of the package. - -filename - Regular expression for the file name. If it contains a matching group, the first group is returned. Otherwise return the whole file name. - -repo - Name of the package repository in which the package resides. If not provided, search all repositories. - -strip_dir - Strip directory from the path before matching. Defaults to ``true``. - -dbpath - Path to the ALPM database directory. Lilac sets this automatically. - -pkgpart - Deprecated, use ``archfiles`` and ``repo`` instead. Has the form ``//``. diff --git a/nvchecker_source/archfiles.py b/nvchecker_source/archfiles.py index fc019bbf..c1b4a032 100644 --- a/nvchecker_source/archfiles.py +++ b/nvchecker_source/archfiles.py @@ -1,47 +1,17 @@ -from asyncio import create_subprocess_exec -from asyncio.subprocess import PIPE import re from nvchecker.api import GetVersionError -async def get_files(info: tuple) -> list: - dbpath, pkg = info - # there's no pyalpm bindings for the file databases - cmd = ['pacman', '-Flq', '--dbpath', dbpath, pkg] - - p = await create_subprocess_exec(*cmd, stdout = PIPE, stderr = PIPE) - stdout, stderr = await p.communicate() - - if p.returncode == 0: - return stdout.decode().splitlines() - else: - raise GetVersionError( - 'pacman failed to get file list', - pkg = pkg, - cmd = cmd, - stdout = stdout.decode(errors='replace'), - stderr = stderr.decode(errors='replace'), - returncode = p.returncode, - ) +PKG_URL = 'https://archlinux.org/packages/%s/files/json/' async def get_version(name, conf, *, cache, **kwargs): - pkg = conf.get('archfiles') - repo = conf.get('repo') - if pkg is None: - repo, _, pkg = conf['pkgpart'].split('/') - if repo is not None: - pkg = f'{repo}/{pkg}' - dbpath = conf.get('dbpath', '/var/lib/pacman') + key = conf['pkgpart'] regex = re.compile(conf['filename']) - strip_dir = conf.get('strip_dir', True) - - files = await cache.get((dbpath, pkg), get_files) + j = await cache.get_json(PKG_URL % key) - for f in files: - fn = f.rsplit('/', 1)[-1] if strip_dir else f - match = regex.fullmatch(fn) - if match: - groups = match.groups() - return groups[0] if len(groups) > 0 else fn + for f in j['files']: + fn = f.rsplit('/', 1)[-1] + if regex.fullmatch(fn): + return fn raise GetVersionError('no file matches specified regex') From 969e75d951853a94f5bc59d0d0d3edc6a3141826 Mon Sep 17 00:00:00 2001 From: Pekka Ristola Date: Sat, 3 Jun 2023 11:36:41 +0300 Subject: [PATCH 03/63] implement rpkgs nvchecker source for CRAN and Bioconductor This source fetches and caches the `PACKAGES.gz` file of the repo. Optimized and expanded from lilydjwg/nvchecker#184 --- nvchecker_source/README.rst | 17 +++++++++++ nvchecker_source/rpkgs.py | 56 +++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 nvchecker_source/rpkgs.py diff --git a/nvchecker_source/README.rst b/nvchecker_source/README.rst index 0271b908..a6ec7d2d 100644 --- a/nvchecker_source/README.rst +++ b/nvchecker_source/README.rst @@ -16,3 +16,20 @@ use_max_tag This source supports `list options`_ when ``use_max_tag`` is set. .. _list options: https://github.com/lilydjwg/nvchecker#list-options + +R packages from CRAN and Bioconductor +------------------------------------- +:: + + source = "rpkgs" + +Check versions from CRAN and Bioconductor. This source is optimized for checking large amounts of packages at once. If you want to check only a few, the ``cran`` source is better for CRAN packages. + +pkgname + Name of the R package. + +repo + The repo of the package. Possible values are ``cran``, ``bioc``, ``bioc-data-annotation``, ``bioc-data-experiment`` and ``bioc-workflows``. + +md5 + If set to ``true``, a ``#`` character and the md5sum of the source archive is appended to the version. Defaults to ``false``. diff --git a/nvchecker_source/rpkgs.py b/nvchecker_source/rpkgs.py new file mode 100644 index 00000000..a42ceb05 --- /dev/null +++ b/nvchecker_source/rpkgs.py @@ -0,0 +1,56 @@ +from typing import Dict +from zlib import decompress + +from nvchecker.api import GetVersionError, session + +BIOC_TEMPLATE = 'https://bioconductor.org/packages/release/%s/src/contrib/PACKAGES.gz' + +URL_MAP = { + 'cran': 'https://cran.r-project.org/src/contrib/PACKAGES.gz', + 'bioc': BIOC_TEMPLATE % 'bioc', + 'bioc-data-annotation': BIOC_TEMPLATE % 'data/annotation', + 'bioc-data-experiment': BIOC_TEMPLATE % 'data/experiment', + 'bioc-workflows': BIOC_TEMPLATE % 'workflows', +} + +PKG_FIELD = b'Package: ' +VER_FIELD = b'Version: ' +MD5_FIELD = b'MD5sum: ' + +PKG_FLEN = len(PKG_FIELD) +VER_FLEN = len(VER_FIELD) +MD5_FLEN = len(MD5_FIELD) + +async def get_versions(repo: str) -> Dict[str, str]: + url = URL_MAP.get(repo) + if url is None: + raise GetVersionError(f'Unknown repo {repo}') + res = await session.get(url) + data = decompress(res.body, wbits = 31) + + result = {} + for section in data.split(b'\n\n'): + pkg = ver = md5 = None + for line in section.split(b'\n'): + if line.startswith(PKG_FIELD): + pkg = line[PKG_FLEN:].decode('utf8') + elif line.startswith(VER_FIELD): + ver = line[VER_FLEN:].decode('utf8') + elif line.startswith(MD5_FIELD): + md5 = line[MD5_FLEN:].decode('utf8') + if pkg is None or ver is None or md5 is None: + raise GetVersionError('Invalid package data', pkg = pkg, ver = ver, md5 = md5) + result[pkg] = (ver, md5) + + return result + +async def get_version(name, conf, *, cache, **kwargs): + pkgname = conf.get('pkgname', name) + repo = conf['repo'] + versions = await cache.get(repo, get_versions) + data = versions.get(pkgname) + if data is None: + raise GetVersionError(f'Package {pkgname} not found in repo {repo}') + add_md5 = conf.get('md5', False) + ver, md5 = data + return f'{ver}#{md5}' if add_md5 else ver From 4857421a9f10832db6f956a41f60b05102ae124e Mon Sep 17 00:00:00 2001 From: Pekka Ristola Date: Sat, 3 Jun 2023 11:47:50 +0300 Subject: [PATCH 04/63] rpkgs: fix typing --- nvchecker_source/rpkgs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nvchecker_source/rpkgs.py b/nvchecker_source/rpkgs.py index a42ceb05..7781a1b1 100644 --- a/nvchecker_source/rpkgs.py +++ b/nvchecker_source/rpkgs.py @@ -1,4 +1,4 @@ -from typing import Dict +from typing import Dict, Tuple from zlib import decompress from nvchecker.api import GetVersionError, session @@ -21,7 +21,7 @@ VER_FLEN = len(VER_FIELD) MD5_FLEN = len(MD5_FIELD) -async def get_versions(repo: str) -> Dict[str, str]: +async def get_versions(repo: str) -> Dict[str, Tuple[str, str]]: url = URL_MAP.get(repo) if url is None: raise GetVersionError(f'Unknown repo {repo}') From ed1d06efac02974c53319fc5ae6de620610ab099 Mon Sep 17 00:00:00 2001 From: Pekka Ristola Date: Sun, 4 Jun 2023 08:56:04 +0300 Subject: [PATCH 05/63] rpkgs: add tests, improve error message --- nvchecker_source/rpkgs.py | 2 +- tests/test_rpkgs.py | 81 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 tests/test_rpkgs.py diff --git a/nvchecker_source/rpkgs.py b/nvchecker_source/rpkgs.py index 7781a1b1..22ac830b 100644 --- a/nvchecker_source/rpkgs.py +++ b/nvchecker_source/rpkgs.py @@ -24,7 +24,7 @@ async def get_versions(repo: str) -> Dict[str, Tuple[str, str]]: url = URL_MAP.get(repo) if url is None: - raise GetVersionError(f'Unknown repo {repo}') + raise GetVersionError('Unknown repo', repo = repo) res = await session.get(url) data = decompress(res.body, wbits = 31) diff --git a/tests/test_rpkgs.py b/tests/test_rpkgs.py new file mode 100644 index 00000000..0029a7ef --- /dev/null +++ b/tests/test_rpkgs.py @@ -0,0 +1,81 @@ +import asyncio + +import pytest +import pytest_asyncio + +pytestmark = pytest.mark.asyncio + +from nvchecker import core, __main__ as main +from nvchecker.util import Entries, VersData, RawResult + +async def run(entries: Entries) -> VersData: + task_sem = asyncio.Semaphore(20) + result_q: asyncio.Queue[RawResult] = asyncio.Queue() + keymanager = core.KeyManager(None) + + dispatcher = core.setup_httpclient() + entry_waiter = core.EntryWaiter() + futures = dispatcher.dispatch( + entries, task_sem, result_q, + keymanager, entry_waiter, 1, {}, + ) + + oldvers: VersData = {} + result_coro = core.process_result(oldvers, result_q, entry_waiter) + runner_coro = core.run_tasks(futures) + + vers, _has_failures = await main.run(result_coro, runner_coro) + return vers + +@pytest_asyncio.fixture(scope='module') +async def get_version(): + async def __call__(name, config): + entries = {name: config} + newvers = await run(entries) + return newvers.get(name) + + return __call__ + +loop = asyncio.new_event_loop() +@pytest.fixture(scope='module') +def event_loop(request): + yield loop + loop.close() + + +async def test_cran(get_version): + assert await get_version('xml2', { + 'source': 'rpkgs', + 'pkgname': 'xml2', + 'repo': 'cran', + 'md5': True, + }) == '1.3.4#1921b6cba1051577019d190895dbaeb4' + +async def test_bioc(get_version): + assert await get_version('BiocVersion', { + 'source': 'rpkgs', + 'pkgname': 'BiocVersion', + 'repo': 'bioc', + }) == '3.17.1' + +async def test_bioc_data_annotation(get_version): + assert await get_version('GO.db', { + 'source': 'rpkgs', + 'pkgname': 'GO.db', + 'repo': 'bioc-data-annotation', + }) == '3.17.0' + +async def test_bioc_data_experiment(get_version): + assert await get_version('ALL', { + 'source': 'rpkgs', + 'pkgname': 'ALL', + 'repo': 'bioc-data-experiment', + }) == '1.42.0' + +async def test_bioc_workflows(get_version): + assert await get_version('liftOver', { + 'source': 'rpkgs', + 'pkgname': 'liftOver', + 'repo': 'bioc-workflows', + 'md5': True, + }) == '1.24.0#ac7f9d6cd479fa829c7b26088297d882' From 5329c9bb9793dc3135e13c38455ee5ce9916c093 Mon Sep 17 00:00:00 2001 From: Pekka Ristola Date: Tue, 6 Jun 2023 20:40:30 +0300 Subject: [PATCH 06/63] Fix get_pkgver_and_pkgrel when pkgver uses parameter expansion --- lilac2/api.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lilac2/api.py b/lilac2/api.py index 5c55f641..c66fb726 100644 --- a/lilac2/api.py +++ b/lilac2/api.py @@ -194,15 +194,18 @@ def get_pkgver_and_pkgrel( ) -> Tuple[Optional[str], Optional[PkgRel]]: pkgrel = None pkgver = None - with suppress(FileNotFoundError), open('PKGBUILD') as f: - for l in f: - if l.startswith('pkgrel='): - pkgrel = l.rstrip().split( - '=', 1)[-1].strip('\'"') + cmd = 'source PKGBUILD && declare -p pkgver pkgrel || :' + output = run_protected(['/bin/bash', '-c', cmd], silent = True) + pattern = re.compile('declare -- pkg(ver|rel)="([^"]+)"') + for line in output.splitlines(): + m = pattern.fullmatch(line) + if m: + value = m.group(2) + if m.group(1) == "rel": with suppress(ValueError, TypeError): - pkgrel = int(pkgrel) # type: ignore - elif l.startswith('pkgver='): - pkgver = l.rstrip().split('=', 1)[-1].strip('\'"') + pkgrel = int(value) # type: ignore + else: + pkgver = value return pkgver, pkgrel From 6508a47ec0326f7de0d5abd0f663a03a98c7c7f7 Mon Sep 17 00:00:00 2001 From: Pekka Ristola Date: Sat, 10 Jun 2023 15:23:59 +0300 Subject: [PATCH 07/63] Add support for repo_makedepends configuration in lilac.yaml `makedepends` and `checkdepends` of the package should be listed in repo_makedepends instead of repo_depends so that no unnecessary dependencies get installed in the build chroots. --- lilac | 5 +-- lilac2/lilacyaml.py | 8 +++++ lilac2/packages.py | 53 +++++++++++++++++++++++------ lilac2/typing.py | 1 + schema-docs/lilac-yaml-schema.yaml | 15 +++++++- tests/test_dependency_resolution.py | 41 ++++++++++++++++++++++ 6 files changed, 109 insertions(+), 14 deletions(-) create mode 100644 tests/test_dependency_resolution.py diff --git a/lilac b/lilac index 893f25eb..8d330bfd 100755 --- a/lilac +++ b/lilac @@ -65,6 +65,7 @@ MYNAME = config['lilac']['name'] nvdata: dict[str, NvResults] = {} DEPMAP: dict[str, set[Dependency]] = {} +BUILD_DEPMAP: dict[str, set[Dependency]] = {} build_reasons: DefaultDict[str, list[BuildReason]] = defaultdict(list) logger = logging.getLogger(__name__) @@ -365,7 +366,7 @@ def build_it( update_info = nvdata[pkg], bindmounts = repo.bindmounts, tmpfs = repo.tmpfs, - depends = DEPMAP.get(pkg, ()), + depends = BUILD_DEPMAP.get(pkg, ()), repo = REPO, myname = MYNAME, destdir = DESTDIR, @@ -505,7 +506,7 @@ def main_may_raise( failed = REPO.load_managed_lilac_and_report() depman = DependencyManager(REPO.repodir) - DEPMAP = get_dependency_map(depman, REPO.lilacinfos) + DEPMAP, BUILD_DEPMAP = get_dependency_map(depman, REPO.lilacinfos) # packages we care about care_pkgs: set[str] = set() diff --git a/lilac2/lilacyaml.py b/lilac2/lilacyaml.py index 49d18095..1d6eda00 100644 --- a/lilac2/lilacyaml.py +++ b/lilac2/lilacyaml.py @@ -51,6 +51,13 @@ def load_lilac_yaml(dir: Path) -> dict[str, Any]: depends[i] = next(iter(entry.items())) else: depends[i] = entry, entry + makedepends = conf.get('repo_makedepends') + if makedepends: + for i, entry in enumerate(makedepends): + if isinstance(entry, dict): + makedepends[i] = next(iter(entry.items())) + else: + makedepends[i] = entry, entry for func in FUNCTIONS: name = conf.get(func) @@ -92,6 +99,7 @@ def load_lilacinfo(dir: Path) -> LilacInfo: update_on_build = [OnBuildEntry(**x) for x in yamlconf.get('update_on_build', [])], throttle_info = throttle_info, repo_depends = yamlconf.get('repo_depends', []), + repo_makedepends = yamlconf.get('repo_makedepends', []), time_limit_hours = yamlconf.get('time_limit_hours', 1), staging = yamlconf.get('staging', False), managed = yamlconf.get('managed', True), diff --git a/lilac2/packages.py b/lilac2/packages.py index 9a15ff74..31085125 100644 --- a/lilac2/packages.py +++ b/lilac2/packages.py @@ -15,25 +15,44 @@ def get_dependency_map( depman: DependencyManager, lilacinfos: LilacInfos, -) -> Dict[str, Set[Dependency]]: +) -> Tuple[Dict[str, Set[Dependency]], Dict[str, Set[Dependency]]]: '''compute ordered, complete dependency relations between pkgbases (the directory names) This function does not make use of pkgname because they maybe the same for different pkgdir. Those are carried by Dependency and used elsewhere. + + The first returned dict has the complete set of dependencies of the given pkgbase, including + build-time dependencies of other dependencies. The second dict has only the dependnecies + required to be installed in the build chroot. For example, if A depends on B, and B makedepends + on C, then the first dict has "A: {B, C}" while the second dict has only "A: {B}". ''' map: DefaultDict[str, Set[Dependency]] = defaultdict(set) pkgdir_map: DefaultDict[str, Set[str]] = defaultdict(set) rmap: DefaultDict[str, Set[str]] = defaultdict(set) + # same as above maps, but contain only normal dependencies, not makedepends or checkdepends + norm_map: DefaultDict[str, Set[Dependency]] = defaultdict(set) + norm_pkgdir_map: DefaultDict[str, Set[str]] = defaultdict(set) + norm_rmap: DefaultDict[str, Set[str]] = defaultdict(set) + for pkgbase, info in lilacinfos.items(): - depends = info.repo_depends + for d in info.repo_depends: + d = depman.get(d) + + pkgdir_map[pkgbase].add(d.pkgdir.name) + rmap[d.pkgdir.name].add(pkgbase) + map[pkgbase].add(d) + + norm_pkgdir_map[pkgbase].add(d.pkgdir.name) + norm_rmap[d.pkgdir.name].add(pkgbase) + norm_map[pkgbase].add(d) - ds = [depman.get(d) for d in depends] - if ds: - for d in ds: - pkgdir_map[pkgbase].add(d.pkgdir.name) - rmap[d.pkgdir.name].add(pkgbase) - map[pkgbase].update(ds) + for d in info.repo_makedepends: + d = depman.get(d) + + pkgdir_map[pkgbase].add(d.pkgdir.name) + rmap[d.pkgdir.name].add(pkgbase) + map[pkgbase].add(d) dep_order = graphlib.TopologicalSorter(pkgdir_map).static_order() for pkgbase in dep_order: @@ -42,8 +61,21 @@ def get_dependency_map( dependers = rmap[pkgbase] for dd in dependers: map[dd].update(deps) + if pkgbase in norm_rmap: + deps = norm_map[pkgbase] + dependers = norm_rmap[pkgbase] + for dd in dependers: + norm_map[dd].update(deps) + + build_dep_map: DefaultDict[str, Set[Dependency]] = defaultdict(set) + for pkgbase, info in lilacinfos.items(): + build_deps = build_dep_map[pkgbase] + build_deps.update(norm_map[pkgbase]) + for d in info.repo_makedepends: + build_deps.add(depman.get(d)) + build_deps.update(norm_map[d]) - return map + return map, build_dep_map _DependencyTuple = namedtuple( '_DependencyTuple', 'pkgdir pkgname') @@ -70,8 +102,7 @@ def resolve(self) -> Optional[Path]: elif not pkgs: return None else: - ret = sorted( - pkgs, reverse=True, key=lambda x: x.stat().st_mtime)[0] + ret = max(pkgs, key=lambda x: x.stat().st_mtime) return ret class DependencyManager: diff --git a/lilac2/typing.py b/lilac2/typing.py index 96bb4eac..bbfa92a8 100644 --- a/lilac2/typing.py +++ b/lilac2/typing.py @@ -35,6 +35,7 @@ class LilacInfo: update_on_build: list[OnBuildEntry] throttle_info: dict[int, datetime.timedelta] repo_depends: list[tuple[str, str]] + repo_makedepends: list[tuple[str, str]] time_limit_hours: float staging: bool managed: bool diff --git a/schema-docs/lilac-yaml-schema.yaml b/schema-docs/lilac-yaml-schema.yaml index 7d17954f..3ab36b59 100644 --- a/schema-docs/lilac-yaml-schema.yaml +++ b/schema-docs/lilac-yaml-schema.yaml @@ -36,7 +36,20 @@ properties: description: Time limit in hours. The build will be aborted if it doesn't finish in time. Default is one hour. type: number repo_depends: - description: Packages in the repo to be built and installed before building the current package. + description: Packages in the repo that are direct dependencies of the current package. + type: array + items: + anyOf: + - type: string + description: Package (directory) name + - type: object + description: Package base (directory) as key and package name as value + minProperties: 1 + maxProperties: 1 + additionalProperties: + type: string + repo_makedepends: + description: Packages in the repo that are in makedepends or checkdepends of the current package. type: array items: anyOf: diff --git a/tests/test_dependency_resolution.py b/tests/test_dependency_resolution.py new file mode 100644 index 00000000..44333e17 --- /dev/null +++ b/tests/test_dependency_resolution.py @@ -0,0 +1,41 @@ +from collections import namedtuple +from pathlib import Path + +from lilac2.packages import DependencyManager, get_dependency_map + +def test_dependency_map(): + depman = DependencyManager(Path('.')) + Info = namedtuple('Info', ['repo_depends', 'repo_makedepends']) + lilacinfos = { + 'A': Info(['B'], ['C']), + 'B': Info(['D'], ['C']), + 'C': Info([], ['E']), + 'D': Info([], []), + 'E': Info(['D'], []), + 'F': Info([], ['C', 'D']), + 'G': Info([], ['F']), + } + expected_all = { + 'A': { 'B', 'C', 'D', 'E' }, + 'B': { 'C', 'D', 'E' }, + 'C': { 'D', 'E' }, + 'D': set(), + 'E': { 'D' }, + 'F': { 'C', 'D', 'E' }, + 'G': { 'C', 'D', 'E', 'F' }, + } + expected_build = { + 'A': { 'B', 'C', 'D' }, + 'B': { 'C', 'D' }, + 'C': { 'D', 'E' }, + 'D': set(), + 'E': { 'D' }, + 'F': { 'C', 'D' }, + 'G': { 'F' }, + } + + res_all, res_build = get_dependency_map(depman, lilacinfos) + def parse_map(m): + return { key: { val.pkgdir.name for val in s } for key, s in m.items() } + assert parse_map(res_all) == expected_all + assert parse_map(res_build) == expected_build From 6067512e3d48b37a1592ea8fc0cd1661815a9060 Mon Sep 17 00:00:00 2001 From: Pekka Ristola Date: Sun, 11 Jun 2023 11:09:43 +0300 Subject: [PATCH 08/63] Fix errors with repo_makedepends implementation --- lilac | 4 ++-- lilac2/packages.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lilac b/lilac index 8d330bfd..b636fb01 100755 --- a/lilac +++ b/lilac @@ -308,7 +308,7 @@ def try_pick_some( if rs := build_reasons.get(pkg): if len(rs) == 1 and isinstance(rs[0], BuildReason.FailedByDeps): - ds = DEPMAP[pkg] + ds = BUILD_DEPMAP[pkg] if not all(d.resolve() for d in ds): buildsorter.done(pkg) if db.USE: @@ -485,7 +485,7 @@ def setup_thread(): def main_may_raise( D: dict[str, Any], pkgs_from_args: List[str], logdir: Path, ) -> None: - global DEPMAP + global DEPMAP, BUILD_DEPMAP if get_git_branch() not in ['master', 'main']: raise Exception('repo not on master or main, aborting.') diff --git a/lilac2/packages.py b/lilac2/packages.py index 31085125..abbdbdf9 100644 --- a/lilac2/packages.py +++ b/lilac2/packages.py @@ -72,8 +72,9 @@ def get_dependency_map( build_deps = build_dep_map[pkgbase] build_deps.update(norm_map[pkgbase]) for d in info.repo_makedepends: - build_deps.add(depman.get(d)) - build_deps.update(norm_map[d]) + d = depman.get(d) + build_deps.add(d) + build_deps.update(norm_map[d.pkgdir.name]) return map, build_dep_map From 637eabbcd8f94a7d8ff705803bdda392fbf7486c Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sun, 11 Jun 2023 18:07:34 +0800 Subject: [PATCH 09/63] remove unused # type: ignore --- lilac2/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lilac2/api.py b/lilac2/api.py index c66fb726..866a4f1a 100644 --- a/lilac2/api.py +++ b/lilac2/api.py @@ -203,7 +203,7 @@ def get_pkgver_and_pkgrel( value = m.group(2) if m.group(1) == "rel": with suppress(ValueError, TypeError): - pkgrel = int(value) # type: ignore + pkgrel = int(value) else: pkgver = value From d99defc9412402ba314fd4114fc0586000a0baf2 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 16 Jun 2023 14:30:36 +0800 Subject: [PATCH 10/63] get_pkgver_and_pkgrel: fix decimal pkgrel not parsed It was introduced by #202. --- lilac2/api.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lilac2/api.py b/lilac2/api.py index 866a4f1a..db410c94 100644 --- a/lilac2/api.py +++ b/lilac2/api.py @@ -190,9 +190,8 @@ def run_cmd(cmd: Cmd, **kwargs) -> str: else: return _run_cmd(cmd, **kwargs) -def get_pkgver_and_pkgrel( -) -> Tuple[Optional[str], Optional[PkgRel]]: - pkgrel = None +def get_pkgver_and_pkgrel() -> Tuple[Optional[str], Optional[PkgRel]]: + pkgrel: Optional[PkgRel] = None pkgver = None cmd = 'source PKGBUILD && declare -p pkgver pkgrel || :' output = run_protected(['/bin/bash', '-c', cmd], silent = True) @@ -202,8 +201,10 @@ def get_pkgver_and_pkgrel( if m: value = m.group(2) if m.group(1) == "rel": - with suppress(ValueError, TypeError): + try: pkgrel = int(value) + except (ValueError, TypeError): + pkgrel = value else: pkgver = value From e055ecc088e765b647f8b28d2f282c8f0ebdbae1 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Tue, 27 Jun 2023 14:26:12 +0800 Subject: [PATCH 11/63] add simple AUR blacklist --- lilac2/api.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lilac2/api.py b/lilac2/api.py index db410c94..bb7c18a0 100644 --- a/lilac2/api.py +++ b/lilac2/api.py @@ -42,6 +42,9 @@ s.headers['User-Agent'] = UserAgent VCS_SUFFIXES = ('-git', '-hg', '-svn', '-bzr') +AUR_BLACKLIST = { + 'dnrops': "creates packages that install packages into the packager's system", +} def _unquote_item(s: str) -> Optional[str]: m = re.search(r'''[ \t'"]*([^ '"]+)[ \t'"]*''', s) @@ -529,6 +532,8 @@ def aur_pre_build( error = who not in maintainers if error: raise Exception('unexpected AUR package maintainer / packager', who) + if msg := AUR_BLACKLIST.get(who): # type: ignore + raise Exception('blacklisted AUR package maintainer / packager', who, msg) pkgver, pkgrel = get_pkgver_and_pkgrel() _g.aur_pre_files = clean_directory() From 9780b359b6f7e47097e599b8addf41cd42ed3353 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Tue, 27 Jun 2023 14:27:51 +0800 Subject: [PATCH 12/63] fix AUR blacklist --- lilac2/api.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lilac2/api.py b/lilac2/api.py index bb7c18a0..dacccf91 100644 --- a/lilac2/api.py +++ b/lilac2/api.py @@ -518,13 +518,13 @@ def aur_pre_build( if name is None: name = os.path.basename(os.getcwd()) - if maintainers: - maintainer, last_packager = _get_aur_packager(name) - if last_packager == 'lilac': - who = maintainer - else: - who = last_packager + maintainer, last_packager = _get_aur_packager(name) + if last_packager == 'lilac': + who = maintainer + else: + who = last_packager + if maintainers: error = False if isinstance(maintainers, str): error = who != maintainers @@ -532,8 +532,9 @@ def aur_pre_build( error = who not in maintainers if error: raise Exception('unexpected AUR package maintainer / packager', who) - if msg := AUR_BLACKLIST.get(who): # type: ignore - raise Exception('blacklisted AUR package maintainer / packager', who, msg) + + if msg := AUR_BLACKLIST.get(who): # type: ignore + raise Exception('blacklisted AUR package maintainer / packager', who, msg) pkgver, pkgrel = get_pkgver_and_pkgrel() _g.aur_pre_files = clean_directory() From b2e0ba5aeeea7d8c7a84869915609bf7b4465af0 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Tue, 27 Jun 2023 20:20:47 +0800 Subject: [PATCH 13/63] AUR blacklist: fix potiential bug --- lilac2/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lilac2/api.py b/lilac2/api.py index dacccf91..85eddd43 100644 --- a/lilac2/api.py +++ b/lilac2/api.py @@ -533,7 +533,7 @@ def aur_pre_build( if error: raise Exception('unexpected AUR package maintainer / packager', who) - if msg := AUR_BLACKLIST.get(who): # type: ignore + if who and (msg := AUR_BLACKLIST.get(who)): raise Exception('blacklisted AUR package maintainer / packager', who, msg) pkgver, pkgrel = get_pkgver_and_pkgrel() From 82189371baa835217f7fcbde30f7d6cc02e34d6b Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 15 Jul 2023 13:30:10 +0800 Subject: [PATCH 14/63] mediawiki2pkgbuild: license can be a list --- lilac2/mediawiki2pkgbuild.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lilac2/mediawiki2pkgbuild.py b/lilac2/mediawiki2pkgbuild.py index fbd83d8f..90bf5423 100644 --- a/lilac2/mediawiki2pkgbuild.py +++ b/lilac2/mediawiki2pkgbuild.py @@ -40,12 +40,15 @@ def gen_pkgbuild( name: str, mwver: str, desc: str, - license: str, + license: str | list[str], s: requests.Session, ) -> str: major, minor = mwver.split('.') mwver_next = f'{major}.{int(minor)+1}' link = get_link(name, mwver, s) + if isinstance(license, str): + license = [license] + license_str = ' '.join(f"'{x}'" for x in license) vars = { 'name': name, 'name_lower': name.lower(), @@ -54,6 +57,6 @@ def gen_pkgbuild( 'link': link, 'mwver_cur': mwver, 'mwver_next': mwver_next, - 'license': license, + 'license': license_str, } return template.format_map(vars) From a6ae9ea0f59e0b83c9e35468c7de8e3e2ffd3109 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 15 Jul 2023 20:29:47 +0800 Subject: [PATCH 15/63] update tests --- tests/test_rpkgs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_rpkgs.py b/tests/test_rpkgs.py index 0029a7ef..616178a1 100644 --- a/tests/test_rpkgs.py +++ b/tests/test_rpkgs.py @@ -49,7 +49,7 @@ async def test_cran(get_version): 'pkgname': 'xml2', 'repo': 'cran', 'md5': True, - }) == '1.3.4#1921b6cba1051577019d190895dbaeb4' + }) == '1.3.5#20780f576451bb22e74ba6bb3aa09435' async def test_bioc(get_version): assert await get_version('BiocVersion', { From b5ad9a76e2734a0e0f1a87024778ac6091f1c098 Mon Sep 17 00:00:00 2001 From: SamLukeYes Date: Sun, 13 Aug 2023 15:32:00 +0800 Subject: [PATCH 16/63] Avoid more_pkgs being updated during iteration --- lilac | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lilac b/lilac index b636fb01..811aae6b 100755 --- a/lilac +++ b/lilac @@ -583,15 +583,17 @@ def main_may_raise( for i in info.update_on_build: if_this_then_those[i.pkgbase].add(p) more_pkgs = set() - count = 0 for p in build_reasons: if pkgs := if_this_then_those.get(p): more_pkgs.update(pkgs) - while len(more_pkgs) != count: # has new - count = len(more_pkgs) + while True: + add_to_more_pkgs = set() for p in more_pkgs: if pkgs := if_this_then_those.get(p): - more_pkgs.update(pkgs) + add_to_more_pkgs.update(pkgs) + if add_to_more_pkgs.issubset(more_pkgs): + break + more_pkgs.update(add_to_more_pkgs) for p in more_pkgs: update_on_build = REPO.lilacinfos[p].update_on_build build_reasons[p].append(BuildReason.OnBuild(update_on_build)) From a5e77fb516f27c08078bc30a3d956fa80fcacdd6 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Wed, 27 Sep 2023 17:04:47 +0800 Subject: [PATCH 17/63] update .typos.toml --- .typos.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/.typos.toml b/.typos.toml index f86fa38c..b6bcf56a 100644 --- a/.typos.toml +++ b/.typos.toml @@ -1,2 +1,3 @@ [default.extend-identifiers] update_ons = "update_ons" +O_WRONLY = "O_WRONLY" From dedae32a083b6f7d84050560063764acbb2f7736 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Wed, 27 Sep 2023 17:04:58 +0800 Subject: [PATCH 18/63] download_official_pkgbuild: don't extract .SRCINFO and .gitignore files --- lilac2/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lilac2/api.py b/lilac2/api.py index 85eddd43..58253a81 100644 --- a/lilac2/api.py +++ b/lilac2/api.py @@ -592,6 +592,8 @@ def download_official_pkgbuild(name: str) -> list[str]: dirname, filename = os.path.split(tarinfo.name) if dirname != path: continue + if filename in ('.SRCINFO', '.gitignore'): + continue tarinfo.name = filename logger.debug('extract file %s.', filename) tarf.extract(tarinfo) From 8599407a7385f88c87d1a73af393c45495a8e9e5 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 26 Oct 2023 10:25:49 +0800 Subject: [PATCH 19/63] scripts/tailf-build-log: limit printed message length --- scripts/tailf-build-log | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tailf-build-log b/scripts/tailf-build-log index 24b49f21..2fc2859a 100755 --- a/scripts/tailf-build-log +++ b/scripts/tailf-build-log @@ -85,7 +85,7 @@ def pretty_print(log): fmt = FMT[result] out = c(7) + fmt % args + FMT['_rusage'] % args if result == 'failed': - out += f'{c(8)}{log["msg"]}\n' + out += f'{c(8)}{log["msg"][:1000]}\n' sys.stdout.write(out) def iter_pkglog(): From 31e125bcedbd9f522f3074a13d390c646bf3f1d0 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 27 Oct 2023 17:30:01 +0800 Subject: [PATCH 20/63] support for alternative pacman.conf for updating repository databases fixes #191. --- config.toml.sample | 3 +++ lilac | 3 ++- lilac2/pkgbuild.py | 22 ++++++++++++++-------- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/config.toml.sample b/config.toml.sample index a3b37a2c..9a8daae5 100644 --- a/config.toml.sample +++ b/config.toml.sample @@ -84,4 +84,7 @@ tmpfs = [ "/build/.cache/bazel" ] +# pacman.conf to use for repository databases +pacman_conf = "/etc/pacman.conf" + # vim: se ft=toml: diff --git a/lilac b/lilac index 811aae6b..7690e74c 100755 --- a/lilac +++ b/lilac @@ -490,7 +490,8 @@ def main_may_raise( if get_git_branch() not in ['master', 'main']: raise Exception('repo not on master or main, aborting.') - pkgbuild.update_data(PACMAN_DB_DIR) + pacman_conf = config['misc'].get('pacman_conf') + pkgbuild.update_data(PACMAN_DB_DIR, pacman_conf) if dburl := config['lilac'].get('dburl'): import sqlalchemy diff --git a/lilac2/pkgbuild.py b/lilac2/pkgbuild.py index e76062d3..3ae39edd 100644 --- a/lilac2/pkgbuild.py +++ b/lilac2/pkgbuild.py @@ -5,7 +5,7 @@ import os import time import subprocess -from typing import Dict, List +from typing import Dict, List, Optional, Union from pathlib import Path from contextlib import suppress @@ -51,22 +51,28 @@ def _save_timed_dict( data_str = ''.join(f'{k} {v}\n' for k, v in data.items()) safe_overwrite(str(path), data_str, mode='w') -def update_pacmandb(dbpath: Path, *, quiet: bool = False) -> None: +def update_pacmandb(dbpath: Path, pacman_conf: Optional[str], + *, quiet: bool = False) -> None: stdout = subprocess.DEVNULL if quiet else None for update_arg in ['-Sy', '-Fy']: + + cmd: List[Union[str, Path]] = [ + 'fakeroot', 'pacman', update_arg, '--dbpath', dbpath, + ] + if pacman_conf is not None: + cmd += ['--config', pacman_conf] + for _ in range(3): - p = subprocess.run( - ['fakeroot', 'pacman', update_arg, '--dbpath', dbpath], - stdout = stdout, - ) + p = subprocess.run(cmd, stdout = stdout) if p.returncode == 0: break else: p.check_returncode() -def update_data(dbpath: Path, *, quiet: bool = False) -> None: - update_pacmandb(dbpath, quiet=quiet) +def update_data(dbpath: Path, pacman_conf: Optional[str], + *, quiet: bool = False) -> None: + update_pacmandb(dbpath, pacman_conf, quiet=quiet) now = int(time.time()) deadline = now - 90 * 86400 From 7ddf8c3cf4546db67c32e67d3d783395a94b345f Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen <645432-yan12125@users.noreply.gitlab.com> Date: Sat, 28 Oct 2023 13:51:32 +0800 Subject: [PATCH 21/63] Update tests for R packages --- tests/test_rpkgs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_rpkgs.py b/tests/test_rpkgs.py index 616178a1..29e22abb 100644 --- a/tests/test_rpkgs.py +++ b/tests/test_rpkgs.py @@ -56,21 +56,21 @@ async def test_bioc(get_version): 'source': 'rpkgs', 'pkgname': 'BiocVersion', 'repo': 'bioc', - }) == '3.17.1' + }) == '3.18.0' async def test_bioc_data_annotation(get_version): assert await get_version('GO.db', { 'source': 'rpkgs', 'pkgname': 'GO.db', 'repo': 'bioc-data-annotation', - }) == '3.17.0' + }) == '3.18.0' async def test_bioc_data_experiment(get_version): assert await get_version('ALL', { 'source': 'rpkgs', 'pkgname': 'ALL', 'repo': 'bioc-data-experiment', - }) == '1.42.0' + }) == '1.44.0' async def test_bioc_workflows(get_version): assert await get_version('liftOver', { @@ -78,4 +78,4 @@ async def test_bioc_workflows(get_version): 'pkgname': 'liftOver', 'repo': 'bioc-workflows', 'md5': True, - }) == '1.24.0#ac7f9d6cd479fa829c7b26088297d882' + }) == '1.26.0#65b97e4b79a79c7a4bbdebcb647f1faf' From 139765e009dadd2a8497a4edd5612bd470db5520 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Mon, 30 Oct 2023 15:36:19 +0800 Subject: [PATCH 22/63] update_pacmandb: set default value for pacman_conf argument --- lilac2/pkgbuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lilac2/pkgbuild.py b/lilac2/pkgbuild.py index 3ae39edd..63e4d344 100644 --- a/lilac2/pkgbuild.py +++ b/lilac2/pkgbuild.py @@ -51,7 +51,7 @@ def _save_timed_dict( data_str = ''.join(f'{k} {v}\n' for k, v in data.items()) safe_overwrite(str(path), data_str, mode='w') -def update_pacmandb(dbpath: Path, pacman_conf: Optional[str], +def update_pacmandb(dbpath: Path, pacman_conf: Optional[str] = None, *, quiet: bool = False) -> None: stdout = subprocess.DEVNULL if quiet else None From 595627ab7debb29ad3e5e484b57f57bfea2fd26c Mon Sep 17 00:00:00 2001 From: Roald Clark Date: Sat, 3 Feb 2024 17:27:18 +0000 Subject: [PATCH 23/63] update_pkgrel: Allow non-UTF-8 encoding --- lilac2/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lilac2/api.py b/lilac2/api.py index 58253a81..30eb6d64 100644 --- a/lilac2/api.py +++ b/lilac2/api.py @@ -243,7 +243,7 @@ def update_pkgver_and_pkgrel( def update_pkgrel( rel: Optional[PkgRel] = None, ) -> None: - with open('PKGBUILD') as f: + with open('PKGBUILD', errors='replace') as f: pkgbuild = f.read() def replacer(m): From 1774b728418f6786f9afda7c70170b8bf487900d Mon Sep 17 00:00:00 2001 From: Pekka Ristola Date: Sun, 4 Feb 2024 13:40:25 +0200 Subject: [PATCH 24/63] Update package versions in test_rpkgs --- tests/test_rpkgs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_rpkgs.py b/tests/test_rpkgs.py index 29e22abb..bd2696fb 100644 --- a/tests/test_rpkgs.py +++ b/tests/test_rpkgs.py @@ -49,14 +49,14 @@ async def test_cran(get_version): 'pkgname': 'xml2', 'repo': 'cran', 'md5': True, - }) == '1.3.5#20780f576451bb22e74ba6bb3aa09435' + }) == '1.3.6#fc6679028dca1f3047c8c745fb724524' async def test_bioc(get_version): assert await get_version('BiocVersion', { 'source': 'rpkgs', 'pkgname': 'BiocVersion', 'repo': 'bioc', - }) == '3.18.0' + }) == '3.18.1' async def test_bioc_data_annotation(get_version): assert await get_version('GO.db', { From 6e1f28192d7ceff22be6aa311d9debb188981187 Mon Sep 17 00:00:00 2001 From: Pekka Ristola Date: Sun, 4 Feb 2024 13:44:16 +0200 Subject: [PATCH 25/63] Fix pytest_asyncio warning about replacing the `event_loop` fixture in test_rpkgs See https://github.com/lilydjwg/nvchecker/commit/4ca61ba11aefe82f470cc35b2169c28d4ed45198 --- tests/test_rpkgs.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/test_rpkgs.py b/tests/test_rpkgs.py index bd2696fb..5fc0829e 100644 --- a/tests/test_rpkgs.py +++ b/tests/test_rpkgs.py @@ -3,7 +3,7 @@ import pytest import pytest_asyncio -pytestmark = pytest.mark.asyncio +pytestmark = pytest.mark.asyncio(scope='session') from nvchecker import core, __main__ as main from nvchecker.util import Entries, VersData, RawResult @@ -27,7 +27,7 @@ async def run(entries: Entries) -> VersData: vers, _has_failures = await main.run(result_coro, runner_coro) return vers -@pytest_asyncio.fixture(scope='module') +@pytest_asyncio.fixture(scope='session') async def get_version(): async def __call__(name, config): entries = {name: config} @@ -36,12 +36,6 @@ async def __call__(name, config): return __call__ -loop = asyncio.new_event_loop() -@pytest.fixture(scope='module') -def event_loop(request): - yield loop - loop.close() - async def test_cran(get_version): assert await get_version('xml2', { From 0f39bc02dc92adfae7907c162f98233a30f167d6 Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen <645432-yan12125@users.noreply.gitlab.com> Date: Sun, 4 Feb 2024 14:12:21 +0800 Subject: [PATCH 26/63] Run CI tests --- .github/workflows/test.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..277f0ddf --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,34 @@ +name: run tests + +on: [push, pull_request] + +jobs: + tests: + runs-on: ubuntu-latest + + # Use the base-devel image of Arch Linux for building pyalpm + container: archlinux:base-devel + + strategy: + fail-fast: false + matrix: + python-version: + - "3.10" + - "3.11" + - "3.12" + exclude: [] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Python deps + run: python -m pip install -U pytest pytest-asyncio nvchecker requests lxml PyYAML pyalpm structlog python_prctl + + - name: Run pytest + run: pytest From 53523d35168fec5cc2f82a5938be6f4b5307907d Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen <645432-yan12125@users.noreply.gitlab.com> Date: Sun, 4 Feb 2024 20:44:48 +0800 Subject: [PATCH 27/63] Update supported Python versions Needs Python >= 3.10 as PEP 604 syntax is used extensively --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index bdbbbc9c..c3b2de5e 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ description = 'The build bot for archlinuxcn', author = 'lilydjwg', author_email = 'lilydjwg@gmail.com', - python_requires = '>=3.7.0', + python_requires = '>=3.10.0', url = 'https://github.com/archlinuxcn/lilac', zip_safe = False, packages = find_packages(exclude=('tests',)) + ['nvchecker_source'], @@ -25,7 +25,8 @@ classifiers = [ 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ], ) From 46bdbf267084fe1cee2a5ece0b4cca1aef908842 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 23 Mar 2024 23:50:35 +0800 Subject: [PATCH 28/63] bwrap: bind /etc/resolv.conf symlink dest --- lilac2/cmd.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lilac2/cmd.py b/lilac2/cmd.py index bf5a4833..4bd5d7de 100644 --- a/lilac2/cmd.py +++ b/lilac2/cmd.py @@ -167,3 +167,9 @@ def pkgrel_changed(from_: str, to: str, pkgname: str) -> bool: '--tmpfs', '/run', '--die-with-parent', '--tmpfs', '/tmp', '--proc', '/proc', '--dev', '/dev', ] +if os.path.islink('/etc/resolv.conf'): + UNTRUSTED_PREFIX += [ # type: ignore + '--ro-bind', + os.path.realpath('/etc/resolv.conf'), + '/etc/resolv.conf', + ] From 6f2ad7e585fc6a186b3b2625b289cbe1b6925484 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sun, 24 Mar 2024 10:03:32 +0800 Subject: [PATCH 29/63] workaround a pycurl wheel issue to fix ci --- .github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 277f0ddf..da9d4c27 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,5 +30,10 @@ jobs: - name: Install Python deps run: python -m pip install -U pytest pytest-asyncio nvchecker requests lxml PyYAML pyalpm structlog python_prctl + - name: workaround pycurl wheel + run: | + sudo mkdir -p /etc/pki/tls/certs + sudo ln -s /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt + - name: Run pytest run: pytest From ce8f610005bec5dd8fd6836025229765ed19504c Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 29 Mar 2024 12:51:38 +0800 Subject: [PATCH 30/63] Revert "bwrap: bind /etc/resolv.conf symlink dest" This reverts commit 46bdbf267084fe1cee2a5ece0b4cca1aef908842. bwrap followed the broken symlink. --- lilac2/cmd.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lilac2/cmd.py b/lilac2/cmd.py index 4bd5d7de..bf5a4833 100644 --- a/lilac2/cmd.py +++ b/lilac2/cmd.py @@ -167,9 +167,3 @@ def pkgrel_changed(from_: str, to: str, pkgname: str) -> bool: '--tmpfs', '/run', '--die-with-parent', '--tmpfs', '/tmp', '--proc', '/proc', '--dev', '/dev', ] -if os.path.islink('/etc/resolv.conf'): - UNTRUSTED_PREFIX += [ # type: ignore - '--ro-bind', - os.path.realpath('/etc/resolv.conf'), - '/etc/resolv.conf', - ] From dc3218df9c51990bce3f8caee4cd58cf3e5e1b78 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 29 Mar 2024 22:46:34 +0800 Subject: [PATCH 31/63] skip packages not in nvdata --- lilac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lilac b/lilac index 7690e74c..9856e9da 100755 --- a/lilac +++ b/lilac @@ -259,6 +259,11 @@ def start_build( limit = max_concurrency - len(futures), ) for pkg in pkgs: + if pkg not in nvdata: + # this can happen when cmdline packages are specified and + # a package is pulled in by OnBuild + logger.warning('%s not in nvdata, skipping', pkg) + continue fu = executor.submit( build_it, pkg, repo, buildsorter, built, failed) futures[fu] = pkg From b6e62df080c0a41a7d639ae2281a059bda1e5438 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sun, 31 Mar 2024 13:46:01 +0800 Subject: [PATCH 32/63] also check commits for rebuilds when building packages from cmdline or they may get skipped. --- lilac | 48 +++++++++++++++++++++++------------------------- lilac2/repo.py | 6 +++--- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/lilac b/lilac index 9856e9da..735b2253 100755 --- a/lilac +++ b/lilac @@ -528,18 +528,17 @@ def main_may_raise( failed_info = D.get('failed', {}) - if not pkgs_from_args: - U = set(REPO.lilacinfos) - last_commit = D.get('last_commit', EMPTY_COMMIT) - changed = get_changed_packages(last_commit, 'HEAD') & U - - failed_prev = set(failed_info.keys()) - # no update from upstream, but build instructions have changed; rebuild - # failed ones - need_rebuild_failed = failed_prev & changed - # if pkgrel is updated, build a new release - need_rebuild_pkgrel = {x for x in changed - if pkgrel_changed(last_commit, 'HEAD', x)} - unknown + U = set(REPO.lilacinfos) + last_commit = D.get('last_commit', EMPTY_COMMIT) + changed = get_changed_packages(last_commit, 'HEAD') & U + + failed_prev = set(failed_info.keys()) + # no update from upstream, but build instructions have changed; rebuild + # failed ones + need_rebuild_failed = failed_prev & changed + # if pkgrel is updated, build a new release + need_rebuild_pkgrel = {x for x in changed + if pkgrel_changed(last_commit, 'HEAD', x)} - unknown nv_changed = {} for p, vers in nvdata.items(): @@ -572,17 +571,17 @@ def main_may_raise( if pkgs_from_args: for p in pkgs_from_args: build_reasons[p].append(BuildReason.Cmdline()) - else: - for p in need_rebuild_pkgrel: - build_reasons[p].append(BuildReason.UpdatedPkgrel()) - for p in need_rebuild_failed: - build_reasons[p].append(BuildReason.UpdatedFailed()) + for p in need_rebuild_pkgrel: + build_reasons[p].append(BuildReason.UpdatedPkgrel()) + + for p in need_rebuild_failed: + build_reasons[p].append(BuildReason.UpdatedFailed()) - for p, i in failed_info.items(): - # p might have been removed - if p in REPO.lilacinfos and (deps := i['missing']): - build_reasons[p].append(BuildReason.FailedByDeps(deps)) + for p, i in failed_info.items(): + # p might have been removed + if p in REPO.lilacinfos and (deps := i['missing']): + build_reasons[p].append(BuildReason.FailedByDeps(deps)) if_this_then_those = defaultdict(set) for p, info in REPO.lilacinfos.items(): @@ -630,10 +629,9 @@ def main_may_raise( if x in failed_info: del failed_info[x] # cleanup removed package failed_info - if not pkgs_from_args: - for x in tuple(failed_info.keys()): - if x not in REPO.lilacinfos: - del failed_info[x] + for x in tuple(failed_info.keys()): + if x not in REPO.lilacinfos: + del failed_info[x] D['failed'] = failed_info if config['lilac']['rebuild_failed_pkgs']: diff --git a/lilac2/repo.py b/lilac2/repo.py index 2aa155e1..225dca7c 100644 --- a/lilac2/repo.py +++ b/lilac2/repo.py @@ -333,14 +333,14 @@ def load_managed_lilac_and_report(self) -> dict[str, tuple[str, ...]]: self.lilacinfos, errors = lilacyaml.load_managed_lilacinfos(self.repodir) failed: dict[str, tuple[str, ...]] = {p: () for p in errors} for name, exc_info in errors.items(): - logger.error('error while loading lilac.py for %s', name, exc_info=exc_info) + logger.error('error while loading lilac.yaml for %s', name, exc_info=exc_info) exc = exc_info[1] if not isinstance(exc, Exception): raise self.send_error_report(name, exc=exc, - subject='为软件包 %s 载入 lilac.py 时失败') + subject='为软件包 %s 载入 lilac.yaml 时失败') build_logger_old.error('%s failed', name) - build_logger.exception('lilac.py error', pkgbase = name, exc_info=exc_info) + build_logger.exception('lilac.yaml error', pkgbase = name, exc_info=exc_info) return failed From 327ddd634563a1979b0168fb6cf9ecebbb8e9909 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sun, 31 Mar 2024 18:11:02 +0800 Subject: [PATCH 33/63] mark skipped packages as done --- lilac | 1 + 1 file changed, 1 insertion(+) diff --git a/lilac b/lilac index 735b2253..2c207ec3 100755 --- a/lilac +++ b/lilac @@ -263,6 +263,7 @@ def start_build( # this can happen when cmdline packages are specified and # a package is pulled in by OnBuild logger.warning('%s not in nvdata, skipping', pkg) + buildsorter.done(pkg) continue fu = executor.submit( build_it, pkg, repo, buildsorter, built, failed) From 0454bb9a771e0b979ab2997d383bfe41338d6d7b Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sun, 31 Mar 2024 23:23:33 +0800 Subject: [PATCH 34/63] don't try FailedByDeps packages when packages are given on the cmdline They are a bit scary on the current building page, and will be picked up next time. --- lilac | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lilac b/lilac index 2c207ec3..4b2f1def 100755 --- a/lilac +++ b/lilac @@ -579,10 +579,11 @@ def main_may_raise( for p in need_rebuild_failed: build_reasons[p].append(BuildReason.UpdatedFailed()) - for p, i in failed_info.items(): - # p might have been removed - if p in REPO.lilacinfos and (deps := i['missing']): - build_reasons[p].append(BuildReason.FailedByDeps(deps)) + if not pkgs_from_args: + for p, i in failed_info.items(): + # p might have been removed + if p in REPO.lilacinfos and (deps := i['missing']): + build_reasons[p].append(BuildReason.FailedByDeps(deps)) if_this_then_those = defaultdict(set) for p, info in REPO.lilacinfos.items(): From dd15e22fb1cebda571bc3972247e72f1b74dc15b Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 4 Apr 2024 19:49:57 +0800 Subject: [PATCH 35/63] minor fix for unbound variables --- lilac2/systemd.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lilac2/systemd.py b/lilac2/systemd.py index b6bfab6b..5b30b067 100644 --- a/lilac2/systemd.py +++ b/lilac2/systemd.py @@ -79,6 +79,7 @@ def _poll_cmd(pid: int) -> Generator[None, None, None]: except OSError as e: if e.errno == 22: return + raise poll = select.poll() poll.register(pidfd, select.POLLIN) @@ -98,6 +99,7 @@ def poll_rusage(name: str, deadline: float) -> tuple[RUsage, bool]: done_state = ['exited', 'failed'] try: + cgroup = '' time_start = time.monotonic() while True: pid, cgroup, state = _get_service_info(name) From 2a20d709c005bbd8f86d60b703bc401b52ca7554 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Mon, 22 Apr 2024 22:16:13 +0800 Subject: [PATCH 36/63] fix test_rpkgs due to nvchecker update --- tests/test_rpkgs.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_rpkgs.py b/tests/test_rpkgs.py index 5fc0829e..b465c0be 100644 --- a/tests/test_rpkgs.py +++ b/tests/test_rpkgs.py @@ -6,9 +6,9 @@ pytestmark = pytest.mark.asyncio(scope='session') from nvchecker import core, __main__ as main -from nvchecker.util import Entries, VersData, RawResult +from nvchecker.util import Entries, RichResult, RawResult -async def run(entries: Entries) -> VersData: +async def run(entries: Entries) -> RichResult: task_sem = asyncio.Semaphore(20) result_q: asyncio.Queue[RawResult] = asyncio.Queue() keymanager = core.KeyManager(None) @@ -20,7 +20,7 @@ async def run(entries: Entries) -> VersData: keymanager, entry_waiter, 1, {}, ) - oldvers: VersData = {} + oldvers: RichResult = {} result_coro = core.process_result(oldvers, result_q, entry_waiter) runner_coro = core.run_tasks(futures) @@ -32,7 +32,8 @@ async def get_version(): async def __call__(name, config): entries = {name: config} newvers = await run(entries) - return newvers.get(name) + if r := newvers.get(name): + return r.version return __call__ From 1921e8bb7a818a80857f153719fb79fd4df62b7c Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sun, 28 Apr 2024 16:30:53 +0800 Subject: [PATCH 37/63] update nicelogger --- lilac2/vendor/nicelogger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lilac2/vendor/nicelogger.py b/lilac2/vendor/nicelogger.py index 326728a5..1675e642 100644 --- a/lilac2/vendor/nicelogger.py +++ b/lilac2/vendor/nicelogger.py @@ -57,6 +57,7 @@ def format(self, record): 'filename', 'exc_info', 'exc_text', 'created', 'funcName', 'processName', 'process', 'msecs', 'relativeCreated', 'thread', 'threadName', 'name', 'levelno', 'msg', 'pathname', 'stack_info', + 'taskName', }) if record.exc_info: From cc1ab8b9cce19e9941ce21355feed6feac625649 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Mon, 29 Apr 2024 16:02:30 +0800 Subject: [PATCH 38/63] cmdline packages: make sure rebuilding packages have nvdata --- lilac | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/lilac b/lilac index 4b2f1def..11113c40 100755 --- a/lilac +++ b/lilac @@ -515,18 +515,6 @@ def main_may_raise( depman = DependencyManager(REPO.repodir) DEPMAP, BUILD_DEPMAP = get_dependency_map(depman, REPO.lilacinfos) - # packages we care about - care_pkgs: set[str] = set() - for pkg_to_build in pkgs_from_args: - care_pkgs.update(dep.pkgname for dep in DEPMAP[pkg_to_build]) - care_pkgs.add(pkg_to_build) - - proxy = config['nvchecker'].get('proxy') - _nvdata, unknown, rebuild = packages_need_update( - REPO, proxy, care_pkgs, - ) - nvdata.update(_nvdata) # update to the global object - failed_info = D.get('failed', {}) U = set(REPO.lilacinfos) @@ -539,7 +527,24 @@ def main_may_raise( need_rebuild_failed = failed_prev & changed # if pkgrel is updated, build a new release need_rebuild_pkgrel = {x for x in changed - if pkgrel_changed(last_commit, 'HEAD', x)} - unknown + if pkgrel_changed(last_commit, 'HEAD', x)} + + # packages we care about + care_pkgs: set[str] = set() + for pkg_to_build in pkgs_from_args: + care_pkgs.update(dep.pkgname for dep in DEPMAP[pkg_to_build]) + care_pkgs.add(pkg_to_build) + # make sure they have nvdata + care_pkgs.update(need_rebuild_failed) + care_pkgs.update(need_rebuild_pkgrel) + + proxy = config['nvchecker'].get('proxy') + _nvdata, unknown, rebuild = packages_need_update( + REPO, proxy, care_pkgs, + ) + nvdata.update(_nvdata) # update to the global object + + need_rebuild_pkgrel -= unknown nv_changed = {} for p, vers in nvdata.items(): From 7a844636240cb62d71e6f706aa2cc70994fc7be1 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Tue, 30 Apr 2024 19:30:54 +0800 Subject: [PATCH 39/63] replace the deprecated datetime.datetime.utcnow method --- lilac2/mediawiki2pkgbuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lilac2/mediawiki2pkgbuild.py b/lilac2/mediawiki2pkgbuild.py index 90bf5423..87fd04e9 100644 --- a/lilac2/mediawiki2pkgbuild.py +++ b/lilac2/mediawiki2pkgbuild.py @@ -52,7 +52,7 @@ def gen_pkgbuild( vars = { 'name': name, 'name_lower': name.lower(), - 'version': datetime.datetime.utcnow().strftime('%Y%m%d'), + 'version': datetime.datetime.now(tz=datetime.UTC).strftime('%Y%m%d'), 'desc': desc[0].lower() + desc[1:], 'link': link, 'mwver_cur': mwver, From 6d40aa22e2d358e5982b1e8129a0ac5aa3e48e25 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sun, 5 May 2024 15:50:55 +0800 Subject: [PATCH 40/63] new environment "LOGDEST" to keep across sudo --- docs/setup.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/setup.rst b/docs/setup.rst index ea55fdec..10c24f0b 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -183,7 +183,7 @@ Setup the database tables (run as lilac): Edit ``/etc/sudoers`` like:: - Defaults env_keep += "PACKAGER MAKEFLAGS GNUPGHOME BUILDTOOL" + Defaults env_keep += "PACKAGER MAKEFLAGS GNUPGHOME BUILDTOOL LOGDEST" %pkg ALL= NOPASSWD: /usr/bin/build-cleaner, /usr/bin/extra-x86_64-build, /usr/bin/multilib-build From ddd524db7707e6194eeb7db4125309bfb7751e7c Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sun, 5 May 2024 16:18:08 +0800 Subject: [PATCH 41/63] update tests --- .github/workflows/test.yml | 1 - tests/test_rpkgs.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index da9d4c27..50639698 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,6 @@ jobs: fail-fast: false matrix: python-version: - - "3.10" - "3.11" - "3.12" exclude: [] diff --git a/tests/test_rpkgs.py b/tests/test_rpkgs.py index b465c0be..bbc9de5f 100644 --- a/tests/test_rpkgs.py +++ b/tests/test_rpkgs.py @@ -51,21 +51,21 @@ async def test_bioc(get_version): 'source': 'rpkgs', 'pkgname': 'BiocVersion', 'repo': 'bioc', - }) == '3.18.1' + }) == '3.19.1' async def test_bioc_data_annotation(get_version): assert await get_version('GO.db', { 'source': 'rpkgs', 'pkgname': 'GO.db', 'repo': 'bioc-data-annotation', - }) == '3.18.0' + }) == '3.19.0' async def test_bioc_data_experiment(get_version): assert await get_version('ALL', { 'source': 'rpkgs', 'pkgname': 'ALL', 'repo': 'bioc-data-experiment', - }) == '1.44.0' + }) == '1.46.0' async def test_bioc_workflows(get_version): assert await get_version('liftOver', { From bfbc3c3cd463b0eacc99063026fbb432a511c32c Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sun, 5 May 2024 16:21:11 +0800 Subject: [PATCH 42/63] still update tests --- scripts/at-maintainer | 2 +- tests/test_rpkgs.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/at-maintainer b/scripts/at-maintainer index 943b9b15..61457696 100755 --- a/scripts/at-maintainer +++ b/scripts/at-maintainer @@ -6,7 +6,7 @@ import re from lilac2.lilacyaml import iter_pkgdir, load_lilac_yaml -REPOPATH = pathlib.Path('/ldata/src/archgitrepo/archlinuxcn') +REPOPATH = pathlib.Path('/ssddata/src/archgitrepo/archlinuxcn') PkgPattern = re.compile(r'[\w.+-]+') diff --git a/tests/test_rpkgs.py b/tests/test_rpkgs.py index bbc9de5f..543b3bee 100644 --- a/tests/test_rpkgs.py +++ b/tests/test_rpkgs.py @@ -58,7 +58,7 @@ async def test_bioc_data_annotation(get_version): 'source': 'rpkgs', 'pkgname': 'GO.db', 'repo': 'bioc-data-annotation', - }) == '3.19.0' + }) == '3.19.1' async def test_bioc_data_experiment(get_version): assert await get_version('ALL', { @@ -73,4 +73,4 @@ async def test_bioc_workflows(get_version): 'pkgname': 'liftOver', 'repo': 'bioc-workflows', 'md5': True, - }) == '1.26.0#65b97e4b79a79c7a4bbdebcb647f1faf' + }) == '1.28.0#336a9b7f29647ba8b26eb4dc139d755c' From 4154053a0b74305fcde062d47c067c79a9cf87a7 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Wed, 8 May 2024 09:16:43 +0800 Subject: [PATCH 43/63] send SIGINT to systemd worker so that it can do the cleanup --- lilac2/systemd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lilac2/systemd.py b/lilac2/systemd.py index 5b30b067..fb277827 100644 --- a/lilac2/systemd.py +++ b/lilac2/systemd.py @@ -142,7 +142,7 @@ def poll_rusage(name: str, deadline: float) -> tuple[RUsage, bool]: finally: if timedout: logger.debug('killing worker service') - subprocess.run(['systemctl', '--user', 'kill', '--signal=SIGKILL', name]) + subprocess.run(['systemctl', '--user', 'kill', '--signal=SIGINT', name]) logger.debug('stopping worker service') # stop whatever may be running (even from a previous batch) subprocess.run(['systemctl', '--user', 'stop', '--quiet', name]) From 7861b504da3f2c135f78172ee7e2f1643b9cd373 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 17 May 2024 15:01:09 +0800 Subject: [PATCH 44/63] download_official_pkgbuild: use correct version --- lilac2/api.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lilac2/api.py b/lilac2/api.py index 30eb6d64..311fa81c 100644 --- a/lilac2/api.py +++ b/lilac2/api.py @@ -577,16 +577,24 @@ def download_official_pkgbuild(name: str) -> list[str]: url = 'https://archlinux.org/packages/search/json/?name=' + name logger.info('download PKGBUILD for %s.', name) info = s.get(url).json() - pkgbase = [r['pkgbase'] for r in info['results'] if r['repo'] != 'testing'][0] + pkg = [r for r in info['results'] if not r['repo'].endswith('testing')][0] + pkgbase = pkg['pkgbase'] + epoch = pkg['epoch'] + pkgver = pkg['pkgver'] + pkgrel = pkg['pkgrel'] + if epoch: + tag = f'{epoch}:{pkgver}-{pkgrel}' + else: + tag = f'{pkgver}-{pkgrel}' - tarball_url = 'https://gitlab.archlinux.org/archlinux/packaging/packages/{0}/-/archive/main/{0}-main.tar.bz2'.format(pkgbase) + tarball_url = 'https://gitlab.archlinux.org/archlinux/packaging/packages/{0}/-/archive/main/{0}-{1}.tar.bz2'.format(pkgbase, tag) logger.debug('downloading Arch package tarball from: %s', tarball_url) tarball = s.get(tarball_url).content path = f'{pkgbase}-main' files = [] with tarfile.open( - name=f"{pkgbase}-main.tar.bz2", fileobj=io.BytesIO(tarball) + name=f"{pkgbase}-{tag}.tar.bz2", fileobj=io.BytesIO(tarball) ) as tarf: for tarinfo in tarf: dirname, filename = os.path.split(tarinfo.name) From f93973a26d8786250d0fe84c0b7365be28d33bf4 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 17 May 2024 15:18:19 +0800 Subject: [PATCH 45/63] download_official_pkgbuild: fix url --- lilac2/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lilac2/api.py b/lilac2/api.py index 311fa81c..d1ccbad9 100644 --- a/lilac2/api.py +++ b/lilac2/api.py @@ -587,7 +587,7 @@ def download_official_pkgbuild(name: str) -> list[str]: else: tag = f'{pkgver}-{pkgrel}' - tarball_url = 'https://gitlab.archlinux.org/archlinux/packaging/packages/{0}/-/archive/main/{0}-{1}.tar.bz2'.format(pkgbase, tag) + tarball_url = 'https://gitlab.archlinux.org/archlinux/packaging/packages/{0}/-/archive/{1}/{0}-{1}.tar.bz2'.format(pkgbase, tag) logger.debug('downloading Arch package tarball from: %s', tarball_url) tarball = s.get(tarball_url).content path = f'{pkgbase}-main' From d6962b8f1074a4b484f6a1f280032beb0ca9028e Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 17 May 2024 15:22:52 +0800 Subject: [PATCH 46/63] download_official_pkgbuild: still fix url and pathname --- lilac2/api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lilac2/api.py b/lilac2/api.py index d1ccbad9..f51d0e79 100644 --- a/lilac2/api.py +++ b/lilac2/api.py @@ -583,14 +583,14 @@ def download_official_pkgbuild(name: str) -> list[str]: pkgver = pkg['pkgver'] pkgrel = pkg['pkgrel'] if epoch: - tag = f'{epoch}:{pkgver}-{pkgrel}' + tag = f'{epoch}-{pkgver}-{pkgrel}' else: tag = f'{pkgver}-{pkgrel}' tarball_url = 'https://gitlab.archlinux.org/archlinux/packaging/packages/{0}/-/archive/{1}/{0}-{1}.tar.bz2'.format(pkgbase, tag) logger.debug('downloading Arch package tarball from: %s', tarball_url) tarball = s.get(tarball_url).content - path = f'{pkgbase}-main' + path = f'{pkgbase}-{tag}' files = [] with tarfile.open( @@ -600,7 +600,7 @@ def download_official_pkgbuild(name: str) -> list[str]: dirname, filename = os.path.split(tarinfo.name) if dirname != path: continue - if filename in ('.SRCINFO', '.gitignore'): + if filename in ('.SRCINFO', '.gitignore', '.nvchecker.toml'): continue tarinfo.name = filename logger.debug('extract file %s.', filename) From ef1de731da7766c94a14e11563e04071522aa859 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 24 May 2024 21:03:54 +0800 Subject: [PATCH 47/63] set systemd worker KillMode=process so that the worker has time to clean up. --- lilac2/systemd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lilac2/systemd.py b/lilac2/systemd.py index fb277827..846ecdd4 100644 --- a/lilac2/systemd.py +++ b/lilac2/systemd.py @@ -41,7 +41,7 @@ def start_cmd( cmd_s: Cmd = [ 'systemd-run', '--pipe', '--quiet', '--user', '--wait', '--remain-after-exit', '-u', name, - '-p', 'CPUWeight=100', + '-p', 'CPUWeight=100', '-p', 'KillMode=process', ] if cwd := kwargs.pop('cwd', None): From fe176e9ee705fef5ca878cc08fddf19fe863b5b2 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 31 May 2024 14:26:39 +0800 Subject: [PATCH 48/63] don't finish when packages are skipped --- lilac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lilac b/lilac index 11113c40..74db6641 100755 --- a/lilac +++ b/lilac @@ -269,8 +269,8 @@ def start_build( build_it, pkg, repo, buildsorter, built, failed) futures[fu] = pkg - if not futures: - # no task is running: we're done + if not pkgs and not futures: + # no more packages and no task is running: we're done break done, pending = futures_wait(futures, return_when=FIRST_COMPLETED) From 8f130618b65499ce294640bfcff8bf63c29fd1f1 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 31 May 2024 18:51:41 +0800 Subject: [PATCH 49/63] support comments in package.list --- lilac2/packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lilac2/packages.py b/lilac2/packages.py index abbdbdf9..45ae74c6 100644 --- a/lilac2/packages.py +++ b/lilac2/packages.py @@ -140,7 +140,7 @@ def get_split_packages(pkg: Path) -> Set[Tuple[str, str]]: pkgfile = pkg / 'package.list' if pkgfile.exists(): with open(pkgfile) as f: - packages.update((pkgbase, x) for x in f.read().split()) + packages.update((pkgbase, l.rstrip()) for l in f if not l.startswith('#')) return packages found = False From 78ad6ee83c4250f542805a7b61e11f527584420a Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 31 May 2024 22:00:20 +0800 Subject: [PATCH 50/63] py.typed --- lilac2/py.typed | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 lilac2/py.typed diff --git a/lilac2/py.typed b/lilac2/py.typed new file mode 100644 index 00000000..e69de29b From 65f8ed4180186150277be7d759da84a0a9829c5e Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 1 Jun 2024 18:01:47 +0800 Subject: [PATCH 51/63] kill systemd worker with --kill-whom=main The "kill" command doesn't take "KillMode" into account and defaults to "all". --- lilac2/systemd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lilac2/systemd.py b/lilac2/systemd.py index 846ecdd4..0188dee8 100644 --- a/lilac2/systemd.py +++ b/lilac2/systemd.py @@ -142,7 +142,7 @@ def poll_rusage(name: str, deadline: float) -> tuple[RUsage, bool]: finally: if timedout: logger.debug('killing worker service') - subprocess.run(['systemctl', '--user', 'kill', '--signal=SIGINT', name]) + subprocess.run(['systemctl', '--user', 'kill', '--kill-whom=main', '--signal=SIGINT', name]) logger.debug('stopping worker service') # stop whatever may be running (even from a previous batch) subprocess.run(['systemctl', '--user', 'stop', '--quiet', name]) From 6d02d3181274a9482203917d0bbeb629b087635a Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 21 Jun 2024 09:19:24 +0800 Subject: [PATCH 52/63] systemd: use SIGINT to stop lilac-worker and there is no need to kill it when timed out now. --- lilac2/systemd.py | 4 +--- lilac2/worker.py | 6 ++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lilac2/systemd.py b/lilac2/systemd.py index 0188dee8..c940c380 100644 --- a/lilac2/systemd.py +++ b/lilac2/systemd.py @@ -42,6 +42,7 @@ def start_cmd( 'systemd-run', '--pipe', '--quiet', '--user', '--wait', '--remain-after-exit', '-u', name, '-p', 'CPUWeight=100', '-p', 'KillMode=process', + '-p', 'KillSignal=INT', ] if cwd := kwargs.pop('cwd', None): @@ -140,9 +141,6 @@ def poll_rusage(name: str, deadline: float) -> tuple[RUsage, bool]: nsec = int(v) finally: - if timedout: - logger.debug('killing worker service') - subprocess.run(['systemctl', '--user', 'kill', '--kill-whom=main', '--signal=SIGINT', name]) logger.debug('stopping worker service') # stop whatever may be running (even from a previous batch) subprocess.run(['systemctl', '--user', 'stop', '--quiet', name]) diff --git a/lilac2/worker.py b/lilac2/worker.py index 95632ecd..c8b0030f 100644 --- a/lilac2/worker.py +++ b/lilac2/worker.py @@ -241,6 +241,12 @@ def main() -> None: # mod failed to load info = load_lilacinfo(Path('.')) handle_failure(e, repo, info, Path(input['logfile'])) + except KeyboardInterrupt: + logger.info('KeyboardInterrupt received') + r = { + 'status': 'failed', + 'msg': 'KeyboardInterrupt', + } finally: # say goodbye to all our children kill_child_processes() From cca0541cfa205712fb0a19fb5699d6f54fd2d334 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 21 Jun 2024 09:20:19 +0800 Subject: [PATCH 53/63] systemd: CPUUsageNSec may not be set with systemd v256 https://github.com/systemd/systemd/pull/33258 --- lilac2/systemd.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lilac2/systemd.py b/lilac2/systemd.py index c940c380..0691ddca 100644 --- a/lilac2/systemd.py +++ b/lilac2/systemd.py @@ -138,7 +138,11 @@ def poll_rusage(name: str, deadline: float) -> tuple[RUsage, bool]: for l in out.splitlines(): k, v = l.split('=', 1) if k == 'CPUUsageNSec': - nsec = int(v) + try: + nsec = int(v) + except ValueError: + # [not set] + pass finally: logger.debug('stopping worker service') From e97ab7c053153f265ad9b1eb8107db99ca772cdd Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 27 Jun 2024 14:46:37 +0800 Subject: [PATCH 54/63] add filter arg to tarfile.extract --- lilac2/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lilac2/api.py b/lilac2/api.py index f51d0e79..31bcc494 100644 --- a/lilac2/api.py +++ b/lilac2/api.py @@ -489,7 +489,7 @@ def _download_aur_pkgbuild(name: str) -> List[str]: if remain in SPECIAL_FILES + ('.AURINFO', '.SRCINFO', '.gitignore'): continue tarinfo.name = remain - tarf.extract(tarinfo) + tarf.extract(tarinfo, filter='tar') files.append(remain) return files @@ -604,7 +604,7 @@ def download_official_pkgbuild(name: str) -> list[str]: continue tarinfo.name = filename logger.debug('extract file %s.', filename) - tarf.extract(tarinfo) + tarf.extract(tarinfo, filter='tar') files.append(filename) return files From ba672803f604dc4d6497ad788378f33a2dfa6e84 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 27 Jun 2024 14:52:10 +0800 Subject: [PATCH 55/63] minor update docs --- docs/setup.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/setup.rst b/docs/setup.rst index 10c24f0b..f0423568 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -10,7 +10,7 @@ It's recommended to run lilac on full-fledged Arch Linux (or derived) system, no An easy way to install lilac and its dependencies is to install the ``lilac-git`` package from the `[archlinuxcn] repository `_ or AUR. -As a workaround, instead of ``devtools``, ``devtools-archlinuxcn`` from ``[archlinuxcn]`` should be used until `FS#64265 `_ and `FS#64698 `_ are resolved. +As a workaround, instead of ``devtools``, ``devtools-archlinuxcn`` from ``[archlinuxcn]`` should be used until `this --keep-unit issue `_ is resolved. .. code-block:: sh From 7c22bebba3fedcc49b3d9af0692958105a422db9 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 28 Jun 2024 19:15:37 +0800 Subject: [PATCH 56/63] db: fix build_reasons stored as text --- lilac | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lilac b/lilac index 74db6641..2c609767 100755 --- a/lilac +++ b/lilac @@ -12,7 +12,6 @@ from typing import List, Any, DefaultDict, Tuple from collections.abc import Set from pathlib import Path import graphlib -import json import datetime import threading from concurrent.futures import ( @@ -159,7 +158,7 @@ def packages_with_depends( pkgbase = pkg, index = idx, status = 'pending', - build_reasons = json.dumps(rs), + build_reasons = rs, ) s.add(p) db.build_updated(s) @@ -465,7 +464,7 @@ def build_it( cputime = cputime, memory = memory, msg = msg, - build_reasons = json.dumps(rs), + build_reasons = rs, ) s.add(p) db.mark_pkg_as(s, pkg, 'done') From ef508b9978caaf915b8f4cb9fe988a35aca82fe4 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 28 Jun 2024 19:32:06 +0800 Subject: [PATCH 57/63] record current maintainers when building --- README.md | 13 +++++++++++++ lilac | 2 ++ scripts/dbsetup.sql | 3 ++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8ac7d256..b4cf58c5 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,19 @@ Docs * [lilac.py API](https://lilac.readthedocs.io/en/latest/api.html) * [Setup and run your own](https://lilac.readthedocs.io/en/latest/) +Update +---- + +### 2024-06-28 + +if database is in use, run the following SQL to update: + +```sql + +alter table lilac.pkglog add column maintainers jsonb; +``` + + License ------- diff --git a/lilac b/lilac index 2c609767..3b579654 100755 --- a/lilac +++ b/lilac @@ -455,6 +455,7 @@ def build_it( cputime = memory = None rs = [r.to_dict() for r in build_reasons[pkg]] with db.get_session() as s: + maintainers = repo.lilacinfos[pkg].maintainers p = db.PkgLog( pkgbase = pkg, nv_version = newver, @@ -465,6 +466,7 @@ def build_it( memory = memory, msg = msg, build_reasons = rs, + maintainers = maintainers, ) s.add(p) db.mark_pkg_as(s, pkg, 'done') diff --git a/scripts/dbsetup.sql b/scripts/dbsetup.sql index 167e98f8..ad8a7da2 100644 --- a/scripts/dbsetup.sql +++ b/scripts/dbsetup.sql @@ -14,7 +14,8 @@ create table pkglog ( cputime int, memory bigint, msg text, - build_reasons jsonb + build_reasons jsonb, + maintainers jsonb ); create index pkglog_ts_idx on pkglog (ts); From b10eb9201741f62aee701d9871f7e131662f0047 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 28 Jun 2024 19:50:33 +0800 Subject: [PATCH 58/63] support specifying a name on cmdline building --- lilac | 6 +++++- lilac2/nomypy.py | 12 ++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/lilac b/lilac index 3b579654..0058632a 100755 --- a/lilac +++ b/lilac @@ -577,7 +577,11 @@ def main_may_raise( if pkgs_from_args: for p in pkgs_from_args: - build_reasons[p].append(BuildReason.Cmdline()) + if ':' in p: + p, runner = p.split(':', 1) + else: + runner = None + build_reasons[p].append(BuildReason.Cmdline(runner)) for p in need_rebuild_pkgrel: build_reasons[p].append(BuildReason.UpdatedPkgrel()) diff --git a/lilac2/nomypy.py b/lilac2/nomypy.py index 14809267..c230a640 100644 --- a/lilac2/nomypy.py +++ b/lilac2/nomypy.py @@ -1,6 +1,6 @@ # type: ignore -from typing import Union +from typing import Union, Optional from .typing import OnBuildEntry @@ -96,7 +96,15 @@ class FailedByDeps(BuildReason): def __init__(self, deps: tuple[str]) -> None: self.deps = deps -class Cmdline(BuildReason): pass +class Cmdline(BuildReason): + def __init__(self, runner: Optional[str]) -> None: + self.runner = runner + + def _extra_info(self) -> str: + if self.runner: + return repr(self.runner) + else: + return '' class OnBuild(BuildReason): def __init__(self, update_on_build: list[OnBuildEntry]) -> None: From b8ca9c4ce84a8e7da17c79e38319af8dc492a364 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 28 Jun 2024 20:09:34 +0800 Subject: [PATCH 59/63] useful.sql: build_reasons has been fixed --- scripts/useful.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/useful.sql b/scripts/useful.sql index 5f350511..47a4a7f5 100644 --- a/scripts/useful.sql +++ b/scripts/useful.sql @@ -1,10 +1,10 @@ -- some useful SQL commands (for PostgreSQL) -- show build log -select id, ts, pkgbase, nv_version, pkg_version, elapsed, result, cputime, case when elapsed = 0 then 0 else cputime * 100 / elapsed end as "cpu%", round(memory / 1073741824.0, 3) as "memory (GiB)", substring(msg for 20) as msg, build_reasons #>> '{}' as build_reasons from pkglog order by id desc; +select id, ts, pkgbase, nv_version, pkg_version, elapsed, result, cputime, case when elapsed = 0 then 0 else cputime * 100 / elapsed end as "cpu%", round(memory / 1073741824.0, 3) as "memory (GiB)", substring(msg for 20) as msg, build_reasons from pkglog order by id desc limit 10; -- show current build status and expected time -select index, c.pkgbase, updated_at, status, elapsed as last_time, c.build_reasons #>> '{}' as build_reasons from pkgcurrent as c left join lateral ( +select index, c.pkgbase, updated_at, status, elapsed as last_time, c.build_reasons from pkgcurrent as c left join lateral ( select elapsed from pkglog where pkgbase = c.pkgbase order by ts desc limit 1 ) as log on true order by c.index asc; From ce183795691798426a31211c5dc50cbe0d165f4a Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 28 Jun 2024 20:26:46 +0800 Subject: [PATCH 60/63] useful.sql: add maintainers to SQL output --- scripts/useful.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/useful.sql b/scripts/useful.sql index 47a4a7f5..83f1268c 100644 --- a/scripts/useful.sql +++ b/scripts/useful.sql @@ -1,7 +1,7 @@ -- some useful SQL commands (for PostgreSQL) -- show build log -select id, ts, pkgbase, nv_version, pkg_version, elapsed, result, cputime, case when elapsed = 0 then 0 else cputime * 100 / elapsed end as "cpu%", round(memory / 1073741824.0, 3) as "memory (GiB)", substring(msg for 20) as msg, build_reasons from pkglog order by id desc limit 10; +select id, ts, pkgbase, nv_version, pkg_version, elapsed, result, cputime, case when elapsed = 0 then 0 else cputime * 100 / elapsed end as "cpu%", round(memory / 1073741824.0, 3) as "memory (GiB)", substring(msg for 20) as msg, build_reasons, (select array_agg(github) from jsonb_to_recordset(maintainers) as m(github text)) as maintainers from pkglog order by id desc limit 10; -- show current build status and expected time select index, c.pkgbase, updated_at, status, elapsed as last_time, c.build_reasons from pkgcurrent as c left join lateral ( From 0a271c2d8201f14f9fc7a3c7eb3bbd3f6b5f962e Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 29 Jun 2024 23:43:22 +0800 Subject: [PATCH 61/63] use systemd's CPUUsageNSec and MemoryPeak properties if available --- lilac2/systemd.py | 91 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 68 insertions(+), 23 deletions(-) diff --git a/lilac2/systemd.py b/lilac2/systemd.py index 0691ddca..09c9db92 100644 --- a/lilac2/systemd.py +++ b/lilac2/systemd.py @@ -1,6 +1,6 @@ import os import subprocess -from typing import Generator, Any +from typing import Generator, Any, Optional import select import time import logging @@ -13,7 +13,7 @@ _available = None _check_lock = threading.Lock() -def available() -> bool: +def available() -> bool | dict[str, bool]: global _available with _check_lock: @@ -22,14 +22,61 @@ def available() -> bool: logger.debug('systemd availability: %s', _available) return _available -def _check_availability() -> bool: +def _cgroup_memory_usage(cgroup: str) -> int: + mem_file = f'/sys/fs/cgroup{cgroup}/memory.peak' + with open(mem_file) as f: + return int(f.read().rstrip()) + +def _cgroup_cpu_usage(cgroup: str) -> int: + cpu_file = f'/sys/fs/cgroup{cgroup}/cpu.stat' + with open(cpu_file) as f: + for l in f: + if l.startswith('usage_usec '): + return int(l.split()[1]) * 1000 + return 0 + +def _check_availability() -> bool | dict[str, bool]: if 'DBUS_SESSION_BUS_ADDRESS' not in os.environ: dbus = f'/run/user/{os.getuid()}/bus' if not os.path.exists(dbus): return False os.environ['DBUS_SESSION_BUS_ADDRESS'] = f'unix:path={dbus}' - p = subprocess.run(['systemd-run', '--quiet', '--user', '-u', 'lilac-check', 'true']) - return p.returncode == 0 + p = subprocess.run([ + 'systemd-run', '--quiet', '--user', + '--remain-after-exit', '-u', 'lilac-check', 'true', + ]) + if p.returncode != 0: + return False + + try: + ps: dict[str, Optional[int]] = { + 'CPUUsageNSec': None, + 'MemoryPeak': None, + } + _read_service_int_properties('lilac-check', ps) + + ret = {} + for k, v in ps.items(): + ret[k] = v is not None + + return ret + finally: + subprocess.run(['systemctl', '--user', 'stop', '--quiet', 'lilac-check']) + +def _read_service_int_properties(name: str, properties: dict[str, Optional[int]]) -> None: + cmd = [ + 'systemctl', '--user', 'show', f'{name}.service', + ] + [f'--property={k}' for k in properties] + + out = subprocess.check_output(cmd, text=True) + for l in out.splitlines(): + k, v = l.split('=', 1) + if k in properties: + try: + properties[k] = int(v) + except ValueError: + # [not set] + pass def start_cmd( name: str, cmd: Cmd, @@ -117,32 +164,30 @@ def poll_rusage(name: str, deadline: float) -> tuple[RUsage, bool]: logger.warning('%s.service already finished: %s', name, state) return RUsage(0, 0), False - mem_file = f'/sys/fs/cgroup{cgroup}/memory.peak' - + nsec = 0 mem_max = 0 + availability = available() + assert isinstance(availability, dict) for _ in _poll_cmd(pid): - with open(mem_file) as f: - mem_cur = int(f.read().rstrip()) - mem_max = max(mem_cur, mem_max) + if not availability['CPUUsageNSec']: + nsec = _cgroup_cpu_usage(cgroup) + if not availability['MemoryPeak']: + mem_max = _cgroup_memory_usage(cgroup) if time.time() > deadline: timedout = True break # systemd will remove the cgroup as soon as the process exits # instead of racing with systemd, we just ask it for the data - nsec = 0 - out = subprocess.check_output([ - 'systemctl', '--user', 'show', f'{name}.service', - '--property=CPUUsageNSec', - ], text=True) - for l in out.splitlines(): - k, v = l.split('=', 1) - if k == 'CPUUsageNSec': - try: - nsec = int(v) - except ValueError: - # [not set] - pass + ps: dict[str, Optional[int]] = { + 'CPUUsageNSec': None, + 'MemoryPeak': None, + } + _read_service_int_properties(name, ps) + if n := ps['CPUUsageNSec']: + nsec = n + if n := ps['MemoryPeak']: + mem_max = n finally: logger.debug('stopping worker service') From 8c70b425f80fc5bd20497dc31cbd534c03ccd694 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 29 Jun 2024 23:46:22 +0800 Subject: [PATCH 62/63] fix argument handling --- lilac | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lilac b/lilac index 0058632a..853461d1 100755 --- a/lilac +++ b/lilac @@ -532,9 +532,13 @@ def main_may_raise( # packages we care about care_pkgs: set[str] = set() - for pkg_to_build in pkgs_from_args: - care_pkgs.update(dep.pkgname for dep in DEPMAP[pkg_to_build]) - care_pkgs.add(pkg_to_build) + for p in pkgs_from_args: + if ':' in p: + pkg = p.split(':', 1)[0] + else: + pkg = p + care_pkgs.update(dep.pkgname for dep in DEPMAP[pkg]) + care_pkgs.add(pkg) # make sure they have nvdata care_pkgs.update(need_rebuild_failed) care_pkgs.update(need_rebuild_pkgrel) From e8b23affe934a85f97f46e3c3b7e42223c700e9a Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sun, 30 Jun 2024 11:46:09 +0800 Subject: [PATCH 63/63] fix systemd availability check when the command is running, resource properties are always available. --- lilac2/systemd.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/lilac2/systemd.py b/lilac2/systemd.py index 09c9db92..91257d65 100644 --- a/lilac2/systemd.py +++ b/lilac2/systemd.py @@ -49,17 +49,22 @@ def _check_availability() -> bool | dict[str, bool]: return False try: - ps: dict[str, Optional[int]] = { - 'CPUUsageNSec': None, - 'MemoryPeak': None, - } - _read_service_int_properties('lilac-check', ps) - - ret = {} - for k, v in ps.items(): - ret[k] = v is not None - - return ret + while True: + ps: dict[str, Optional[int]] = { + 'CPUUsageNSec': None, + 'MemoryPeak': None, + 'MainPID': None, + } + _read_service_int_properties('lilac-check', ps) + if ps['MainPID'] != 0: + time.sleep(0.01) + continue + + ret = {} + for k, v in ps.items(): + ret[k] = v is not None + + return ret finally: subprocess.run(['systemctl', '--user', 'stop', '--quiet', 'lilac-check'])