diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..50639698 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,38 @@ +name: run tests + +on: [push, pull_request] + +jobs: + tests: + runs-on: ubuntu-latest + + # Use the base-devel image of Arch Linux for building pyalpm + container: archlinux:base-devel + + strategy: + fail-fast: false + matrix: + python-version: + - "3.11" + - "3.12" + exclude: [] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Python deps + run: python -m pip install -U pytest pytest-asyncio nvchecker requests lxml PyYAML pyalpm structlog python_prctl + + - name: workaround pycurl wheel + run: | + sudo mkdir -p /etc/pki/tls/certs + sudo ln -s /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt + + - name: Run pytest + run: pytest diff --git a/.typos.toml b/.typos.toml index f86fa38c..b6bcf56a 100644 --- a/.typos.toml +++ b/.typos.toml @@ -1,2 +1,3 @@ [default.extend-identifiers] update_ons = "update_ons" +O_WRONLY = "O_WRONLY" diff --git a/README.md b/README.md index 8ac7d256..b4cf58c5 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,19 @@ Docs * [lilac.py API](https://lilac.readthedocs.io/en/latest/api.html) * [Setup and run your own](https://lilac.readthedocs.io/en/latest/) +Update +---- + +### 2024-06-28 + +if database is in use, run the following SQL to update: + +```sql + +alter table lilac.pkglog add column maintainers jsonb; +``` + + License ------- diff --git a/config.toml.sample b/config.toml.sample index a3b37a2c..9a8daae5 100644 --- a/config.toml.sample +++ b/config.toml.sample @@ -84,4 +84,7 @@ tmpfs = [ "/build/.cache/bazel" ] +# pacman.conf to use for repository databases +pacman_conf = "/etc/pacman.conf" + # vim: se ft=toml: diff --git a/docs/setup.rst b/docs/setup.rst index ea55fdec..f0423568 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -10,7 +10,7 @@ It's recommended to run lilac on full-fledged Arch Linux (or derived) system, no An easy way to install lilac and its dependencies is to install the ``lilac-git`` package from the `[archlinuxcn] repository `_ or AUR. -As a workaround, instead of ``devtools``, ``devtools-archlinuxcn`` from ``[archlinuxcn]`` should be used until `FS#64265 `_ and `FS#64698 `_ are resolved. +As a workaround, instead of ``devtools``, ``devtools-archlinuxcn`` from ``[archlinuxcn]`` should be used until `this --keep-unit issue `_ is resolved. .. code-block:: sh @@ -183,7 +183,7 @@ Setup the database tables (run as lilac): Edit ``/etc/sudoers`` like:: - Defaults env_keep += "PACKAGER MAKEFLAGS GNUPGHOME BUILDTOOL" + Defaults env_keep += "PACKAGER MAKEFLAGS GNUPGHOME BUILDTOOL LOGDEST" %pkg ALL= NOPASSWD: /usr/bin/build-cleaner, /usr/bin/extra-x86_64-build, /usr/bin/multilib-build diff --git a/lilac b/lilac index 15ee2a22..42201d45 100755 --- a/lilac +++ b/lilac @@ -12,7 +12,6 @@ from typing import List, Any, DefaultDict, Tuple from collections.abc import Set from pathlib import Path import graphlib -import json import datetime import threading from concurrent.futures import ( @@ -70,6 +69,7 @@ MYNAME = config['lilac']['name'] nvdata: dict[str, NvResults] = {} DEPMAP: dict[str, set[Dependency]] = {} +BUILD_DEPMAP: dict[str, set[Dependency]] = {} build_reasons: DefaultDict[str, list[BuildReason]] = defaultdict(list) logger = logging.getLogger(__name__) @@ -163,7 +163,7 @@ def packages_with_depends( pkgbase = pkg, index = idx, status = 'pending', - build_reasons = json.dumps(rs), + build_reasons = rs, ) s.add(p) db.build_updated(s) @@ -263,12 +263,18 @@ def start_build( limit = max_concurrency - len(futures), ) for pkg in pkgs: + if pkg not in nvdata: + # this can happen when cmdline packages are specified and + # a package is pulled in by OnBuild + logger.warning('%s not in nvdata, skipping', pkg) + buildsorter.done(pkg) + continue fu = executor.submit( build_it, pkg, repo, buildsorter, built, failed) futures[fu] = pkg - if not futures: - # no task is running: we're done + if not pkgs and not futures: + # no more packages and no task is running: we're done break done, pending = futures_wait(futures, return_when=FIRST_COMPLETED) @@ -312,7 +318,7 @@ def try_pick_some( if rs := build_reasons.get(pkg): if len(rs) == 1 and isinstance(rs[0], BuildReason.FailedByDeps): - ds = DEPMAP[pkg] + ds = BUILD_DEPMAP[pkg] if not all(d.resolve() for d in ds): buildsorter.done(pkg) if db.USE: @@ -370,7 +376,7 @@ def build_it( update_info = nvdata[pkg], bindmounts = repo.bindmounts, tmpfs = repo.tmpfs, - depends = DEPMAP.get(pkg, ()), + depends = BUILD_DEPMAP.get(pkg, ()), repo = REPO, myname = MYNAME, destdir = DESTDIR, @@ -454,6 +460,7 @@ def build_it( cputime = memory = None rs = [r.to_dict() for r in build_reasons[pkg]] with db.get_session() as s: + maintainers = repo.lilacinfos[pkg].maintainers p = db.PkgLog( pkgbase = pkg, nv_version = newver, @@ -463,7 +470,8 @@ def build_it( cputime = cputime, memory = memory, msg = msg, - build_reasons = json.dumps(rs), + build_reasons = rs, + maintainers = maintainers, ) s.add(p) db.mark_pkg_as(s, pkg, 'done') @@ -489,12 +497,13 @@ def setup_thread(): def main_may_raise( D: dict[str, Any], pkgs_from_args: List[str], logdir: Path, ) -> None: - global DEPMAP + global DEPMAP, BUILD_DEPMAP if get_git_branch() not in ['master', 'main']: raise Exception('repo not on master or main, aborting.') - pkgbuild.update_data(PACMAN_DB_DIR) + pacman_conf = config['misc'].get('pacman_conf') + pkgbuild.update_data(PACMAN_DB_DIR, pacman_conf) if dburl := config['lilac'].get('dburl'): import sqlalchemy @@ -510,13 +519,34 @@ def main_may_raise( failed = REPO.load_managed_lilac_and_report() depman = DependencyManager(REPO.repodir) - DEPMAP = get_dependency_map(depman, REPO.lilacinfos) + DEPMAP, BUILD_DEPMAP = get_dependency_map(depman, REPO.lilacinfos) + + failed_info = D.get('failed', {}) + + U = set(REPO.lilacinfos) + last_commit = D.get('last_commit', EMPTY_COMMIT) + changed = get_changed_packages(last_commit, 'HEAD') & U + + failed_prev = set(failed_info.keys()) + # no update from upstream, but build instructions have changed; rebuild + # failed ones + need_rebuild_failed = failed_prev & changed + # if pkgrel is updated, build a new release + need_rebuild_pkgrel = {x for x in changed + if pkgrel_changed(last_commit, 'HEAD', x)} # packages we care about care_pkgs: set[str] = set() - for pkg_to_build in pkgs_from_args: - care_pkgs.update(dep.pkgname for dep in DEPMAP[pkg_to_build]) - care_pkgs.add(pkg_to_build) + for p in pkgs_from_args: + if ':' in p: + pkg = p.split(':', 1)[0] + else: + pkg = p + care_pkgs.update(dep.pkgname for dep in DEPMAP[pkg]) + care_pkgs.add(pkg) + # make sure they have nvdata + care_pkgs.update(need_rebuild_failed) + care_pkgs.update(need_rebuild_pkgrel) proxy = config['nvchecker'].get('proxy') _nvdata, unknown, rebuild = packages_need_update( @@ -524,20 +554,7 @@ def main_may_raise( ) nvdata.update(_nvdata) # update to the global object - failed_info = D.get('failed', {}) - - if not pkgs_from_args: - U = set(REPO.lilacinfos) - last_commit = D.get('last_commit', EMPTY_COMMIT) - changed = get_changed_packages(last_commit, 'HEAD') & U - - failed_prev = set(failed_info.keys()) - # no update from upstream, but build instructions have changed; rebuild - # failed ones - need_rebuild_failed = failed_prev & changed - # if pkgrel is updated, build a new release - need_rebuild_pkgrel = {x for x in changed - if pkgrel_changed(last_commit, 'HEAD', x)} - unknown + need_rebuild_pkgrel -= unknown nv_changed = {} for p, vers in nvdata.items(): @@ -569,14 +586,19 @@ def main_may_raise( if pkgs_from_args: for p in pkgs_from_args: - build_reasons[p].append(BuildReason.Cmdline()) - else: - for p in need_rebuild_pkgrel: - build_reasons[p].append(BuildReason.UpdatedPkgrel()) + if ':' in p: + p, runner = p.split(':', 1) + else: + runner = None + build_reasons[p].append(BuildReason.Cmdline(runner)) - for p in need_rebuild_failed: - build_reasons[p].append(BuildReason.UpdatedFailed()) + for p in need_rebuild_pkgrel: + build_reasons[p].append(BuildReason.UpdatedPkgrel()) + for p in need_rebuild_failed: + build_reasons[p].append(BuildReason.UpdatedFailed()) + + if not pkgs_from_args: for p, i in failed_info.items(): # p might have been removed if p in REPO.lilacinfos and (deps := i['missing']): @@ -587,15 +609,17 @@ def main_may_raise( for i in info.update_on_build: if_this_then_those[i.pkgbase].add(p) more_pkgs = set() - count = 0 for p in build_reasons: if pkgs := if_this_then_those.get(p): more_pkgs.update(pkgs) - while len(more_pkgs) != count: # has new - count = len(more_pkgs) + while True: + add_to_more_pkgs = set() for p in more_pkgs: if pkgs := if_this_then_those.get(p): - more_pkgs.update(pkgs) + add_to_more_pkgs.update(pkgs) + if add_to_more_pkgs.issubset(more_pkgs): + break + more_pkgs.update(add_to_more_pkgs) for p in more_pkgs: update_on_build = REPO.lilacinfos[p].update_on_build build_reasons[p].append(BuildReason.OnBuild(update_on_build)) @@ -626,10 +650,9 @@ def main_may_raise( if x in failed_info: del failed_info[x] # cleanup removed package failed_info - if not pkgs_from_args: - for x in tuple(failed_info.keys()): - if x not in REPO.lilacinfos: - del failed_info[x] + for x in tuple(failed_info.keys()): + if x not in REPO.lilacinfos: + del failed_info[x] D['failed'] = failed_info if config['lilac']['rebuild_failed_pkgs']: diff --git a/lilac2/api.py b/lilac2/api.py index 2c255e2f..0bf09cf2 100644 --- a/lilac2/api.py +++ b/lilac2/api.py @@ -42,6 +42,9 @@ s.headers['User-Agent'] = UserAgent VCS_SUFFIXES = ('-git', '-hg', '-svn', '-bzr') +AUR_BLACKLIST = { + 'dnrops': "creates packages that install packages into the packager's system", +} def _unquote_item(s: str) -> Optional[str]: m = re.search(r'''[ \t'"]*([^ '"]+)[ \t'"]*''', s) @@ -190,19 +193,23 @@ def run_cmd(cmd: Cmd, **kwargs) -> str: else: return _run_cmd(cmd, **kwargs) -def get_pkgver_and_pkgrel( -) -> Tuple[Optional[str], Optional[PkgRel]]: - pkgrel = None +def get_pkgver_and_pkgrel() -> Tuple[Optional[str], Optional[PkgRel]]: + pkgrel: Optional[PkgRel] = None pkgver = None - with suppress(FileNotFoundError), open('PKGBUILD') as f: - for l in f: - if l.startswith('pkgrel='): - pkgrel = l.rstrip().split( - '=', 1)[-1].strip('\'"') - with suppress(ValueError, TypeError): - pkgrel = int(pkgrel) # type: ignore - elif l.startswith('pkgver='): - pkgver = l.rstrip().split('=', 1)[-1].strip('\'"') + cmd = 'source PKGBUILD && declare -p pkgver pkgrel || :' + output = run_protected(['/bin/bash', '-c', cmd], silent = True) + pattern = re.compile('declare -- pkg(ver|rel)="([^"]+)"') + for line in output.splitlines(): + m = pattern.fullmatch(line) + if m: + value = m.group(2) + if m.group(1) == "rel": + try: + pkgrel = int(value) + except (ValueError, TypeError): + pkgrel = value + else: + pkgver = value return pkgver, pkgrel @@ -236,7 +243,7 @@ def update_pkgver_and_pkgrel( def update_pkgrel( rel: Optional[PkgRel] = None, ) -> None: - with open('PKGBUILD') as f: + with open('PKGBUILD', errors='replace') as f: pkgbuild = f.read() def replacer(m): @@ -482,7 +489,7 @@ def _download_aur_pkgbuild(name: str) -> List[str]: if remain in SPECIAL_FILES + ('.AURINFO', '.SRCINFO', '.gitignore'): continue tarinfo.name = remain - tarf.extract(tarinfo) + tarf.extract(tarinfo, filter='tar') files.append(remain) return files @@ -511,13 +518,13 @@ def aur_pre_build( if name is None: name = os.path.basename(os.getcwd()) - if maintainers: - maintainer, last_packager = _get_aur_packager(name) - if last_packager == 'lilac': - who = maintainer - else: - who = last_packager + maintainer, last_packager = _get_aur_packager(name) + if last_packager == 'lilac': + who = maintainer + else: + who = last_packager + if maintainers: error = False if isinstance(maintainers, str): error = who != maintainers @@ -526,6 +533,9 @@ def aur_pre_build( if error: raise Exception('unexpected AUR package maintainer / packager', who) + if who and (msg := AUR_BLACKLIST.get(who)): + raise Exception('blacklisted AUR package maintainer / packager', who, msg) + pkgver, pkgrel = get_pkgver_and_pkgrel() _g.aur_pre_files = clean_directory() _g.aur_building_files = _download_aur_pkgbuild(name) @@ -567,24 +577,34 @@ def download_official_pkgbuild(name: str) -> list[str]: url = 'https://archlinux.org/packages/search/json/?name=' + name logger.info('download PKGBUILD for %s.', name) info = s.get(url).json() - pkgbase = [r['pkgbase'] for r in info['results'] if r['repo'] != 'testing'][0] + pkg = [r for r in info['results'] if not r['repo'].endswith('testing')][0] + pkgbase = pkg['pkgbase'] + epoch = pkg['epoch'] + pkgver = pkg['pkgver'] + pkgrel = pkg['pkgrel'] + if epoch: + tag = f'{epoch}-{pkgver}-{pkgrel}' + else: + tag = f'{pkgver}-{pkgrel}' - tarball_url = 'https://gitlab.archlinux.org/archlinux/packaging/packages/{0}/-/archive/main/{0}-main.tar.bz2'.format(pkgbase) + tarball_url = 'https://gitlab.archlinux.org/archlinux/packaging/packages/{0}/-/archive/{1}/{0}-{1}.tar.bz2'.format(pkgbase, tag) logger.debug('downloading Arch package tarball from: %s', tarball_url) tarball = s.get(tarball_url).content - path = f'{pkgbase}-main' + path = f'{pkgbase}-{tag}' files = [] with tarfile.open( - name=f"{pkgbase}-main.tar.bz2", fileobj=io.BytesIO(tarball) + name=f"{pkgbase}-{tag}.tar.bz2", fileobj=io.BytesIO(tarball) ) as tarf: for tarinfo in tarf: dirname, filename = os.path.split(tarinfo.name) if dirname != path: continue + if filename in ('.SRCINFO', '.gitignore', '.nvchecker.toml'): + continue tarinfo.name = filename logger.debug('extract file %s.', filename) - tarf.extract(tarinfo) + tarf.extract(tarinfo, filter='tar') files.append(filename) return files diff --git a/lilac2/lilacyaml.py b/lilac2/lilacyaml.py index 5100610e..1d6eda00 100644 --- a/lilac2/lilacyaml.py +++ b/lilac2/lilacyaml.py @@ -51,6 +51,13 @@ def load_lilac_yaml(dir: Path) -> dict[str, Any]: depends[i] = next(iter(entry.items())) else: depends[i] = entry, entry + makedepends = conf.get('repo_makedepends') + if makedepends: + for i, entry in enumerate(makedepends): + if isinstance(entry, dict): + makedepends[i] = next(iter(entry.items())) + else: + makedepends[i] = entry, entry for func in FUNCTIONS: name = conf.get(func) @@ -92,6 +99,7 @@ def load_lilacinfo(dir: Path) -> LilacInfo: update_on_build = [OnBuildEntry(**x) for x in yamlconf.get('update_on_build', [])], throttle_info = throttle_info, repo_depends = yamlconf.get('repo_depends', []), + repo_makedepends = yamlconf.get('repo_makedepends', []), time_limit_hours = yamlconf.get('time_limit_hours', 1), staging = yamlconf.get('staging', False), managed = yamlconf.get('managed', True), @@ -136,7 +144,8 @@ def parse_update_on( entry.setdefault(k, v) # fill our dbpath if not provided - if entry.get('source') == 'alpm': + source = entry.get('source') + if source == 'alpm' or source == 'alpmfiles': entry.setdefault('dbpath', str(PACMAN_DB_DIR)) ret_update.append(entry) diff --git a/lilac2/mediawiki2pkgbuild.py b/lilac2/mediawiki2pkgbuild.py index fbd83d8f..87fd04e9 100644 --- a/lilac2/mediawiki2pkgbuild.py +++ b/lilac2/mediawiki2pkgbuild.py @@ -40,20 +40,23 @@ def gen_pkgbuild( name: str, mwver: str, desc: str, - license: str, + license: str | list[str], s: requests.Session, ) -> str: major, minor = mwver.split('.') mwver_next = f'{major}.{int(minor)+1}' link = get_link(name, mwver, s) + if isinstance(license, str): + license = [license] + license_str = ' '.join(f"'{x}'" for x in license) vars = { 'name': name, 'name_lower': name.lower(), - 'version': datetime.datetime.utcnow().strftime('%Y%m%d'), + 'version': datetime.datetime.now(tz=datetime.UTC).strftime('%Y%m%d'), 'desc': desc[0].lower() + desc[1:], 'link': link, 'mwver_cur': mwver, 'mwver_next': mwver_next, - 'license': license, + 'license': license_str, } return template.format_map(vars) diff --git a/lilac2/nomypy.py b/lilac2/nomypy.py index 14809267..c230a640 100644 --- a/lilac2/nomypy.py +++ b/lilac2/nomypy.py @@ -1,6 +1,6 @@ # type: ignore -from typing import Union +from typing import Union, Optional from .typing import OnBuildEntry @@ -96,7 +96,15 @@ class FailedByDeps(BuildReason): def __init__(self, deps: tuple[str]) -> None: self.deps = deps -class Cmdline(BuildReason): pass +class Cmdline(BuildReason): + def __init__(self, runner: Optional[str]) -> None: + self.runner = runner + + def _extra_info(self) -> str: + if self.runner: + return repr(self.runner) + else: + return '' class OnBuild(BuildReason): def __init__(self, update_on_build: list[OnBuildEntry]) -> None: diff --git a/lilac2/packages.py b/lilac2/packages.py index 9a15ff74..45ae74c6 100644 --- a/lilac2/packages.py +++ b/lilac2/packages.py @@ -15,25 +15,44 @@ def get_dependency_map( depman: DependencyManager, lilacinfos: LilacInfos, -) -> Dict[str, Set[Dependency]]: +) -> Tuple[Dict[str, Set[Dependency]], Dict[str, Set[Dependency]]]: '''compute ordered, complete dependency relations between pkgbases (the directory names) This function does not make use of pkgname because they maybe the same for different pkgdir. Those are carried by Dependency and used elsewhere. + + The first returned dict has the complete set of dependencies of the given pkgbase, including + build-time dependencies of other dependencies. The second dict has only the dependnecies + required to be installed in the build chroot. For example, if A depends on B, and B makedepends + on C, then the first dict has "A: {B, C}" while the second dict has only "A: {B}". ''' map: DefaultDict[str, Set[Dependency]] = defaultdict(set) pkgdir_map: DefaultDict[str, Set[str]] = defaultdict(set) rmap: DefaultDict[str, Set[str]] = defaultdict(set) + # same as above maps, but contain only normal dependencies, not makedepends or checkdepends + norm_map: DefaultDict[str, Set[Dependency]] = defaultdict(set) + norm_pkgdir_map: DefaultDict[str, Set[str]] = defaultdict(set) + norm_rmap: DefaultDict[str, Set[str]] = defaultdict(set) + for pkgbase, info in lilacinfos.items(): - depends = info.repo_depends + for d in info.repo_depends: + d = depman.get(d) + + pkgdir_map[pkgbase].add(d.pkgdir.name) + rmap[d.pkgdir.name].add(pkgbase) + map[pkgbase].add(d) + + norm_pkgdir_map[pkgbase].add(d.pkgdir.name) + norm_rmap[d.pkgdir.name].add(pkgbase) + norm_map[pkgbase].add(d) - ds = [depman.get(d) for d in depends] - if ds: - for d in ds: - pkgdir_map[pkgbase].add(d.pkgdir.name) - rmap[d.pkgdir.name].add(pkgbase) - map[pkgbase].update(ds) + for d in info.repo_makedepends: + d = depman.get(d) + + pkgdir_map[pkgbase].add(d.pkgdir.name) + rmap[d.pkgdir.name].add(pkgbase) + map[pkgbase].add(d) dep_order = graphlib.TopologicalSorter(pkgdir_map).static_order() for pkgbase in dep_order: @@ -42,8 +61,22 @@ def get_dependency_map( dependers = rmap[pkgbase] for dd in dependers: map[dd].update(deps) + if pkgbase in norm_rmap: + deps = norm_map[pkgbase] + dependers = norm_rmap[pkgbase] + for dd in dependers: + norm_map[dd].update(deps) + + build_dep_map: DefaultDict[str, Set[Dependency]] = defaultdict(set) + for pkgbase, info in lilacinfos.items(): + build_deps = build_dep_map[pkgbase] + build_deps.update(norm_map[pkgbase]) + for d in info.repo_makedepends: + d = depman.get(d) + build_deps.add(d) + build_deps.update(norm_map[d.pkgdir.name]) - return map + return map, build_dep_map _DependencyTuple = namedtuple( '_DependencyTuple', 'pkgdir pkgname') @@ -70,8 +103,7 @@ def resolve(self) -> Optional[Path]: elif not pkgs: return None else: - ret = sorted( - pkgs, reverse=True, key=lambda x: x.stat().st_mtime)[0] + ret = max(pkgs, key=lambda x: x.stat().st_mtime) return ret class DependencyManager: @@ -108,7 +140,7 @@ def get_split_packages(pkg: Path) -> Set[Tuple[str, str]]: pkgfile = pkg / 'package.list' if pkgfile.exists(): with open(pkgfile) as f: - packages.update((pkgbase, x) for x in f.read().split()) + packages.update((pkgbase, l.rstrip()) for l in f if not l.startswith('#')) return packages found = False diff --git a/lilac2/pkgbuild.py b/lilac2/pkgbuild.py index 3c4518cb..63e4d344 100644 --- a/lilac2/pkgbuild.py +++ b/lilac2/pkgbuild.py @@ -5,7 +5,7 @@ import os import time import subprocess -from typing import Dict, List +from typing import Dict, List, Optional, Union from pathlib import Path from contextlib import suppress @@ -51,24 +51,28 @@ def _save_timed_dict( data_str = ''.join(f'{k} {v}\n' for k, v in data.items()) safe_overwrite(str(path), data_str, mode='w') -def update_pacmandb(dbpath: Path, *, quiet: bool = False) -> None: - if quiet: - kwargs = {'stdout': subprocess.DEVNULL} - else: - kwargs = {} - - for _ in range(3): - p = subprocess.run( # type: ignore # what a mess... - ['fakeroot', 'pacman', '-Sy', '--dbpath', dbpath], - **kwargs, - ) - if p.returncode == 0: - break - else: - p.check_returncode() - -def update_data(dbpath: Path, *, quiet: bool = False) -> None: - update_pacmandb(dbpath, quiet=quiet) +def update_pacmandb(dbpath: Path, pacman_conf: Optional[str] = None, + *, quiet: bool = False) -> None: + stdout = subprocess.DEVNULL if quiet else None + + for update_arg in ['-Sy', '-Fy']: + + cmd: List[Union[str, Path]] = [ + 'fakeroot', 'pacman', update_arg, '--dbpath', dbpath, + ] + if pacman_conf is not None: + cmd += ['--config', pacman_conf] + + for _ in range(3): + p = subprocess.run(cmd, stdout = stdout) + if p.returncode == 0: + break + else: + p.check_returncode() + +def update_data(dbpath: Path, pacman_conf: Optional[str], + *, quiet: bool = False) -> None: + update_pacmandb(dbpath, pacman_conf, quiet=quiet) now = int(time.time()) deadline = now - 90 * 86400 diff --git a/lilac2/py.typed b/lilac2/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/lilac2/repo.py b/lilac2/repo.py index 45e1bdc9..d2f3c622 100644 --- a/lilac2/repo.py +++ b/lilac2/repo.py @@ -333,14 +333,14 @@ def load_managed_lilac_and_report(self) -> dict[str, tuple[str, ...]]: self.lilacinfos, errors = lilacyaml.load_managed_lilacinfos(self.repodir) failed: dict[str, tuple[str, ...]] = {p: () for p in errors} for name, exc_info in errors.items(): - logger.error('error while loading lilac.py for %s', name, exc_info=exc_info) + logger.error('error while loading lilac.yaml for %s', name, exc_info=exc_info) exc = exc_info[1] if not isinstance(exc, Exception): raise self.send_error_report(name, exc=exc, subject=_('Loading lilac.py for package %s face error')) build_logger_old.error('%s failed', name) - build_logger.exception('lilac.py error', pkgbase = name, exc_info=exc_info) + build_logger.exception('lilac.yaml error', pkgbase = name, exc_info=exc_info) return failed diff --git a/lilac2/systemd.py b/lilac2/systemd.py index b6bfab6b..91257d65 100644 --- a/lilac2/systemd.py +++ b/lilac2/systemd.py @@ -1,6 +1,6 @@ import os import subprocess -from typing import Generator, Any +from typing import Generator, Any, Optional import select import time import logging @@ -13,7 +13,7 @@ _available = None _check_lock = threading.Lock() -def available() -> bool: +def available() -> bool | dict[str, bool]: global _available with _check_lock: @@ -22,14 +22,66 @@ def available() -> bool: logger.debug('systemd availability: %s', _available) return _available -def _check_availability() -> bool: +def _cgroup_memory_usage(cgroup: str) -> int: + mem_file = f'/sys/fs/cgroup{cgroup}/memory.peak' + with open(mem_file) as f: + return int(f.read().rstrip()) + +def _cgroup_cpu_usage(cgroup: str) -> int: + cpu_file = f'/sys/fs/cgroup{cgroup}/cpu.stat' + with open(cpu_file) as f: + for l in f: + if l.startswith('usage_usec '): + return int(l.split()[1]) * 1000 + return 0 + +def _check_availability() -> bool | dict[str, bool]: if 'DBUS_SESSION_BUS_ADDRESS' not in os.environ: dbus = f'/run/user/{os.getuid()}/bus' if not os.path.exists(dbus): return False os.environ['DBUS_SESSION_BUS_ADDRESS'] = f'unix:path={dbus}' - p = subprocess.run(['systemd-run', '--quiet', '--user', '-u', 'lilac-check', 'true']) - return p.returncode == 0 + p = subprocess.run([ + 'systemd-run', '--quiet', '--user', + '--remain-after-exit', '-u', 'lilac-check', 'true', + ]) + if p.returncode != 0: + return False + + try: + while True: + ps: dict[str, Optional[int]] = { + 'CPUUsageNSec': None, + 'MemoryPeak': None, + 'MainPID': None, + } + _read_service_int_properties('lilac-check', ps) + if ps['MainPID'] != 0: + time.sleep(0.01) + continue + + ret = {} + for k, v in ps.items(): + ret[k] = v is not None + + return ret + finally: + subprocess.run(['systemctl', '--user', 'stop', '--quiet', 'lilac-check']) + +def _read_service_int_properties(name: str, properties: dict[str, Optional[int]]) -> None: + cmd = [ + 'systemctl', '--user', 'show', f'{name}.service', + ] + [f'--property={k}' for k in properties] + + out = subprocess.check_output(cmd, text=True) + for l in out.splitlines(): + k, v = l.split('=', 1) + if k in properties: + try: + properties[k] = int(v) + except ValueError: + # [not set] + pass def start_cmd( name: str, cmd: Cmd, @@ -41,7 +93,8 @@ def start_cmd( cmd_s: Cmd = [ 'systemd-run', '--pipe', '--quiet', '--user', '--wait', '--remain-after-exit', '-u', name, - '-p', 'CPUWeight=100', + '-p', 'CPUWeight=100', '-p', 'KillMode=process', + '-p', 'KillSignal=INT', ] if cwd := kwargs.pop('cwd', None): @@ -79,6 +132,7 @@ def _poll_cmd(pid: int) -> Generator[None, None, None]: except OSError as e: if e.errno == 22: return + raise poll = select.poll() poll.register(pidfd, select.POLLIN) @@ -98,6 +152,7 @@ def poll_rusage(name: str, deadline: float) -> tuple[RUsage, bool]: done_state = ['exited', 'failed'] try: + cgroup = '' time_start = time.monotonic() while True: pid, cgroup, state = _get_service_info(name) @@ -114,33 +169,32 @@ def poll_rusage(name: str, deadline: float) -> tuple[RUsage, bool]: logger.warning('%s.service already finished: %s', name, state) return RUsage(0, 0), False - mem_file = f'/sys/fs/cgroup{cgroup}/memory.peak' - + nsec = 0 mem_max = 0 + availability = available() + assert isinstance(availability, dict) for _ in _poll_cmd(pid): - with open(mem_file) as f: - mem_cur = int(f.read().rstrip()) - mem_max = max(mem_cur, mem_max) + if not availability['CPUUsageNSec']: + nsec = _cgroup_cpu_usage(cgroup) + if not availability['MemoryPeak']: + mem_max = _cgroup_memory_usage(cgroup) if time.time() > deadline: timedout = True break # systemd will remove the cgroup as soon as the process exits # instead of racing with systemd, we just ask it for the data - nsec = 0 - out = subprocess.check_output([ - 'systemctl', '--user', 'show', f'{name}.service', - '--property=CPUUsageNSec', - ], text=True) - for l in out.splitlines(): - k, v = l.split('=', 1) - if k == 'CPUUsageNSec': - nsec = int(v) + ps: dict[str, Optional[int]] = { + 'CPUUsageNSec': None, + 'MemoryPeak': None, + } + _read_service_int_properties(name, ps) + if n := ps['CPUUsageNSec']: + nsec = n + if n := ps['MemoryPeak']: + mem_max = n finally: - if timedout: - logger.debug('killing worker service') - subprocess.run(['systemctl', '--user', 'kill', '--signal=SIGKILL', name]) logger.debug('stopping worker service') # stop whatever may be running (even from a previous batch) subprocess.run(['systemctl', '--user', 'stop', '--quiet', name]) diff --git a/lilac2/typing.py b/lilac2/typing.py index 96bb4eac..bbfa92a8 100644 --- a/lilac2/typing.py +++ b/lilac2/typing.py @@ -35,6 +35,7 @@ class LilacInfo: update_on_build: list[OnBuildEntry] throttle_info: dict[int, datetime.timedelta] repo_depends: list[tuple[str, str]] + repo_makedepends: list[tuple[str, str]] time_limit_hours: float staging: bool managed: bool diff --git a/lilac2/vendor/nicelogger.py b/lilac2/vendor/nicelogger.py index 326728a5..1675e642 100644 --- a/lilac2/vendor/nicelogger.py +++ b/lilac2/vendor/nicelogger.py @@ -57,6 +57,7 @@ def format(self, record): 'filename', 'exc_info', 'exc_text', 'created', 'funcName', 'processName', 'process', 'msecs', 'relativeCreated', 'thread', 'threadName', 'name', 'levelno', 'msg', 'pathname', 'stack_info', + 'taskName', }) if record.exc_info: diff --git a/lilac2/worker.py b/lilac2/worker.py index 188b7213..1d5cf683 100644 --- a/lilac2/worker.py +++ b/lilac2/worker.py @@ -241,6 +241,12 @@ def main() -> None: # mod failed to load info = load_lilacinfo(Path('.')) handle_failure(e, repo, info, Path(input['logfile'])) + except KeyboardInterrupt: + logger.info('KeyboardInterrupt received') + r = { + 'status': 'failed', + 'msg': 'KeyboardInterrupt', + } finally: # say goodbye to all our children kill_child_processes() diff --git a/nvchecker_source/README.rst b/nvchecker_source/README.rst index 0271b908..a6ec7d2d 100644 --- a/nvchecker_source/README.rst +++ b/nvchecker_source/README.rst @@ -16,3 +16,20 @@ use_max_tag This source supports `list options`_ when ``use_max_tag`` is set. .. _list options: https://github.com/lilydjwg/nvchecker#list-options + +R packages from CRAN and Bioconductor +------------------------------------- +:: + + source = "rpkgs" + +Check versions from CRAN and Bioconductor. This source is optimized for checking large amounts of packages at once. If you want to check only a few, the ``cran`` source is better for CRAN packages. + +pkgname + Name of the R package. + +repo + The repo of the package. Possible values are ``cran``, ``bioc``, ``bioc-data-annotation``, ``bioc-data-experiment`` and ``bioc-workflows``. + +md5 + If set to ``true``, a ``#`` character and the md5sum of the source archive is appended to the version. Defaults to ``false``. diff --git a/nvchecker_source/rpkgs.py b/nvchecker_source/rpkgs.py new file mode 100644 index 00000000..22ac830b --- /dev/null +++ b/nvchecker_source/rpkgs.py @@ -0,0 +1,56 @@ +from typing import Dict, Tuple +from zlib import decompress + +from nvchecker.api import GetVersionError, session + +BIOC_TEMPLATE = 'https://bioconductor.org/packages/release/%s/src/contrib/PACKAGES.gz' + +URL_MAP = { + 'cran': 'https://cran.r-project.org/src/contrib/PACKAGES.gz', + 'bioc': BIOC_TEMPLATE % 'bioc', + 'bioc-data-annotation': BIOC_TEMPLATE % 'data/annotation', + 'bioc-data-experiment': BIOC_TEMPLATE % 'data/experiment', + 'bioc-workflows': BIOC_TEMPLATE % 'workflows', +} + +PKG_FIELD = b'Package: ' +VER_FIELD = b'Version: ' +MD5_FIELD = b'MD5sum: ' + +PKG_FLEN = len(PKG_FIELD) +VER_FLEN = len(VER_FIELD) +MD5_FLEN = len(MD5_FIELD) + +async def get_versions(repo: str) -> Dict[str, Tuple[str, str]]: + url = URL_MAP.get(repo) + if url is None: + raise GetVersionError('Unknown repo', repo = repo) + res = await session.get(url) + data = decompress(res.body, wbits = 31) + + result = {} + for section in data.split(b'\n\n'): + pkg = ver = md5 = None + for line in section.split(b'\n'): + if line.startswith(PKG_FIELD): + pkg = line[PKG_FLEN:].decode('utf8') + elif line.startswith(VER_FIELD): + ver = line[VER_FLEN:].decode('utf8') + elif line.startswith(MD5_FIELD): + md5 = line[MD5_FLEN:].decode('utf8') + if pkg is None or ver is None or md5 is None: + raise GetVersionError('Invalid package data', pkg = pkg, ver = ver, md5 = md5) + result[pkg] = (ver, md5) + + return result + +async def get_version(name, conf, *, cache, **kwargs): + pkgname = conf.get('pkgname', name) + repo = conf['repo'] + versions = await cache.get(repo, get_versions) + data = versions.get(pkgname) + if data is None: + raise GetVersionError(f'Package {pkgname} not found in repo {repo}') + add_md5 = conf.get('md5', False) + ver, md5 = data + return f'{ver}#{md5}' if add_md5 else ver diff --git a/schema-docs/lilac-yaml-schema.yaml b/schema-docs/lilac-yaml-schema.yaml index 7d17954f..3ab36b59 100644 --- a/schema-docs/lilac-yaml-schema.yaml +++ b/schema-docs/lilac-yaml-schema.yaml @@ -36,7 +36,20 @@ properties: description: Time limit in hours. The build will be aborted if it doesn't finish in time. Default is one hour. type: number repo_depends: - description: Packages in the repo to be built and installed before building the current package. + description: Packages in the repo that are direct dependencies of the current package. + type: array + items: + anyOf: + - type: string + description: Package (directory) name + - type: object + description: Package base (directory) as key and package name as value + minProperties: 1 + maxProperties: 1 + additionalProperties: + type: string + repo_makedepends: + description: Packages in the repo that are in makedepends or checkdepends of the current package. type: array items: anyOf: diff --git a/scripts/at-maintainer b/scripts/at-maintainer index 943b9b15..61457696 100755 --- a/scripts/at-maintainer +++ b/scripts/at-maintainer @@ -6,7 +6,7 @@ import re from lilac2.lilacyaml import iter_pkgdir, load_lilac_yaml -REPOPATH = pathlib.Path('/ldata/src/archgitrepo/archlinuxcn') +REPOPATH = pathlib.Path('/ssddata/src/archgitrepo/archlinuxcn') PkgPattern = re.compile(r'[\w.+-]+') diff --git a/scripts/dbsetup.sql b/scripts/dbsetup.sql index 167e98f8..ad8a7da2 100644 --- a/scripts/dbsetup.sql +++ b/scripts/dbsetup.sql @@ -14,7 +14,8 @@ create table pkglog ( cputime int, memory bigint, msg text, - build_reasons jsonb + build_reasons jsonb, + maintainers jsonb ); create index pkglog_ts_idx on pkglog (ts); diff --git a/scripts/tailf-build-log b/scripts/tailf-build-log index 24b49f21..2fc2859a 100755 --- a/scripts/tailf-build-log +++ b/scripts/tailf-build-log @@ -85,7 +85,7 @@ def pretty_print(log): fmt = FMT[result] out = c(7) + fmt % args + FMT['_rusage'] % args if result == 'failed': - out += f'{c(8)}{log["msg"]}\n' + out += f'{c(8)}{log["msg"][:1000]}\n' sys.stdout.write(out) def iter_pkglog(): diff --git a/scripts/useful.sql b/scripts/useful.sql index 5f350511..83f1268c 100644 --- a/scripts/useful.sql +++ b/scripts/useful.sql @@ -1,10 +1,10 @@ -- some useful SQL commands (for PostgreSQL) -- show build log -select id, ts, pkgbase, nv_version, pkg_version, elapsed, result, cputime, case when elapsed = 0 then 0 else cputime * 100 / elapsed end as "cpu%", round(memory / 1073741824.0, 3) as "memory (GiB)", substring(msg for 20) as msg, build_reasons #>> '{}' as build_reasons from pkglog order by id desc; +select id, ts, pkgbase, nv_version, pkg_version, elapsed, result, cputime, case when elapsed = 0 then 0 else cputime * 100 / elapsed end as "cpu%", round(memory / 1073741824.0, 3) as "memory (GiB)", substring(msg for 20) as msg, build_reasons, (select array_agg(github) from jsonb_to_recordset(maintainers) as m(github text)) as maintainers from pkglog order by id desc limit 10; -- show current build status and expected time -select index, c.pkgbase, updated_at, status, elapsed as last_time, c.build_reasons #>> '{}' as build_reasons from pkgcurrent as c left join lateral ( +select index, c.pkgbase, updated_at, status, elapsed as last_time, c.build_reasons from pkgcurrent as c left join lateral ( select elapsed from pkglog where pkgbase = c.pkgbase order by ts desc limit 1 ) as log on true order by c.index asc; diff --git a/setup.py b/setup.py index bdbbbc9c..c3b2de5e 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ description = 'The build bot for archlinuxcn', author = 'lilydjwg', author_email = 'lilydjwg@gmail.com', - python_requires = '>=3.7.0', + python_requires = '>=3.10.0', url = 'https://github.com/archlinuxcn/lilac', zip_safe = False, packages = find_packages(exclude=('tests',)) + ['nvchecker_source'], @@ -25,7 +25,8 @@ classifiers = [ 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ], ) diff --git a/tests/test_dependency_resolution.py b/tests/test_dependency_resolution.py new file mode 100644 index 00000000..44333e17 --- /dev/null +++ b/tests/test_dependency_resolution.py @@ -0,0 +1,41 @@ +from collections import namedtuple +from pathlib import Path + +from lilac2.packages import DependencyManager, get_dependency_map + +def test_dependency_map(): + depman = DependencyManager(Path('.')) + Info = namedtuple('Info', ['repo_depends', 'repo_makedepends']) + lilacinfos = { + 'A': Info(['B'], ['C']), + 'B': Info(['D'], ['C']), + 'C': Info([], ['E']), + 'D': Info([], []), + 'E': Info(['D'], []), + 'F': Info([], ['C', 'D']), + 'G': Info([], ['F']), + } + expected_all = { + 'A': { 'B', 'C', 'D', 'E' }, + 'B': { 'C', 'D', 'E' }, + 'C': { 'D', 'E' }, + 'D': set(), + 'E': { 'D' }, + 'F': { 'C', 'D', 'E' }, + 'G': { 'C', 'D', 'E', 'F' }, + } + expected_build = { + 'A': { 'B', 'C', 'D' }, + 'B': { 'C', 'D' }, + 'C': { 'D', 'E' }, + 'D': set(), + 'E': { 'D' }, + 'F': { 'C', 'D' }, + 'G': { 'F' }, + } + + res_all, res_build = get_dependency_map(depman, lilacinfos) + def parse_map(m): + return { key: { val.pkgdir.name for val in s } for key, s in m.items() } + assert parse_map(res_all) == expected_all + assert parse_map(res_build) == expected_build diff --git a/tests/test_rpkgs.py b/tests/test_rpkgs.py new file mode 100644 index 00000000..543b3bee --- /dev/null +++ b/tests/test_rpkgs.py @@ -0,0 +1,76 @@ +import asyncio + +import pytest +import pytest_asyncio + +pytestmark = pytest.mark.asyncio(scope='session') + +from nvchecker import core, __main__ as main +from nvchecker.util import Entries, RichResult, RawResult + +async def run(entries: Entries) -> RichResult: + task_sem = asyncio.Semaphore(20) + result_q: asyncio.Queue[RawResult] = asyncio.Queue() + keymanager = core.KeyManager(None) + + dispatcher = core.setup_httpclient() + entry_waiter = core.EntryWaiter() + futures = dispatcher.dispatch( + entries, task_sem, result_q, + keymanager, entry_waiter, 1, {}, + ) + + oldvers: RichResult = {} + result_coro = core.process_result(oldvers, result_q, entry_waiter) + runner_coro = core.run_tasks(futures) + + vers, _has_failures = await main.run(result_coro, runner_coro) + return vers + +@pytest_asyncio.fixture(scope='session') +async def get_version(): + async def __call__(name, config): + entries = {name: config} + newvers = await run(entries) + if r := newvers.get(name): + return r.version + + return __call__ + + +async def test_cran(get_version): + assert await get_version('xml2', { + 'source': 'rpkgs', + 'pkgname': 'xml2', + 'repo': 'cran', + 'md5': True, + }) == '1.3.6#fc6679028dca1f3047c8c745fb724524' + +async def test_bioc(get_version): + assert await get_version('BiocVersion', { + 'source': 'rpkgs', + 'pkgname': 'BiocVersion', + 'repo': 'bioc', + }) == '3.19.1' + +async def test_bioc_data_annotation(get_version): + assert await get_version('GO.db', { + 'source': 'rpkgs', + 'pkgname': 'GO.db', + 'repo': 'bioc-data-annotation', + }) == '3.19.1' + +async def test_bioc_data_experiment(get_version): + assert await get_version('ALL', { + 'source': 'rpkgs', + 'pkgname': 'ALL', + 'repo': 'bioc-data-experiment', + }) == '1.46.0' + +async def test_bioc_workflows(get_version): + assert await get_version('liftOver', { + 'source': 'rpkgs', + 'pkgname': 'liftOver', + 'repo': 'bioc-workflows', + 'md5': True, + }) == '1.28.0#336a9b7f29647ba8b26eb4dc139d755c'