dorebuild

#!/usr/bin/python
import argparse
import asyncio
import colorlog
from collections import defaultdict
import logging
import pyalpm
import signal
import sys

parser = argparse.ArgumentParser(description='A simple rebuild scheduler')
parser.add_argument('-d', '--db', nargs='*', default=["core", "extra", "community", "multilib"],
                    help='Pacman sync db to consider. Defaulting to all stable dbs.')
parser.add_argument('-m', '--mock', nargs='?', default=None,
                    help='Mock build, do not actually run builds. set to a threshold (0-1, chance to succeed).')
parser.add_argument('-r', '--retry', nargs='?', default=5,
                    help='Maximum retry on failure, set to 0 to disable retry. Default: 5')
parser.add_argument('-l', '--log', nargs='?', default="dorebuild.log",
                    help="Log filename. Default: dorebuild.log")
parser.add_argument('-c', '--command', nargs='?', default="rebuild-one",
                    help='Build command to use. Default: rebuild-one')
parser.add_argument('-j', '--jobs', nargs='?', default="10",
                    help='Concurrent jobs limit. Default: 10')
parser.add_argument('comment', nargs=1, help='Commit message')
parser.add_argument('package', nargs='+', help='Packages to rebuild')
args = parser.parse_args()

logger = colorlog.getLogger()
logger.setLevel(colorlog.DEBUG)

formatter = colorlog.ColoredFormatter()
handler = colorlog.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)

if args.log:
    fh = logging.FileHandler(args.log)
    fh.setFormatter(formatter)
    logger.addHandler(fh)

package_db = defaultdict(dict)
handle = pyalpm.Handle(".", "/var/lib/pacman")

for db in args.db:
    db_handle = handle.register_syncdb(db, 0)
    for package in db_handle.search(""):
        for field in ("depends", "makedepends", "checkdepends", "arch"):
            package_db[package.name][field] = getattr(package, field)

for pkg in package_db:
    package_db[pkg]["depends"] = {dep.split("=")[0].split(">")[0].split("<")[0] for dep in package_db[pkg]["depends"]}
    package_db[pkg]["makedepends"] = {dep.split("=")[0].split(">")[0].split("<")[0] for dep in package_db[pkg]["makedepends"]}
    package_db[pkg]["checkdepends"] = {dep.split("=")[0].split(">")[0].split("<")[0] for dep in package_db[pkg]["checkdepends"]}


async def main():
    ongoing_pkgs = set()
    loop = asyncio.get_event_loop()
    main.broken = False
    main.next_try = []

    async def status_report():
        while True:
            await asyncio.sleep(60)
            logger.debug(f"ongoing_pkgs: {ongoing_pkgs}")

    loop.create_task(status_report())

    async def run(*cmd):
        _BUFFER_SIZE = 2 * 2 ** 20  # 2 MiB buffer!
        if args.mock:
            import random
            await asyncio.sleep(random.random() * 3)
            return random.random() > float(args.mock)

        try:
            proc = await asyncio.create_subprocess_exec(
                *cmd,
                stdin=asyncio.subprocess.DEVNULL,
                stdout=asyncio.subprocess.PIPE,
                stderr=asyncio.subprocess.STDOUT,
                limit=_BUFFER_SIZE)
            while True:
                try:
                    line = await proc.stdout.readline()
                except ValueError:
                    # Underlying asyncio.exceptions.LimitOverrunError
                    line = await proc.stdout.read(_BUFFER_SIZE)
                if line:
                    logger.debug(f'{cmd[-1]}: ' + line.decode("utf-8", "ignore").rstrip('\n'))
                else:
                    break
            return await proc.wait()
        finally:
            try:
                proc.terminate()
            except:
                pass

    async def rebuild_single(pkg):
        if not main.broken:
            logger.debug(f"Starting rebuild of {pkg}")
            retry_count = 0
            while ret := await run(args.command, args.comment[0], pkg):
                if int(args.retry) and retry_count < int(args.retry):
                    logger.error(f"Failed rebuild of {pkg}! (ret: {ret}) retry: {retry_count}")
                    retry_count += 1
                else:
                    logger.error(f"Failed rebuild of {pkg}! (ret: {ret})")
                    if main.broken is False:
                        main.broken = [pkg]
                    else:
                        main.broken.append(pkg)
                    main.next_try.append(pkg)
                    break
            else:
                if retry_count:
                    logger.info(f"Finished rebuild of {pkg} retried: {retry_count}")
                else:
                    logger.info(f"Finished rebuild of {pkg}")
        else:
            main.next_try.append(pkg)
        ongoing_pkgs.remove(pkg)

    def handler(*args, **kwargs):
        logger.error("Received SIGINT, exiting...")
        logger.warning("Ongoing packages: " + ", ".join(ongoing_pkgs))
        sys.exit(1)

    signal.signal(signal.SIGINT, handler)

    for pkg in args.package:
        pending = True
        while pending:
            for _pkg in ongoing_pkgs:
                pkg_ = pkg
                if pkg_.endswith(":nocheck"):
                    pkg_ = pkg_[:-8]
                if _pkg.endswith(":nocheck"):
                    _pkg = _pkg[:-8]

                # New package, no enough info to check dependencies
                if pkg_ not in package_db:
                    break
                if _pkg not in package_db:
                    break

                if _pkg in package_db[pkg_]["depends"] | package_db[pkg_]["makedepends"] | package_db[pkg_]["checkdepends"] or \
                   pkg_ in package_db[_pkg]["depends"] | package_db[_pkg]["makedepends"] | package_db[_pkg]["checkdepends"] or \
                   _pkg == pkg_:
                    break
            else:
                pending = False

                while len(ongoing_pkgs) > int(args.jobs):
                    await asyncio.sleep(0.1)

                ongoing_pkgs.add(pkg)
                loop.create_task(rebuild_single(pkg))

            await asyncio.sleep(0.01)

    while ongoing_pkgs:
        await asyncio.sleep(0.1)

    if main.broken:
        logger.warning("Broken: " + " ".join(main.broken))
        logger.warning("Next try: " + " ".join(main.next_try))

if __name__ == "__main__":
    asyncio.run(main())