From 79b84aa2aa7844efd15205d34c73681f7d2a4627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 27 Sep 2023 10:51:57 +0200 Subject: [PATCH 1/4] use old nix-build command It seems that the outputs nix build --log-format raw are still different from the old nix-build --- nix_fast_build/__init__.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/nix_fast_build/__init__.py b/nix_fast_build/__init__.py index ba5e97a..15581e5 100644 --- a/nix_fast_build/__init__.py +++ b/nix_fast_build/__init__.py @@ -464,7 +464,7 @@ async def build( self, stack: AsyncExitStack, build_output: IO[str], opts: Options ) -> int: proc = await stack.enter_async_context( - nix_build(self.drv_path + "^*", build_output, opts) + nix_build(self.drv_path, build_output, opts) ) rc = 0 for _ in range(opts.retries + 1): @@ -564,16 +564,11 @@ async def get_context(self) -> AsyncIterator[T]: async def nix_build( installable: str, stderr: IO[Any] | None, opts: Options ) -> AsyncIterator[Process]: - args = nix_command( - [ - "build", - installable, - "--log-format", - "raw", - "--keep-going", - ] - + opts.options - ) + args = [ + "nix-build", + installable, + "--keep-going", + ] + opts.options args = maybe_remote(args, opts) logger.debug("run %s", shlex.join(args)) proc = await asyncio.create_subprocess_exec(*args, stderr=stderr) From 1fb7f8fd8c58496a2ff7cad663d4efbc87d1f899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 28 Sep 2023 12:32:23 +0200 Subject: [PATCH 2/4] fix case where exit status was not propagated correctly --- nix_fast_build/__init__.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/nix_fast_build/__init__.py b/nix_fast_build/__init__.py index 15581e5..f1ed6ff 100644 --- a/nix_fast_build/__init__.py +++ b/nix_fast_build/__init__.py @@ -819,23 +819,26 @@ async def run(stack: AsyncExitStack, opts: Options) -> int: return rc -async def async_main(args: list[str]) -> None: +async def async_main(args: list[str]) -> int: opts = await parse_args(args) if opts.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) - rc = 0 - async with AsyncExitStack() as stack: + stack = AsyncExitStack() + # using async wait here seems to make the return value skipped in the non-execptional case + try: if opts.remote_url: opts.flake_url = upload_sources(opts) - rc = await run(stack, opts) - sys.exit(rc) + return await run(stack, opts) + finally: + await stack.aclose() def main() -> None: try: - asyncio.run(async_main(sys.argv[1:])) - except KeyboardInterrupt: - logger.info("nix-fast-build was canceled by the user") + sys.exit(asyncio.run(async_main(sys.argv[1:]))) + except KeyboardInterrupt as e: + logger.info(f"nix-fast-build was canceled by the user ({e})") + sys.exit(1) From 777c69e4c4c5db7b233ae165d5276babe5a5fec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 28 Sep 2023 12:41:29 +0200 Subject: [PATCH 3/4] await eval process result in evaluation itself --- nix_fast_build/__init__.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/nix_fast_build/__init__.py b/nix_fast_build/__init__.py index f1ed6ff..19734ad 100644 --- a/nix_fast_build/__init__.py +++ b/nix_fast_build/__init__.py @@ -594,7 +594,7 @@ async def run_evaluation( build_queue: Queue[Job | StopTask], failures: list[Failure], opts: Options, -) -> None: +) -> int: assert eval_proc.stdout async for line in eval_proc.stdout: logger.debug(line.decode()) @@ -618,6 +618,7 @@ async def run_evaluation( die(f"nix-eval-jobs did not return a drvPath: {line.decode()}") outputs = job.get("outputs", {}) build_queue.put_nowait(Job(attr, drv_path, outputs)) + return await eval_proc.wait() async def run_builds( @@ -628,14 +629,14 @@ async def run_builds( download_queue: QueueWithContext[Build | StopTask], failures: list[Failure], opts: Options, -) -> None: +) -> int: drv_paths: set[Any] = set() while True: async with build_queue.get_context() as next_job: if isinstance(next_job, StopTask): logger.debug("finish build task") - return + return 0 job = next_job print(f" building {job.attr}") if job.drv_path in drv_paths: @@ -655,12 +656,12 @@ async def run_uploads( upload_queue: QueueWithContext[Build | StopTask], failures: list[Failure], opts: Options, -) -> None: +) -> int: while True: async with upload_queue.get_context() as build: if isinstance(build, StopTask): logger.debug("finish upload task") - return + return 0 rc = await build.upload(stack, opts) if rc != 0: failures.append(UploadFailure(build.attr, f"upload exited with {rc}")) @@ -671,12 +672,12 @@ async def run_downloads( download_queue: QueueWithContext[Build | StopTask], failures: list[Failure], opts: Options, -) -> None: +) -> int: while True: async with download_queue.get_context() as build: if isinstance(build, StopTask): logger.debug("finish download task") - return + return 0 rc = await build.download(stack, opts) if rc != 0: failures.append( @@ -688,7 +689,7 @@ async def report_progress( build_queue: QueueWithContext, upload_queue: QueueWithContext, download_queue: QueueWithContext, -) -> None: +) -> int: old_status = "" try: while True: @@ -701,7 +702,8 @@ async def report_progress( old_status = new_status await asyncio.sleep(0.5) except asyncio.CancelledError: - return + pass + return 0 async def run(stack: AsyncExitStack, opts: Options) -> int: @@ -771,10 +773,8 @@ async def run(stack: AsyncExitStack, opts: Options) -> int: name="progress", ) ) - logger.debug("Waiting for nix-eval to finish...") - await eval_proc.wait() logger.debug("Waiting for evaluation to finish...") - await evaluation + eval_rc = await evaluation logger.debug("Evaluation finished, waiting for builds to finish...") for _ in range(opts.max_jobs): @@ -805,7 +805,7 @@ async def run(stack: AsyncExitStack, opts: Options) -> int: f"{failure_type.__name__} for {failure.attr}: {failure.error_message}" ) rc = 1 - if eval_proc.returncode != 0 and eval_proc.returncode is not None: + if eval_rc != 0: logger.error(f"nix-eval-jobs exited with {eval_proc.returncode}") rc = 1 if ( From 05161807c439dd304c38f8594a45ffc011eafede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 28 Sep 2023 12:56:38 +0200 Subject: [PATCH 4/4] await progress runner result after cancelling --- nix_fast_build/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nix_fast_build/__init__.py b/nix_fast_build/__init__.py index 19734ad..7efdf84 100644 --- a/nix_fast_build/__init__.py +++ b/nix_fast_build/__init__.py @@ -794,6 +794,7 @@ async def run(stack: AsyncExitStack, opts: Options) -> int: if not opts.nom: logger.debug("Stopping progress reporter") tasks[-1].cancel() + await tasks[-1] for task in tasks: assert task.done(), f"Task {task.get_name()} is not done"