From 8b463f19d8306d4cd2b985482d9f89c32f7b7bd9 Mon Sep 17 00:00:00 2001 From: Hu Yueh-Wei Date: Tue, 7 Jan 2025 11:13:14 +0800 Subject: [PATCH 1/4] feat: add support for local dependency in extension --- .vscode/launch.json | 6 +- core/src/ten_manager/src/cmd/cmd_install.rs | 40 +++++-- .../ten_manager/src/dep_and_candidate/mod.rs | 70 +++++++++-- core/src/ten_manager/src/install/mod.rs | 46 +++++--- .../dependency_resolve/test_case.py | 1 + .../invalid_package_type/test_case.py | 1 + .../no_such_package/test_case.py | 1 + .../package_version_not_found/test_case.py | 1 + .../ten_manager/install/mock_app/test_case.py | 2 + .../install/mock_extension/test_case.py | 1 + tests/ten_manager/install_all/BUILD.gn | 1 + .../install_a_before_install_all/test_case.py | 2 + .../test_case.py | 8 +- .../test_case.py | 1 + .../install_breaking_lock/test_case.py | 1 + .../test_case.py | 1 + .../test_case.py | 1 + .../test_case.py | 1 + .../local_dependency_link/test_case.py | 1 + .../test_case.py | 3 +- .../BUILD.gn | 21 ++++ .../__init__.py | 0 .../test_app/expected.json | 12 ++ .../test_app/local_c/manifest.json | 6 + .../test_app/manifest.json | 12 ++ .../extension/local_d/manifest.json | 10 ++ .../test_case.py | 109 ++++++++++++++++++ .../test_case.py | 3 +- .../test_case.py | 1 + tests/ten_manager/publish/test_case.py | 1 + .../default_extension_python/extension.py | 2 +- .../default_extension_python/extension.py | 9 +- 32 files changed, 335 insertions(+), 40 deletions(-) create mode 100644 tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/BUILD.gn create mode 100644 tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/__init__.py create mode 100644 tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_app/expected.json create mode 100644 tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_app/local_c/manifest.json create mode 100644 tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_app/manifest.json create mode 100644 tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_app/ten_packages/extension/local_d/manifest.json create mode 100644 tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_case.py diff --git a/.vscode/launch.json b/.vscode/launch.json index 03882b70f3..5449c24e41 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -388,10 +388,14 @@ "type": "lldb", "request": "launch", "program": "${workspaceFolder}/out/linux/x64/ten_manager/bin/tman", - "cwd": "${workspaceFolder}/out/linux/x64/tests/ten_manager/install_all/local_dependency_link_in_extension/test_app", + "cwd": "${workspaceFolder}/out/linux/x64/tests/ten_manager/install/mock_app/mock_app", "args": [ "--config-file=${workspaceFolder}/out/linux/x64/tests/local_registry/config.json", "install", + "extension", + "ext_a", + "--os=linux", + "--arch=x64" ], "preRunCommands": [ "script import pathlib;import subprocess;import lldb;rustc_sysroot = subprocess.getoutput(\"rustc --print sysroot\");rustlib_etc = pathlib.Path(rustc_sysroot) / \"lib\" / \"rustlib\" / \"etc\";lldb.debugger.HandleCommand(f'command script import \"{rustlib_etc / \"lldb_lookup.py\"}\"');lldb.debugger.HandleCommand(f'command source -s 0 \"{rustlib_etc / \"lldb_commands\"}\"')" diff --git a/core/src/ten_manager/src/cmd/cmd_install.rs b/core/src/ten_manager/src/cmd/cmd_install.rs index 01abc9ac13..b3601d0de2 100644 --- a/core/src/ten_manager/src/cmd/cmd_install.rs +++ b/core/src/ten_manager/src/cmd/cmd_install.rs @@ -205,6 +205,10 @@ pub async fn execute_cmd( // the dependency tree, then users will be questioned whether to overwrite // them with the new packages or quit the installation. let mut all_existing_local_pkgs: Vec = vec![]; + let mut all_compatible_existing_local_pkgs: HashMap< + PkgTypeAndName, + HashMap, + > = HashMap::new(); let mut dep_relationship_from_cmd_line: Option = None; @@ -239,10 +243,16 @@ pub async fn execute_cmd( all_existing_local_pkgs = tman_get_all_existed_pkgs_info_of_app(tman_config, &cwd)?; + all_candidates + .entry((&app_pkg_).into()) + .or_default() + .insert((&app_pkg_).into(), app_pkg_.clone()); + filter_compatible_pkgs_to_candidates( tman_config, &all_existing_local_pkgs, - &mut all_candidates, + // &mut all_candidates, + &mut all_compatible_existing_local_pkgs, &command_data.support, ); @@ -277,16 +287,23 @@ pub async fn execute_cmd( // dependencies on a specific version of an app, so the app also // needs to be included in the package list for dependency tree // calculation. - initial_pkgs_to_find_candidates - .push(get_pkg_info_from_path(&cwd, true)?); + let app_pkg_ = get_pkg_info_from_path(&cwd, true)?; + + initial_pkgs_to_find_candidates.push(app_pkg_.clone()); all_existing_local_pkgs = tman_get_all_existed_pkgs_info_of_app(tman_config, &cwd)?; + all_candidates + .entry((&app_pkg_).into()) + .or_default() + .insert((&app_pkg_).into(), app_pkg_.clone()); + filter_compatible_pkgs_to_candidates( tman_config, &all_existing_local_pkgs, - &mut all_candidates, + // &mut all_candidates, + &mut all_compatible_existing_local_pkgs, &command_data.support, ); } @@ -301,13 +318,20 @@ pub async fn execute_cmd( fs::create_dir_all(path)?; } - initial_pkgs_to_find_candidates - .push(get_pkg_info_from_path(&cwd, true)?); + let extension_pkg_ = get_pkg_info_from_path(&cwd, true)?; + + initial_pkgs_to_find_candidates.push(extension_pkg_.clone()); + + all_candidates + .entry((&extension_pkg_).into()) + .or_default() + .insert((&extension_pkg_).into(), extension_pkg_.clone()); filter_compatible_pkgs_to_candidates( tman_config, &initial_pkgs_to_find_candidates, - &mut all_candidates, + // &mut all_candidates, + &mut all_compatible_existing_local_pkgs, &command_data.support, ); } @@ -337,6 +361,8 @@ pub async fn execute_cmd( dep_relationship_from_cmd_line .as_ref() .map(|rel| &rel.dependency), + // &all_existing_local_pkgs, + &all_compatible_existing_local_pkgs, all_candidates, locked_pkgs.as_ref(), ) diff --git a/core/src/ten_manager/src/dep_and_candidate/mod.rs b/core/src/ten_manager/src/dep_and_candidate/mod.rs index 516d3d2f08..ea535fcd7c 100644 --- a/core/src/ten_manager/src/dep_and_candidate/mod.rs +++ b/core/src/ten_manager/src/dep_and_candidate/mod.rs @@ -128,6 +128,11 @@ async fn process_non_local_dependency_to_get_candidate( tman_config: &TmanConfig, support: &PkgSupport, dependency: &PkgDependency, + // all_existing_local_pkgs: &[PkgInfo], + all_compatible_existing_local_pkgs: &HashMap< + PkgTypeAndName, + HashMap, + >, all_candidates: &mut HashMap< PkgTypeAndName, HashMap, @@ -175,6 +180,7 @@ async fn process_non_local_dependency_to_get_candidate( candidate_pkg_infos.push(candidate_pkg_info); } + // =-=-= // Find packages from the all_candidates that meet the specified // criteria and add them to the candidate_pkg_infos. // @@ -183,12 +189,30 @@ async fn process_non_local_dependency_to_get_candidate( // those locally installed packages to be added to // `new_pkgs_to_be_searched`, ensuring that the // dependencies within those packages are processed. - if let Some(candidates) = all_candidates.get(&dependency.into()) { + // =-=-= + // if let Some(candidates) = all_candidates.get(&dependency.into()) { + // for candidate in candidates { + // if dependency.version_req.matches(&candidate.0.version) { + // tman_verbose_println!( + // tman_config, + // "Collect candidate: {:?}", + // candidate + // ); + + // candidate_pkg_infos.push(candidate.1.clone()); + // } + // } + // } + + // =-=-= + if let Some(candidates) = + all_compatible_existing_local_pkgs.get(&dependency.into()) + { for candidate in candidates { if dependency.version_req.matches(&candidate.0.version) { tman_verbose_println!( tman_config, - "Collect candidate: {:?}", + "Collect candidate from existing local package: {:?}", candidate ); @@ -197,6 +221,24 @@ async fn process_non_local_dependency_to_get_candidate( } } + // =-=-= + // for candidate in all_existing_local_pkgs { + // if dependency.type_and_name == candidate.basic_info.type_and_name + // && dependency + // .version_req + // .matches(&candidate.basic_info.version) + // { + // tman_verbose_println!( + // tman_config, + // "Collect candidate from already installed package: {:?}", + // candidate + // ); + + // candidate_pkg_infos.push(candidate.clone()); + // } + // } + + // Filter suitable candidate packages according to `supports`. for mut candidate_pkg_info in candidate_pkg_infos { tman_verbose_println!( tman_config, @@ -209,12 +251,10 @@ async fn process_non_local_dependency_to_get_candidate( support, ); - // A package is considered a candidate only when - // compatible_score >= 0. + // A package is considered a candidate only when compatible_score >= 0. if compatible_score >= 0 { - // Record the package's compatible_score so that we can - // later select the most appropriate - // one. + // Record the package's compatible_score so that we can later select + // the most appropriate one. candidate_pkg_info.compatible_score = compatible_score; tman_verbose_println!( @@ -256,6 +296,11 @@ async fn process_dependencies_to_get_candidates( support: &PkgSupport, input_dependencies: &Vec, merged_dependencies: &mut HashMap, + // all_existing_local_pkgs: &[PkgInfo], + all_compatible_existing_local_pkgs: &HashMap< + PkgTypeAndName, + HashMap, + >, all_candidates: &mut HashMap< PkgTypeAndName, HashMap, @@ -283,6 +328,8 @@ async fn process_dependencies_to_get_candidates( tman_config, support, dependency, + // all_existing_local_pkgs, + all_compatible_existing_local_pkgs, all_candidates, new_pkgs_to_be_searched, ) @@ -352,6 +399,11 @@ pub async fn get_all_candidates_from_deps( support: &PkgSupport, mut pkgs_to_be_searched: Vec, extra_dep: Option<&PkgDependency>, + // all_existing_local_pkgs: &[PkgInfo], + all_compatible_existing_local_pkgs: &HashMap< + PkgTypeAndName, + HashMap, + >, mut all_candidates: HashMap>, locked_pkgs: Option<&HashMap>, ) -> Result>> { @@ -368,6 +420,8 @@ pub async fn get_all_candidates_from_deps( support, &vec![extra_dep.unwrap().clone()], &mut merged_dependencies, + // all_existing_local_pkgs, + all_compatible_existing_local_pkgs, &mut all_candidates, &mut new_pkgs_to_be_searched, ) @@ -392,6 +446,8 @@ pub async fn get_all_candidates_from_deps( support, &pkg_to_be_search.dependencies, &mut merged_dependencies, + // all_existing_local_pkgs, + all_compatible_existing_local_pkgs, &mut all_candidates, &mut new_pkgs_to_be_searched, ) diff --git a/core/src/ten_manager/src/install/mod.rs b/core/src/ten_manager/src/install/mod.rs index 0d2334aa89..140508d2d7 100644 --- a/core/src/ten_manager/src/install/mod.rs +++ b/core/src/ten_manager/src/install/mod.rs @@ -74,32 +74,42 @@ fn install_local_dependency_pkg_info( "Source path must be a directory." ); - match command_data.local_install_mode { - LocalInstallMode::Invalid => panic!("Should not happen."), - LocalInstallMode::Copy => { - copy_folder_recursively( - &src_dir_path.to_string_lossy().to_string(), - dest_dir_path, - )?; - } - LocalInstallMode::Link => { - #[cfg(unix)] - { - std::os::unix::fs::symlink(src_dir_path, dest_dir_path) - .map_err(|e| { - anyhow::anyhow!("Failed to create symlink: {}", e) - })?; + if Path::new(dest_dir_path).exists() { + println!( + "Destination directory '{}' already exists. Skipping copy/link.", + dest_dir_path + ); + } else { + match command_data.local_install_mode { + LocalInstallMode::Invalid => panic!("Should not happen."), + LocalInstallMode::Copy => { + copy_folder_recursively( + &src_dir_path.to_string_lossy().to_string(), + dest_dir_path, + )?; } + LocalInstallMode::Link => { + #[cfg(unix)] + { + std::os::unix::fs::symlink(src_dir_path, dest_dir_path) + .map_err(|e| { + anyhow::anyhow!("Failed to create symlink: {}", e) + })?; + } - #[cfg(windows)] - { - std::os::windows::fs::symlink_dir(src_dir_path, &dest_dir_path) + #[cfg(windows)] + { + std::os::windows::fs::symlink_dir( + src_dir_path, + &dest_dir_path, + ) .map_err(|e| { anyhow::anyhow!( "Failed to create directory symlink: {}", e ) })?; + } } } } diff --git a/tests/ten_manager/dependency_resolve/test_case.py b/tests/ten_manager/dependency_resolve/test_case.py index 7ecb98c665..ed68d102df 100644 --- a/tests/ten_manager/dependency_resolve/test_case.py +++ b/tests/ten_manager/dependency_resolve/test_case.py @@ -72,6 +72,7 @@ def run_test(tman_bin: str, root_dir: str, test_app_root_folder: str) -> None: command = [ tman_bin, + "--yes", f"--config-file={config_file}", "install", ] diff --git a/tests/ten_manager/error_context/invalid_package_type/test_case.py b/tests/ten_manager/error_context/invalid_package_type/test_case.py index 16fbc360ce..2b2b30bd87 100644 --- a/tests/ten_manager/error_context/invalid_package_type/test_case.py +++ b/tests/ten_manager/error_context/invalid_package_type/test_case.py @@ -34,6 +34,7 @@ def test_invalid_package_type(): cmds = [ tman_bin, + "--yes", f"--config-file={config_file}", "install", ] diff --git a/tests/ten_manager/error_context/no_such_package/test_case.py b/tests/ten_manager/error_context/no_such_package/test_case.py index ba43c9c67f..90fe11bfbc 100644 --- a/tests/ten_manager/error_context/no_such_package/test_case.py +++ b/tests/ten_manager/error_context/no_such_package/test_case.py @@ -34,6 +34,7 @@ def test_invalid_package_type(): cmds = [ tman_bin, + "--yes", f"--config-file={config_file}", "install", "extension", diff --git a/tests/ten_manager/error_context/package_version_not_found/test_case.py b/tests/ten_manager/error_context/package_version_not_found/test_case.py index 6256b09a0e..b59b2b1be3 100644 --- a/tests/ten_manager/error_context/package_version_not_found/test_case.py +++ b/tests/ten_manager/error_context/package_version_not_found/test_case.py @@ -34,6 +34,7 @@ def test_invalid_package_type(): cmds = [ tman_bin, + "--yes", f"--config-file={config_file}", "install", ] diff --git a/tests/ten_manager/install/mock_app/test_case.py b/tests/ten_manager/install/mock_app/test_case.py index 64ec444425..b589ef58c5 100644 --- a/tests/ten_manager/install/mock_app/test_case.py +++ b/tests/ten_manager/install/mock_app/test_case.py @@ -89,6 +89,7 @@ def test_tman_install(): returncode, output_text = cmd_exec.run_cmd_realtime( [ tman_bin, + "--yes", f"--config-file={config_file}", "install", "extension", @@ -106,6 +107,7 @@ def test_tman_install(): returncode, output_text = cmd_exec.run_cmd_realtime( [ tman_bin, + "--yes", f"--config-file={config_file}", "install", "extension", diff --git a/tests/ten_manager/install/mock_extension/test_case.py b/tests/ten_manager/install/mock_extension/test_case.py index b69909e5b5..820748706c 100644 --- a/tests/ten_manager/install/mock_extension/test_case.py +++ b/tests/ten_manager/install/mock_extension/test_case.py @@ -73,6 +73,7 @@ def test_tman_install(): returncode, output_text = cmd_exec.run_cmd_realtime( [ tman_bin, + "--yes", f"--config-file={config_file}", "install", ], diff --git a/tests/ten_manager/install_all/BUILD.gn b/tests/ten_manager/install_all/BUILD.gn index c648e5b3eb..1a75d3238d 100644 --- a/tests/ten_manager/install_all/BUILD.gn +++ b/tests/ten_manager/install_all/BUILD.gn @@ -15,6 +15,7 @@ group("install_all") { "install_under_the_guidance_of_lock", "local_dependency_link", "local_dependency_link_in_extension", + "local_dependency_link_in_extension_not_use", "not_install_again_for_installed_extension", "update_pkgs_that_local_pkgs_depend_on", ] diff --git a/tests/ten_manager/install_all/install_a_before_install_all/test_case.py b/tests/ten_manager/install_all/install_a_before_install_all/test_case.py index 009ef92b1f..097e82ee79 100644 --- a/tests/ten_manager/install_all/install_a_before_install_all/test_case.py +++ b/tests/ten_manager/install_all/install_a_before_install_all/test_case.py @@ -40,6 +40,7 @@ def test_tman_dependency_resolve(): returncode, output_text = cmd_exec.run_cmd_realtime( [ tman_bin, + "--yes", f"--config-file={config_file}", "install", "extension", @@ -58,6 +59,7 @@ def test_tman_dependency_resolve(): returncode, output_text = cmd_exec.run_cmd_realtime( [ tman_bin, + "--yes", f"--config-file={config_file}", "install", ], diff --git a/tests/ten_manager/install_all/install_all_fail_with_local_unsupported_pkg/test_case.py b/tests/ten_manager/install_all/install_all_fail_with_local_unsupported_pkg/test_case.py index 397ab970f4..5ae974daa4 100644 --- a/tests/ten_manager/install_all/install_all_fail_with_local_unsupported_pkg/test_case.py +++ b/tests/ten_manager/install_all/install_all_fail_with_local_unsupported_pkg/test_case.py @@ -29,7 +29,13 @@ def test_tman_dependency_resolve(): ) returncode, output_text = cmd_exec.run_cmd_realtime( - [tman_bin, f"--config-file={config_file}", "install", "--os=linux"], + [ + tman_bin, + "--yes", + f"--config-file={config_file}", + "install", + "--os=linux", + ], cwd=app_dir, ) diff --git a/tests/ten_manager/install_all/install_all_in_local_file_system/test_case.py b/tests/ten_manager/install_all/install_all_in_local_file_system/test_case.py index f3bca3a43d..d5d8357dbf 100644 --- a/tests/ten_manager/install_all/install_all_in_local_file_system/test_case.py +++ b/tests/ten_manager/install_all/install_all_in_local_file_system/test_case.py @@ -44,6 +44,7 @@ def test_tman_dependency_resolve(): returncode, output_text = cmd_exec.run_cmd_realtime( [ tman_bin, + "--yes", f"--config-file={config_file}", "install", ], diff --git a/tests/ten_manager/install_all/install_breaking_lock/test_case.py b/tests/ten_manager/install_all/install_breaking_lock/test_case.py index f1baacf65f..a9b9c0ed85 100644 --- a/tests/ten_manager/install_all/install_breaking_lock/test_case.py +++ b/tests/ten_manager/install_all/install_breaking_lock/test_case.py @@ -80,6 +80,7 @@ def test_tman_dependency_resolve(): command = [ tman_bin, + "--yes", f"--config-file={config_file}", "install", ] diff --git a/tests/ten_manager/install_all/install_dependencies_of_local_pkgs/test_case.py b/tests/ten_manager/install_all/install_dependencies_of_local_pkgs/test_case.py index cae6391309..c1456beca2 100644 --- a/tests/ten_manager/install_all/install_dependencies_of_local_pkgs/test_case.py +++ b/tests/ten_manager/install_all/install_dependencies_of_local_pkgs/test_case.py @@ -110,6 +110,7 @@ def test_tman_dependency_resolve(): returncode, output_text = cmd_exec.run_cmd_realtime( [ tman_bin, + "--yes", f"--config-file={config_file}", "install", ], diff --git a/tests/ten_manager/install_all/install_locked_pkg_same_version/test_case.py b/tests/ten_manager/install_all/install_locked_pkg_same_version/test_case.py index 2191c78358..f5f927ffe3 100644 --- a/tests/ten_manager/install_all/install_locked_pkg_same_version/test_case.py +++ b/tests/ten_manager/install_all/install_locked_pkg_same_version/test_case.py @@ -86,6 +86,7 @@ def test_tman_dependency_resolve(): command = [ tman_bin, + "--yes", f"--config-file={config_file}", "install", ] diff --git a/tests/ten_manager/install_all/install_under_the_guidance_of_lock/test_case.py b/tests/ten_manager/install_all/install_under_the_guidance_of_lock/test_case.py index 4eb44e206b..faa4101966 100644 --- a/tests/ten_manager/install_all/install_under_the_guidance_of_lock/test_case.py +++ b/tests/ten_manager/install_all/install_under_the_guidance_of_lock/test_case.py @@ -75,6 +75,7 @@ def test_tman_dependency_resolve(): returncode, output_text = cmd_exec.run_cmd_realtime( [ tman_bin, + "--yes", f"--config-file={config_file}", "install", ], diff --git a/tests/ten_manager/install_all/local_dependency_link/test_case.py b/tests/ten_manager/install_all/local_dependency_link/test_case.py index 099308b1ff..a13af28056 100644 --- a/tests/ten_manager/install_all/local_dependency_link/test_case.py +++ b/tests/ten_manager/install_all/local_dependency_link/test_case.py @@ -44,6 +44,7 @@ def test_tman_dependency_resolve(): returncode, output_text = cmd_exec.run_cmd_realtime( [ tman_bin, + "--yes", f"--config-file={config_file}", "install", ], diff --git a/tests/ten_manager/install_all/local_dependency_link_in_extension/test_case.py b/tests/ten_manager/install_all/local_dependency_link_in_extension/test_case.py index 83c0493973..5e4873eee1 100644 --- a/tests/ten_manager/install_all/local_dependency_link_in_extension/test_case.py +++ b/tests/ten_manager/install_all/local_dependency_link_in_extension/test_case.py @@ -44,6 +44,7 @@ def test_tman_dependency_resolve(): returncode, output_text = cmd_exec.run_cmd_realtime( [ tman_bin, + "--yes", f"--config-file={config_file}", "install", ], @@ -56,7 +57,7 @@ def test_tman_dependency_resolve(): installed_count = get_installed_extensions_count(app_dir) assert ( installed_count == 3 - ), f"Expected 2 extensions, found {installed_count}." + ), f"Expected 3 extensions, found {installed_count}." ext_c_path = os.path.join(app_dir, "ten_packages/extension/ext_c") local_c_path = os.path.join(app_dir, "local_c") diff --git a/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/BUILD.gn b/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/BUILD.gn new file mode 100644 index 0000000000..136d37bb8c --- /dev/null +++ b/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/BUILD.gn @@ -0,0 +1,21 @@ +# +# Copyright © 2025 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +import("//build/ten_runtime/feature/test.gni") + +ten_package_test_prepare_auxiliary_resources( + "local_dependency_link_in_extension_not_use") { + resources = [ + "//.gnfiles/build/scripts/cmd_exec.py=>common/cmd_exec.py", + "__init__.py", + "test_app", + "test_case.py", + ] + deps = [ + "//core/src/ten_manager", + "//tests/local_registry", + ] +} diff --git a/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/__init__.py b/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_app/expected.json b/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_app/expected.json new file mode 100644 index 0000000000..41517b65e2 --- /dev/null +++ b/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_app/expected.json @@ -0,0 +1,12 @@ +{ + "extension": [ + { + "name": "ext_1", + "version": "3.0.0" + }, + { + "name": "local_d", + "version": "3.0.0" + } + ] +} \ No newline at end of file diff --git a/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_app/local_c/manifest.json b/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_app/local_c/manifest.json new file mode 100644 index 0000000000..2b5c65b5e7 --- /dev/null +++ b/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_app/local_c/manifest.json @@ -0,0 +1,6 @@ +{ + "type": "extension", + "name": "ext_c", + "version": "3.0.0", + "dependencies": [] +} \ No newline at end of file diff --git a/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_app/manifest.json b/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_app/manifest.json new file mode 100644 index 0000000000..b23be09b7b --- /dev/null +++ b/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_app/manifest.json @@ -0,0 +1,12 @@ +{ + "type": "app", + "name": "test_app", + "version": "1.0.0", + "dependencies": [ + { + "type": "extension", + "name": "ext_1", + "version": "3.0.0" + } + ] +} \ No newline at end of file diff --git a/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_app/ten_packages/extension/local_d/manifest.json b/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_app/ten_packages/extension/local_d/manifest.json new file mode 100644 index 0000000000..fc78f61759 --- /dev/null +++ b/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_app/ten_packages/extension/local_d/manifest.json @@ -0,0 +1,10 @@ +{ + "type": "extension", + "name": "local_d", + "version": "3.0.0", + "dependencies": [ + { + "path": "../../../local_c" + } + ] +} \ No newline at end of file diff --git a/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_case.py b/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_case.py new file mode 100644 index 0000000000..cc151f93de --- /dev/null +++ b/tests/ten_manager/install_all/local_dependency_link_in_extension_not_use/test_case.py @@ -0,0 +1,109 @@ +# +# Copyright © 2025 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +import os +import sys +import json +from .common import cmd_exec + + +def get_installed_extensions_count(app_dir: str): + extension_dir = os.path.join(app_dir, "ten_packages/extension/") + extensions = os.listdir(extension_dir) + + return len(extensions) + + +def normalize_path(path): + if path.startswith("\\\\?\\"): + return path[4:] + return path + + +def analyze_resolve_result(app_root_folder: str) -> None: + extension_folder = os.path.join( + app_root_folder, "ten_packages", "extension" + ) + + with open( + os.path.join(app_root_folder, "expected.json"), "r" + ) as expected_json_file: + expected_json = json.load(expected_json_file) + + # @{ + # Check all expected extensions are installed. + for ext in expected_json["extension"]: + found_in_folder = False + + for dir_item in os.listdir(extension_folder): + if dir_item == ext["name"]: + found_in_folder = True + with open( + os.path.join( + extension_folder, ext["name"], "manifest.json" + ), + "r", + ) as ext_manifest_file: + ext_manifest_json = json.load(ext_manifest_file) + assert ext_manifest_json["name"] == ext["name"] + assert ext_manifest_json["version"] == ext["version"] + break + + assert found_in_folder is True + # @} + + # @{ + # Check there is no other unexpected extensions be installed. + installed_extension_cnt = 0 + if os.path.exists(extension_folder): + for dir_item in os.listdir(extension_folder): + installed_extension_cnt += 1 + assert len(expected_json["extension"]) == installed_extension_cnt + # @} + + +def test_tman_dependency_resolve(): + base_path = os.path.dirname(os.path.abspath(__file__)) + root_dir = os.path.join(base_path, "../../../../") + + if sys.platform == "win32": + os.environ["PATH"] = ( + os.path.join(root_dir, "ten_manager/lib") + ";" + os.getenv("PATH") + ) + tman_bin = os.path.join(root_dir, "ten_manager/bin/tman.exe") + else: + tman_bin = os.path.join(root_dir, "ten_manager/bin/tman") + + app_dir = os.path.join(base_path, "test_app") + + config_file = os.path.join( + root_dir, + "tests/local_registry/config.json", + ) + + returncode, output_text = cmd_exec.run_cmd_realtime( + [ + tman_bin, + "--yes", + f"--config-file={config_file}", + "install", + ], + cwd=app_dir, + ) + if returncode != 0: + print(output_text) + assert False + + installed_count = get_installed_extensions_count(app_dir) + assert ( + installed_count == 2 + ), f"Expected 2 extensions, found {installed_count}." + + analyze_resolve_result(app_dir) + + +if __name__ == "__main__": + test_tman_dependency_resolve() diff --git a/tests/ten_manager/install_all/not_install_again_for_installed_extension/test_case.py b/tests/ten_manager/install_all/not_install_again_for_installed_extension/test_case.py index 63c5fd65f8..aeeed9c766 100644 --- a/tests/ten_manager/install_all/not_install_again_for_installed_extension/test_case.py +++ b/tests/ten_manager/install_all/not_install_again_for_installed_extension/test_case.py @@ -53,6 +53,7 @@ def test_tman_dependency_resolve(): returncode, output_text = cmd_exec.run_cmd_realtime( [ tman_bin, + "--yes", f"--config-file={config_file}", "install", ], @@ -62,7 +63,7 @@ def test_tman_dependency_resolve(): print(output_text) assert False - assert check_installed_extensions(app_dir) == True + assert check_installed_extensions(app_dir) is True if __name__ == "__main__": diff --git a/tests/ten_manager/install_all/update_pkgs_that_local_pkgs_depend_on/test_case.py b/tests/ten_manager/install_all/update_pkgs_that_local_pkgs_depend_on/test_case.py index cae6391309..c1456beca2 100644 --- a/tests/ten_manager/install_all/update_pkgs_that_local_pkgs_depend_on/test_case.py +++ b/tests/ten_manager/install_all/update_pkgs_that_local_pkgs_depend_on/test_case.py @@ -110,6 +110,7 @@ def test_tman_dependency_resolve(): returncode, output_text = cmd_exec.run_cmd_realtime( [ tman_bin, + "--yes", f"--config-file={config_file}", "install", ], diff --git a/tests/ten_manager/publish/test_case.py b/tests/ten_manager/publish/test_case.py index 452aa3d6e0..48b841c31c 100644 --- a/tests/ten_manager/publish/test_case.py +++ b/tests/ten_manager/publish/test_case.py @@ -51,6 +51,7 @@ def test_tman_publish(): returncode, output_text = cmd_exec.run_cmd_realtime( [ tman_bin, + "--yes", f"--config-file={config_file}", "publish", ], diff --git a/tests/ten_runtime/integration/python/large_json_python/large_json_python_app/ten_packages/extension/default_extension_python/extension.py b/tests/ten_runtime/integration/python/large_json_python/large_json_python_app/ten_packages/extension/default_extension_python/extension.py index 2ee406e76d..e8efacc551 100644 --- a/tests/ten_runtime/integration/python/large_json_python/large_json_python_app/ten_packages/extension/default_extension_python/extension.py +++ b/tests/ten_runtime/integration/python/large_json_python/large_json_python_app/ten_packages/extension/default_extension_python/extension.py @@ -40,7 +40,7 @@ def __routine(self, ten_env: TenEnv): i += 1 throw_exception = True - assert throw_exception == True + assert throw_exception is True self.queue.get() diff --git a/tests/ten_runtime/integration/python/send_recv_image_python/send_recv_image_python_app/ten_packages/extension/default_extension_python/extension.py b/tests/ten_runtime/integration/python/send_recv_image_python/send_recv_image_python_app/ten_packages/extension/default_extension_python/extension.py index 6243067f9a..021660c5dd 100644 --- a/tests/ten_runtime/integration/python/send_recv_image_python/send_recv_image_python_app/ten_packages/extension/default_extension_python/extension.py +++ b/tests/ten_runtime/integration/python/send_recv_image_python/send_recv_image_python_app/ten_packages/extension/default_extension_python/extension.py @@ -36,7 +36,7 @@ def on_cmd(self, ten_env: TenEnv, cmd: Cmd) -> None: cmd_json = cmd.get_property_to_json() ten_env.log_info(f"DefaultExtension on_cmd json: {cmd_json}") - assert hasattr(self, "request_cmd") != True + assert hasattr(self, "request_cmd") is not True self.request_cmd = cmd @@ -57,7 +57,10 @@ def on_cmd(self, ten_env: TenEnv, cmd: Cmd) -> None: pixel_fmt = new_image.get_pixel_fmt() timestamp = new_image.get_timestamp() print( - "DefaultExtension send video frame, width: %d, height: %d, pixel_fmt: %d, timestamp: %d" + ( + "DefaultExtension send video frame, width: %d, height: %d, " + "pixel_fmt: %d, timestamp: %d" + ) % (width, height, pixel_fmt, timestamp) ) @@ -66,7 +69,7 @@ def on_cmd(self, ten_env: TenEnv, cmd: Cmd) -> None: def on_video_frame(self, ten_env: TenEnv, video_frame: VideoFrame) -> None: ten_env.log_debug("on_video_frame") - assert hasattr(self, "request_cmd") == True + assert hasattr(self, "request_cmd") is True assert video_frame.get_height() == self.pixels.height assert video_frame.get_width() == self.pixels.width From 75f1be5407288b50f04a376d969def56b6911176 Mon Sep 17 00:00:00 2001 From: Hu Yueh-Wei Date: Tue, 7 Jan 2025 11:56:09 +0800 Subject: [PATCH 2/4] feat: add support for local dependency in extension --- .../src/cmd/cmd_check/cmd_check_graph.rs | 4 +- core/src/ten_manager/src/cmd/cmd_install.rs | 92 ++++++++++--------- .../ten_manager/src/dep_and_candidate/mod.rs | 67 +++----------- .../ten_manager/src/designer/get_all_pkgs.rs | 4 +- core/src/ten_manager/src/package_info/mod.rs | 6 +- core/src/ten_rust/src/pkg_info/mod.rs | 8 +- 6 files changed, 73 insertions(+), 108 deletions(-) diff --git a/core/src/ten_manager/src/cmd/cmd_check/cmd_check_graph.rs b/core/src/ten_manager/src/cmd/cmd_check/cmd_check_graph.rs index 8f63a57f56..fa2345f913 100644 --- a/core/src/ten_manager/src/cmd/cmd_check/cmd_check_graph.rs +++ b/core/src/ten_manager/src/cmd/cmd_check/cmd_check_graph.rs @@ -11,7 +11,7 @@ use clap::{Arg, ArgMatches, Command}; use console::Emoji; use ten_rust::pkg_info::{ - get_all_existed_pkgs_info_of_app, graph::Graph, localhost, + get_all_installed_pkgs_info_of_app, graph::Graph, localhost, property::parse_property_in_folder, PkgInfo, }; @@ -107,7 +107,7 @@ fn get_existed_pkgs_of_all_apps( for app in &command.app_dir { let app_path = path::Path::new(app); - let app_existed_pkgs = get_all_existed_pkgs_info_of_app(app_path)?; + let app_existed_pkgs = get_all_installed_pkgs_info_of_app(app_path)?; let app_property = parse_property_in_folder(app_path)?; let app_uri = if let Some(property) = app_property { diff --git a/core/src/ten_manager/src/cmd/cmd_install.rs b/core/src/ten_manager/src/cmd/cmd_install.rs index b3601d0de2..a2a5411902 100644 --- a/core/src/ten_manager/src/cmd/cmd_install.rs +++ b/core/src/ten_manager/src/cmd/cmd_install.rs @@ -42,7 +42,7 @@ use crate::{ }, log::tman_verbose_println, manifest_lock::parse_manifest_lock_in_folder, - package_info::tman_get_all_existed_pkgs_info_of_app, + package_info::tman_get_all_installed_pkgs_info_of_app, solver::{ introducer::extract_introducer_relations_from_raw_solver_results, solve::{solve_all, DependencyRelationship}, @@ -155,6 +155,26 @@ fn get_locked_pkgs(app_dir: &Path) -> Option> { } } +fn add_cwd_pkg_to_initial_pkg_to_find_candidates_and_all_candidates( + cwd: &Path, + initial_pkgs_to_find_candidates: &mut Vec, + all_candidates: &mut HashMap< + PkgTypeAndName, + HashMap, + >, +) -> Result { + let cwd_pkg = get_pkg_info_from_path(cwd, true)?; + + initial_pkgs_to_find_candidates.push(cwd_pkg.clone()); + + all_candidates + .entry((&cwd_pkg).into()) + .or_default() + .insert((&cwd_pkg).into(), cwd_pkg.clone()); + + Ok(cwd_pkg) +} + pub async fn execute_cmd( tman_config: &TmanConfig, command_data: InstallCommand, @@ -204,8 +224,8 @@ pub async fn execute_cmd( // *) If some of these packages are not compatible with packages in // the dependency tree, then users will be questioned whether to overwrite // them with the new packages or quit the installation. - let mut all_existing_local_pkgs: Vec = vec![]; - let mut all_compatible_existing_local_pkgs: HashMap< + let mut all_installed_pkgs: Vec = vec![]; + let mut all_compatible_installed_pkgs: HashMap< PkgTypeAndName, HashMap, > = HashMap::new(); @@ -235,24 +255,22 @@ pub async fn execute_cmd( installing_pkg_name = Some(installing_pkg_name_.clone()); // The `cwd` must be the base directory of a TEN app. - let app_pkg_ = get_pkg_info_from_path(&cwd, true)?; - cwd_pkg_name = app_pkg_.basic_info.type_and_name.name.clone(); - - initial_pkgs_to_find_candidates.push(app_pkg_.clone()); + let app_pkg_ = + add_cwd_pkg_to_initial_pkg_to_find_candidates_and_all_candidates( + &cwd, + &mut initial_pkgs_to_find_candidates, + &mut all_candidates, + )?; - all_existing_local_pkgs = - tman_get_all_existed_pkgs_info_of_app(tman_config, &cwd)?; + cwd_pkg_name = app_pkg_.basic_info.type_and_name.name.clone(); - all_candidates - .entry((&app_pkg_).into()) - .or_default() - .insert((&app_pkg_).into(), app_pkg_.clone()); + all_installed_pkgs = + tman_get_all_installed_pkgs_info_of_app(tman_config, &cwd)?; filter_compatible_pkgs_to_candidates( tman_config, - &all_existing_local_pkgs, - // &mut all_candidates, - &mut all_compatible_existing_local_pkgs, + &all_installed_pkgs, + &mut all_compatible_installed_pkgs, &command_data.support, ); @@ -287,23 +305,20 @@ pub async fn execute_cmd( // dependencies on a specific version of an app, so the app also // needs to be included in the package list for dependency tree // calculation. - let app_pkg_ = get_pkg_info_from_path(&cwd, true)?; - - initial_pkgs_to_find_candidates.push(app_pkg_.clone()); - all_existing_local_pkgs = - tman_get_all_existed_pkgs_info_of_app(tman_config, &cwd)?; + add_cwd_pkg_to_initial_pkg_to_find_candidates_and_all_candidates( + &cwd, + &mut initial_pkgs_to_find_candidates, + &mut all_candidates, + )?; - all_candidates - .entry((&app_pkg_).into()) - .or_default() - .insert((&app_pkg_).into(), app_pkg_.clone()); + all_installed_pkgs = + tman_get_all_installed_pkgs_info_of_app(tman_config, &cwd)?; filter_compatible_pkgs_to_candidates( tman_config, - &all_existing_local_pkgs, - // &mut all_candidates, - &mut all_compatible_existing_local_pkgs, + &all_installed_pkgs, + &mut all_compatible_installed_pkgs, &command_data.support, ); } @@ -318,20 +333,16 @@ pub async fn execute_cmd( fs::create_dir_all(path)?; } - let extension_pkg_ = get_pkg_info_from_path(&cwd, true)?; - - initial_pkgs_to_find_candidates.push(extension_pkg_.clone()); - - all_candidates - .entry((&extension_pkg_).into()) - .or_default() - .insert((&extension_pkg_).into(), extension_pkg_.clone()); + add_cwd_pkg_to_initial_pkg_to_find_candidates_and_all_candidates( + &cwd, + &mut initial_pkgs_to_find_candidates, + &mut all_candidates, + )?; filter_compatible_pkgs_to_candidates( tman_config, &initial_pkgs_to_find_candidates, - // &mut all_candidates, - &mut all_compatible_existing_local_pkgs, + &mut all_compatible_installed_pkgs, &command_data.support, ); } @@ -361,8 +372,7 @@ pub async fn execute_cmd( dep_relationship_from_cmd_line .as_ref() .map(|rel| &rel.dependency), - // &all_existing_local_pkgs, - &all_compatible_existing_local_pkgs, + &all_compatible_installed_pkgs, all_candidates, locked_pkgs.as_ref(), ) @@ -421,7 +431,7 @@ pub async fn execute_cmd( // or replaced. let has_conflict = compare_solver_results_with_existed_pkgs( &remaining_solver_results, - &all_existing_local_pkgs, + &all_installed_pkgs, ); if has_conflict && !tman_config.assume_yes { diff --git a/core/src/ten_manager/src/dep_and_candidate/mod.rs b/core/src/ten_manager/src/dep_and_candidate/mod.rs index ea535fcd7c..9d081a006a 100644 --- a/core/src/ten_manager/src/dep_and_candidate/mod.rs +++ b/core/src/ten_manager/src/dep_and_candidate/mod.rs @@ -128,8 +128,7 @@ async fn process_non_local_dependency_to_get_candidate( tman_config: &TmanConfig, support: &PkgSupport, dependency: &PkgDependency, - // all_existing_local_pkgs: &[PkgInfo], - all_compatible_existing_local_pkgs: &HashMap< + all_compatible_installed_pkgs: &HashMap< PkgTypeAndName, HashMap, >, @@ -180,39 +179,17 @@ async fn process_non_local_dependency_to_get_candidate( candidate_pkg_infos.push(candidate_pkg_info); } - // =-=-= - // Find packages from the all_candidates that meet the specified - // criteria and add them to the candidate_pkg_infos. - // - // Since `all_candidates` includes the ten packages installed - // locally, searching through `all_candidates` allows - // those locally installed packages to be added to - // `new_pkgs_to_be_searched`, ensuring that the - // dependencies within those packages are processed. - // =-=-= - // if let Some(candidates) = all_candidates.get(&dependency.into()) { - // for candidate in candidates { - // if dependency.version_req.matches(&candidate.0.version) { - // tman_verbose_println!( - // tman_config, - // "Collect candidate: {:?}", - // candidate - // ); - - // candidate_pkg_infos.push(candidate.1.clone()); - // } - // } - // } - - // =-=-= + // Find packages from the installed packages that meet the specified + // criteria and add them to the candidate_pkg_infos. This action ensures + // that the dependencies within those packages are processed. if let Some(candidates) = - all_compatible_existing_local_pkgs.get(&dependency.into()) + all_compatible_installed_pkgs.get(&dependency.into()) { for candidate in candidates { if dependency.version_req.matches(&candidate.0.version) { tman_verbose_println!( tman_config, - "Collect candidate from existing local package: {:?}", + "Collect candidate from an already installed package: {:?}", candidate ); @@ -221,23 +198,6 @@ async fn process_non_local_dependency_to_get_candidate( } } - // =-=-= - // for candidate in all_existing_local_pkgs { - // if dependency.type_and_name == candidate.basic_info.type_and_name - // && dependency - // .version_req - // .matches(&candidate.basic_info.version) - // { - // tman_verbose_println!( - // tman_config, - // "Collect candidate from already installed package: {:?}", - // candidate - // ); - - // candidate_pkg_infos.push(candidate.clone()); - // } - // } - // Filter suitable candidate packages according to `supports`. for mut candidate_pkg_info in candidate_pkg_infos { tman_verbose_println!( @@ -296,8 +256,7 @@ async fn process_dependencies_to_get_candidates( support: &PkgSupport, input_dependencies: &Vec, merged_dependencies: &mut HashMap, - // all_existing_local_pkgs: &[PkgInfo], - all_compatible_existing_local_pkgs: &HashMap< + all_compatible_installed_pkgs: &HashMap< PkgTypeAndName, HashMap, >, @@ -328,8 +287,7 @@ async fn process_dependencies_to_get_candidates( tman_config, support, dependency, - // all_existing_local_pkgs, - all_compatible_existing_local_pkgs, + all_compatible_installed_pkgs, all_candidates, new_pkgs_to_be_searched, ) @@ -399,8 +357,7 @@ pub async fn get_all_candidates_from_deps( support: &PkgSupport, mut pkgs_to_be_searched: Vec, extra_dep: Option<&PkgDependency>, - // all_existing_local_pkgs: &[PkgInfo], - all_compatible_existing_local_pkgs: &HashMap< + all_compatible_installed_pkgs: &HashMap< PkgTypeAndName, HashMap, >, @@ -420,8 +377,7 @@ pub async fn get_all_candidates_from_deps( support, &vec![extra_dep.unwrap().clone()], &mut merged_dependencies, - // all_existing_local_pkgs, - all_compatible_existing_local_pkgs, + all_compatible_installed_pkgs, &mut all_candidates, &mut new_pkgs_to_be_searched, ) @@ -446,8 +402,7 @@ pub async fn get_all_candidates_from_deps( support, &pkg_to_be_search.dependencies, &mut merged_dependencies, - // all_existing_local_pkgs, - all_compatible_existing_local_pkgs, + all_compatible_installed_pkgs, &mut all_candidates, &mut new_pkgs_to_be_searched, ) diff --git a/core/src/ten_manager/src/designer/get_all_pkgs.rs b/core/src/ten_manager/src/designer/get_all_pkgs.rs index b2327e42f3..e26fc20e83 100644 --- a/core/src/ten_manager/src/designer/get_all_pkgs.rs +++ b/core/src/ten_manager/src/designer/get_all_pkgs.rs @@ -7,7 +7,7 @@ use anyhow::{anyhow, Result}; use super::DesignerState; -use crate::package_info::tman_get_all_existed_pkgs_info_of_app; +use crate::package_info::tman_get_all_installed_pkgs_info_of_app; pub fn get_all_pkgs(state: &mut DesignerState) -> Result<()> { use std::path::PathBuf; @@ -15,7 +15,7 @@ pub fn get_all_pkgs(state: &mut DesignerState) -> Result<()> { if state.all_pkgs.is_none() { if let Some(base_dir) = &state.base_dir { let app_path = PathBuf::from(base_dir); - match tman_get_all_existed_pkgs_info_of_app( + match tman_get_all_installed_pkgs_info_of_app( &state.tman_config, &app_path, ) { diff --git a/core/src/ten_manager/src/package_info/mod.rs b/core/src/ten_manager/src/package_info/mod.rs index 979795d9c0..050d2bdd06 100644 --- a/core/src/ten_manager/src/package_info/mod.rs +++ b/core/src/ten_manager/src/package_info/mod.rs @@ -12,14 +12,14 @@ use anyhow::Result; use crate::config::TmanConfig; use ten_rust::pkg_info::{ - get_all_existed_pkgs_info_of_app_to_hashmap, PkgInfo, + get_all_installed_pkgs_info_of_app_to_hashmap, PkgInfo, }; -pub fn tman_get_all_existed_pkgs_info_of_app( +pub fn tman_get_all_installed_pkgs_info_of_app( tman_config: &TmanConfig, app_path: &Path, ) -> Result> { - let pkgs_info = get_all_existed_pkgs_info_of_app_to_hashmap(app_path)?; + let pkgs_info = get_all_installed_pkgs_info_of_app_to_hashmap(app_path)?; crate::log::tman_verbose_println!(tman_config, "{:?}", pkgs_info); Ok(pkgs_info.into_values().collect()) } diff --git a/core/src/ten_rust/src/pkg_info/mod.rs b/core/src/ten_rust/src/pkg_info/mod.rs index a0b893f34e..3307e7b667 100644 --- a/core/src/ten_rust/src/pkg_info/mod.rs +++ b/core/src/ten_rust/src/pkg_info/mod.rs @@ -231,7 +231,7 @@ fn collect_pkg_info_from_path<'a>( } } -pub fn get_all_existed_pkgs_info_of_app_to_hashmap( +pub fn get_all_installed_pkgs_info_of_app_to_hashmap( app_path: &Path, ) -> Result> { let mut pkgs_info: HashMap = HashMap::new(); @@ -286,10 +286,10 @@ pub fn get_all_existed_pkgs_info_of_app_to_hashmap( Ok(pkgs_info) } -pub fn get_all_existed_pkgs_info_of_app( +pub fn get_all_installed_pkgs_info_of_app( app_path: &Path, ) -> Result> { - let result = get_all_existed_pkgs_info_of_app_to_hashmap(app_path)?; + let result = get_all_installed_pkgs_info_of_app_to_hashmap(app_path)?; Ok(result.into_values().collect()) } @@ -383,7 +383,7 @@ pub fn ten_rust_check_graph_for_app( } let mut pkgs_of_app: HashMap> = HashMap::new(); - let pkgs_info = get_all_existed_pkgs_info_of_app(app_path)?; + let pkgs_info = get_all_installed_pkgs_info_of_app(app_path)?; pkgs_of_app.insert(app_uri.to_string(), pkgs_info); // `Graph::from_str` calls `validate`, and `validate` checks that there are From d442666f1861ff888d3e68c7881155071abaf78b Mon Sep 17 00:00:00 2001 From: Hu Yueh-Wei Date: Tue, 7 Jan 2025 12:10:04 +0800 Subject: [PATCH 3/4] feat: add support for local dependency in extension --- core/src/ten_manager/src/cmd/cmd_install.rs | 113 +++++++------------- core/src/ten_rust/tests/graph_check.rs | 14 +-- 2 files changed, 48 insertions(+), 79 deletions(-) diff --git a/core/src/ten_manager/src/cmd/cmd_install.rs b/core/src/ten_manager/src/cmd/cmd_install.rs index a2a5411902..c59b39bba6 100644 --- a/core/src/ten_manager/src/cmd/cmd_install.rs +++ b/core/src/ten_manager/src/cmd/cmd_install.rs @@ -18,6 +18,7 @@ use console::Emoji; use indicatif::HumanDuration; use inquire::Confirm; +use ten_rust::pkg_info::pkg_basic_info::PkgBasicInfo; use ten_rust::pkg_info::{ dependencies::PkgDependency, get_pkg_info_from_path, @@ -26,9 +27,6 @@ use ten_rust::pkg_info::{ supports::{Arch, Os, PkgSupport}, PkgInfo, }; -use ten_rust::pkg_info::{ - manifest::parse_manifest_in_folder, pkg_basic_info::PkgBasicInfo, -}; use crate::{ config::TmanConfig, @@ -155,24 +153,22 @@ fn get_locked_pkgs(app_dir: &Path) -> Option> { } } -fn add_cwd_pkg_to_initial_pkg_to_find_candidates_and_all_candidates( - cwd: &Path, +fn add_pkg_to_initial_pkg_to_find_candidates_and_all_candidates( + pkg: &PkgInfo, initial_pkgs_to_find_candidates: &mut Vec, all_candidates: &mut HashMap< PkgTypeAndName, HashMap, >, -) -> Result { - let cwd_pkg = get_pkg_info_from_path(cwd, true)?; - - initial_pkgs_to_find_candidates.push(cwd_pkg.clone()); +) -> Result<()> { + initial_pkgs_to_find_candidates.push(pkg.clone()); all_candidates - .entry((&cwd_pkg).into()) + .entry(pkg.into()) .or_default() - .insert((&cwd_pkg).into(), cwd_pkg.clone()); + .insert(pkg.into(), pkg.clone()); - Ok(cwd_pkg) + Ok(()) } pub async fn execute_cmd( @@ -187,35 +183,24 @@ pub async fn execute_cmd( let cwd = crate::fs::get_cwd()?; - // The package affected by tman install command, which is the root declared - // in the resolver. - // - // 1. If installing a package in standalone mode, the affected package is - // the installed package itself, ex: app or extension. - // - // 2. Otherwise, the affected package is always the app, as the installing - // package must be installed in a TEN app in this case. - let mut cwd_pkg_type = PkgType::App; - let cwd_pkg_name; - // The path of `.ten/app/` if we needed. let mut dot_ten_app_dir_path: Option = None; // If `tman install` is run within the scope of an app, then the app and // those addons (extensions, ...) installed in the app directory are all - // considered initial_input_pkgs. + // considered initial_pkgs_to_find_candidates. let mut initial_pkgs_to_find_candidates = vec![]; let mut all_candidates: HashMap< PkgTypeAndName, HashMap, > = HashMap::new(); - // 'all_existing_local_pkgs' contains all the packages which are already - // located in the `app` directory, including the `app` itself, and all the + // 'all_installed_pkgs' contains all the packages which are already located + // in the `app` directory, including the `app` itself, and all the // addons located in `ten_packages//` // // After the completed dependency tree is resolved, - // 'all_existing_local_pkgs' will be compared with the solver results: + // 'all_installed_pkgs' will be compared with the solver results: // // *) If some of these packages are not included in the dependency tree, // then users will be prompted that these packages can be added to the @@ -233,12 +218,21 @@ pub async fn execute_cmd( let mut dep_relationship_from_cmd_line: Option = None; - let mut app_pkg: Option = None; + let mut is_install_all = false; let mut installing_pkg_type: Option = None; let mut installing_pkg_name: Option = None; + let mut cwd_pkg = get_pkg_info_from_path(&cwd, true)?; + add_pkg_to_initial_pkg_to_find_candidates_and_all_candidates( + &cwd_pkg, + &mut initial_pkgs_to_find_candidates, + &mut all_candidates, + )?; + if let Some(package_type_str) = command_data.package_type.as_ref() { // Case 1: tman install + // + // The `cwd` must be the base directory of a TEN app. let installing_pkg_type_: PkgType = package_type_str.parse()?; @@ -254,16 +248,6 @@ pub async fn execute_cmd( installing_pkg_type = Some(installing_pkg_type_); installing_pkg_name = Some(installing_pkg_name_.clone()); - // The `cwd` must be the base directory of a TEN app. - let app_pkg_ = - add_cwd_pkg_to_initial_pkg_to_find_candidates_and_all_candidates( - &cwd, - &mut initial_pkgs_to_find_candidates, - &mut all_candidates, - )?; - - cwd_pkg_name = app_pkg_.basic_info.type_and_name.name.clone(); - all_installed_pkgs = tman_get_all_installed_pkgs_info_of_app(tman_config, &cwd)?; @@ -276,10 +260,10 @@ pub async fn execute_cmd( dep_relationship_from_cmd_line = Some(DependencyRelationship { type_and_name: PkgTypeAndName { - pkg_type: app_pkg_.basic_info.type_and_name.pkg_type, - name: app_pkg_.basic_info.type_and_name.name.clone(), + pkg_type: cwd_pkg.basic_info.type_and_name.pkg_type, + name: cwd_pkg.basic_info.type_and_name.name.clone(), }, - version: app_pkg_.basic_info.version.clone(), + version: cwd_pkg.basic_info.version.clone(), dependency: PkgDependency { type_and_name: PkgTypeAndName { pkg_type: installing_pkg_type_, @@ -290,28 +274,18 @@ pub async fn execute_cmd( base_dir: None, }, }); - - app_pkg = Some(app_pkg_); } else { // Case 2: tman install - let manifest = parse_manifest_in_folder(&cwd)?; - cwd_pkg_type = manifest.type_and_name.pkg_type; - cwd_pkg_name = manifest.type_and_name.name.clone(); + is_install_all = true; - match cwd_pkg_type { + match cwd_pkg.basic_info.type_and_name.pkg_type { PkgType::App => { // The TEN app itself is also a package. Extensions can declare // dependencies on a specific version of an app, so the app also // needs to be included in the package list for dependency tree // calculation. - add_cwd_pkg_to_initial_pkg_to_find_candidates_and_all_candidates( - &cwd, - &mut initial_pkgs_to_find_candidates, - &mut all_candidates, - )?; - all_installed_pkgs = tman_get_all_installed_pkgs_info_of_app(tman_config, &cwd)?; @@ -333,12 +307,6 @@ pub async fn execute_cmd( fs::create_dir_all(path)?; } - add_cwd_pkg_to_initial_pkg_to_find_candidates_and_all_candidates( - &cwd, - &mut initial_pkgs_to_find_candidates, - &mut all_candidates, - )?; - filter_compatible_pkgs_to_candidates( tman_config, &initial_pkgs_to_find_candidates, @@ -383,8 +351,8 @@ pub async fn execute_cmd( // Find an answer (a dependency tree) that satisfies all dependencies. let (usable_model, non_usable_models) = solve_all( tman_config, - &cwd_pkg_type, - &cwd_pkg_name, + &cwd_pkg.basic_info.type_and_name.pkg_type, + &cwd_pkg.basic_info.type_and_name.name, dep_relationship_from_cmd_line.as_ref(), &all_candidates, locked_pkgs.as_ref(), @@ -421,8 +389,8 @@ pub async fn execute_cmd( // this package already exists and does not need to be installed. let remaining_solver_results = filter_solver_results_by_type_and_name( &solver_results, - Some(&cwd_pkg_type), - Some(&cwd_pkg_name), + Some(&cwd_pkg.basic_info.type_and_name.pkg_type), + Some(&cwd_pkg.basic_info.type_and_name.name), false, )?; @@ -477,15 +445,16 @@ pub async fn execute_cmd( } // Write the installing package info to manifest.json. - if let Some(mut app_pkg) = app_pkg { - if installing_pkg_type.is_some() && installing_pkg_name.is_some() { - write_installing_pkg_into_manifest_file( - &mut app_pkg, - &solver_results, - &installing_pkg_type.unwrap(), - &installing_pkg_name.unwrap(), - )?; - } + if !is_install_all + && installing_pkg_type.is_some() + && installing_pkg_name.is_some() + { + write_installing_pkg_into_manifest_file( + &mut cwd_pkg, + &solver_results, + &installing_pkg_type.unwrap(), + &installing_pkg_name.unwrap(), + )?; } println!( diff --git a/core/src/ten_rust/tests/graph_check.rs b/core/src/ten_rust/tests/graph_check.rs index 5c3e59fd67..799dbfd00f 100644 --- a/core/src/ten_rust/tests/graph_check.rs +++ b/core/src/ten_rust/tests/graph_check.rs @@ -7,7 +7,7 @@ use std::{collections::HashMap, path::Path, str::FromStr}; use ten_rust::pkg_info::{ - get_all_existed_pkgs_info_of_app, graph::Graph, localhost, + get_all_installed_pkgs_info_of_app, graph::Graph, localhost, pkg_type::PkgType, property::predefined_graph::PredefinedGraph, PkgInfo, }; @@ -15,7 +15,7 @@ use ten_rust::pkg_info::{ fn test_graph_check_extension_not_installed_1() { let app_dir = "tests/test_data/graph_check_extension_not_installed_1"; let pkg_infos = - get_all_existed_pkgs_info_of_app(Path::new(app_dir)).unwrap(); + get_all_installed_pkgs_info_of_app(Path::new(app_dir)).unwrap(); assert!(!pkg_infos.is_empty()); let app_pkg_info = pkg_infos @@ -40,7 +40,7 @@ fn test_graph_check_extension_not_installed_1() { fn test_graph_check_extension_not_installed_2() { let app_dir = "tests/test_data/graph_check_extension_not_installed_2"; let pkg_infos = - get_all_existed_pkgs_info_of_app(Path::new(app_dir)).unwrap(); + get_all_installed_pkgs_info_of_app(Path::new(app_dir)).unwrap(); assert!(!pkg_infos.is_empty()); let app_pkg_info = pkg_infos @@ -65,7 +65,7 @@ fn test_graph_check_extension_not_installed_2() { fn test_graph_check_predefined_graph_success() { let app_dir = "tests/test_data/graph_check_predefined_graph_success"; let pkg_infos = - get_all_existed_pkgs_info_of_app(Path::new(app_dir)).unwrap(); + get_all_installed_pkgs_info_of_app(Path::new(app_dir)).unwrap(); assert!(!pkg_infos.is_empty()); let app_pkg_info = pkg_infos @@ -89,7 +89,7 @@ fn test_graph_check_predefined_graph_success() { fn test_graph_check_all_msgs_schema_incompatible() { let app_dir = "tests/test_data/graph_check_all_msgs_schema_incompatible"; let pkg_infos = - get_all_existed_pkgs_info_of_app(Path::new(app_dir)).unwrap(); + get_all_installed_pkgs_info_of_app(Path::new(app_dir)).unwrap(); assert!(!pkg_infos.is_empty()); let app_pkg_info = pkg_infos @@ -114,7 +114,7 @@ fn test_graph_check_all_msgs_schema_incompatible() { fn test_graph_check_single_app() { let app_dir = "tests/test_data/graph_check_single_app"; let pkg_infos = - get_all_existed_pkgs_info_of_app(Path::new(app_dir)).unwrap(); + get_all_installed_pkgs_info_of_app(Path::new(app_dir)).unwrap(); assert!(!pkg_infos.is_empty()); let graph_str = include_str!("test_data/graph_check_single_app/graph.json"); @@ -133,7 +133,7 @@ fn test_graph_check_single_app() { fn test_graph_check_single_app_schema_incompatible() { let app_dir = "tests/test_data/graph_check_single_app_schema_incompatible"; let pkg_infos = - get_all_existed_pkgs_info_of_app(Path::new(app_dir)).unwrap(); + get_all_installed_pkgs_info_of_app(Path::new(app_dir)).unwrap(); assert!(!pkg_infos.is_empty()); let graph_str = include_str!( From 8ebe060d4fecab0c1e3770824ee83d1e4f041779 Mon Sep 17 00:00:00 2001 From: Hu Yueh-Wei Date: Tue, 7 Jan 2025 12:11:08 +0800 Subject: [PATCH 4/4] feat: add support for local dependency in extension --- core/src/ten_manager/src/cmd/cmd_install.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/ten_manager/src/cmd/cmd_install.rs b/core/src/ten_manager/src/cmd/cmd_install.rs index c59b39bba6..f8793b08c6 100644 --- a/core/src/ten_manager/src/cmd/cmd_install.rs +++ b/core/src/ten_manager/src/cmd/cmd_install.rs @@ -223,6 +223,9 @@ pub async fn execute_cmd( let mut installing_pkg_name: Option = None; let mut cwd_pkg = get_pkg_info_from_path(&cwd, true)?; + + // We need to start looking for dependencies outward from the cwd package, + // and the cwd package itself is considered a candidate. add_pkg_to_initial_pkg_to_find_candidates_and_all_candidates( &cwd_pkg, &mut initial_pkgs_to_find_candidates,