Merge branch 'main' into feat-fe-ext-conn-and-ops

.github/workflows/build_linux_arm64.yml

@@ -83,7 +83,7 @@ jobs:
               apt-get install -y nodejs && \
               rustup default nightly && \
               df -h . && \
-              tgn gen linux arm64 ${{ matrix.build_type }} -- is_clang=${{ matrix.compiler == 'gcc' && 'false' || 'true' }} log_level=1 enable_serialized_actions=true ten_enable_serialized_rust_action=true ten_enable_tests=false ten_rust_enable_tests=false ten_manager_enable_tests=false ten_enable_libwebsockets=false ten_enable_cargo_clean=true ten_enable_rust_incremental_build=false ten_manager_enable_frontend=false ten_enable_integration_tests_prebuilt=false && \
+              tgn gen linux arm64 ${{ matrix.build_type }} -- is_clang=${{ matrix.compiler == 'gcc' && 'false' || 'true' }} log_level=1 enable_serialized_actions=true ten_enable_serialized_rust_action=true ten_enable_tests=false ten_rust_enable_tests=false ten_manager_enable_tests=false ten_enable_libwebsockets=false ten_enable_cargo_clean=true ten_enable_rust_incremental_build=false ten_manager_enable_frontend=false ten_enable_integration_tests_prebuilt=false ten_enable_nodejs_binding=false && \
               tgn build linux arm64 ${{ matrix.build_type }} && \
               df -h . && \
               tree -I 'gen|obj' out \

.github/workflows/build_linux_ubuntu1804.yml

@@ -53,7 +53,7 @@ jobs:
           go env -w GOFLAGS="-buildvcs=false"
           rustup default nightly
           df -h .
-          tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=false log_level=1 enable_serialized_actions=true ten_enable_serialized_rust_action=true ten_enable_integration_tests=false ten_enable_ten_manager=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true ten_enable_rust_incremental_build=false ten_manager_enable_frontend=false ten_enable_integration_tests_prebuilt=false
+          tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=false log_level=1 enable_serialized_actions=true ten_enable_serialized_rust_action=true ten_enable_integration_tests=false ten_enable_ten_manager=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true ten_enable_rust_incremental_build=false ten_manager_enable_frontend=false ten_enable_integration_tests_prebuilt=false ten_enable_nodejs_binding=false
           tgn build linux x64 ${{ matrix.build_type }}
           df -h .
           tree -I 'gen|obj' out

.github/workflows/build_mac.yml

@@ -83,7 +83,7 @@ jobs:
 
           df -h .
 
-          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_serialized_rust_action=true ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true ten_enable_python_binding=false ten_enable_rust_incremental_build=false ten_manager_enable_frontend=false ten_enable_integration_tests_prebuilt=false
+          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_serialized_rust_action=true ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true ten_enable_python_binding=false ten_enable_rust_incremental_build=false ten_manager_enable_frontend=false ten_enable_integration_tests_prebuilt=false ten_enable_nodejs_binding=false
 
           tgn build mac arm64 ${{ matrix.build_type }}
 
@@ -253,7 +253,7 @@ jobs:
 
           df -h .
 
-          tgn gen mac x64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_serialized_rust_action=true ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true ten_enable_python_binding=false ten_enable_rust_incremental_build=false ten_manager_enable_frontend=false ten_enable_integration_tests_prebuilt=false
+          tgn gen mac x64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_serialized_rust_action=true ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true ten_enable_python_binding=false ten_enable_rust_incremental_build=false ten_manager_enable_frontend=false ten_enable_integration_tests_prebuilt=false ten_enable_nodejs_binding=false
 
           tgn build mac x64 ${{ matrix.build_type }}
 
@@ -424,7 +424,7 @@ jobs:
 
           df -h .
 
-          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_serialized_rust_action=true ten_rust_enable_gen_cargo_config=false ten_enable_ten_rust_apis=false ten_enable_rust_incremental_build=false ten_manager_enable_frontend=false ten_enable_integration_tests_prebuilt=false
+          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_serialized_rust_action=true ten_rust_enable_gen_cargo_config=false ten_enable_ten_rust_apis=false ten_enable_rust_incremental_build=false ten_manager_enable_frontend=false ten_enable_integration_tests_prebuilt=false ten_enable_nodejs_binding=false
 
           tgn build mac arm64 ${{ matrix.build_type }}
 

.github/workflows/build_win.yml

@@ -71,7 +71,7 @@ jobs:
       - name: Build
         run: |
           $ENV:PATH += ";$PWD/core/ten_gn"
-          tgn gen win x64 ${{ matrix.build_type }} -- vs_version=2022 log_level=1 enable_serialized_actions=true ten_enable_serialized_rust_action=true ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true ten_enable_python_binding=false ten_enable_go_binding=false ten_enable_rust_incremental_build=false ten_manager_enable_frontend=false ten_enable_integration_tests_prebuilt=false
+          tgn gen win x64 ${{ matrix.build_type }} -- vs_version=2022 log_level=1 enable_serialized_actions=true ten_enable_serialized_rust_action=true ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true ten_enable_python_binding=false ten_enable_go_binding=false ten_enable_nodejs_binding=false ten_enable_rust_incremental_build=false ten_manager_enable_frontend=false ten_enable_integration_tests_prebuilt=false
           tgn build win x64 ${{ matrix.build_type }}
 
       - name: Run Tests (ten_utils_unit_test)

.gitignore

@@ -57,6 +57,9 @@ out/
 node_modules/
 *.tsbuildinfo
 
+# Nodejs binding
+core/src/ten_runtime/binding/nodejs/build
+
 # Cache files
 .cache
 *.code-workspace
@@ -70,18 +73,11 @@ node_modules/
 # Clangd index file
 .clangd
 
-# rust generated files
+# Rust generated files
 core/src/ten_manager/target/
 core/src/ten_rust/src/schema/bindings.rs
 core/src/ten_rust/target/
 /.cargo
 
-# private modules
-packages/private_apps
-packages/private_extensions
-tests/private
-third_party/private
-build/private
-
 # zlib auto-generated files
 third_party/zlib/zconf.h.included

.vscode/launch.json

@@ -137,7 +137,7 @@
       "request": "launch",
       "program": "${workspaceFolder}/out/linux/x64/tests/standalone/ten_runtime_smoke_test",
       "args": [
-        "--gtest_filter=BasicTest.HelloWorld1"
+        "--gtest_filter=AudioFrameTest.FromJson"
       ],
       "cwd": "${workspaceFolder}/out/linux/x64/tests/standalone/",
       "env": {
@@ -388,10 +388,14 @@
       "type": "lldb",
       "request": "launch",
       "program": "${workspaceFolder}/out/linux/x64/ten_manager/bin/tman",
-      "cwd": "${workspaceFolder}/out/linux/x64/tests/ten_manager/install_all/install_all_in_local_file_system/test_app",
+      "cwd": "${workspaceFolder}/out/linux/x64/tests/ten_manager/install/mock_app/mock_app",
       "args": [
         "--config-file=${workspaceFolder}/out/linux/x64/tests/local_registry/config.json",
         "install",
+        "extension",
+        "ext_a",
+        "--os=linux",
+        "--arch=x64"
       ],
       "preRunCommands": [
         "script import pathlib;import subprocess;import lldb;rustc_sysroot = subprocess.getoutput(\"rustc --print sysroot\");rustlib_etc = pathlib.Path(rustc_sysroot) / \"lib\" / \"rustlib\" / \"etc\";lldb.debugger.HandleCommand(f'command script import \"{rustlib_etc / \"lldb_lookup.py\"}\"');lldb.debugger.HandleCommand(f'command source -s 0 \"{rustlib_etc / \"lldb_commands\"}\"')"
@@ -669,12 +673,13 @@
       "request": "launch",
       "program": "node",
       "args": [
-        "${workspaceFolder}/out/linux/x64/tests/ten_runtime/integration/nodejs/default_ui_app_ts_basic/default_ui_app_ts_basic_app/build/start.js"
+        "${workspaceFolder}/out/linux/x64/tests/ten_runtime/integration/nodejs/http_server_nodejs/http_server_nodejs_app/build/start.js"
       ],
-      "cwd": "${workspaceFolder}/out/linux/x64/tests/ten_runtime/integration/nodejs/default_ui_app_ts_basic/default_ui_app_ts_basic_app/",
+      "cwd": "${workspaceFolder}/out/linux/x64/tests/ten_runtime/integration/nodejs/http_server_nodejs/http_server_nodejs_app/",
       "env": {
-        "LD_PRELOAD": "lib/libasan.so",
-        "NODE_PATH": "lib:$NODE_PATH",
+        "LD_PRELOAD": "ten_packages/system/ten_runtime/lib/libasan.so",
+        "NODE_PATH": "ten_packages/system/ten_runtime_nodejs/lib:$NODE_PATH",
+        "LD_LIBRARY_PATH": "ten_packages/system/ten_runtime/lib",
       },
     },
     {
@@ -683,17 +688,26 @@
       "request": "launch",
       "program": "/usr/bin/node",
       "args": [
-        "${workspaceFolder}/out/linux/x64/tests/ten_runtime/integration/nodejs/access_app_property_store/access_app_property_store_app/build/start.js"
+        "--expose-gc",
+        "${workspaceFolder}/out/linux/x64/tests/ten_runtime/integration/nodejs/http_server_nodejs/http_server_nodejs_app/build/start.js"
       ],
-      "cwd": "${workspaceFolder}/out/linux/x64/tests/ten_runtime/integration/nodejs/access_app_property_store/access_app_property_store_app",
+      "cwd": "${workspaceFolder}/out/linux/x64/tests/ten_runtime/integration/nodejs/http_server_nodejs/http_server_nodejs_app/",
       "environment": [
         {
           "name": "LD_PRELOAD",
-          "value": "lib/libasan.so"
+          "value": "ten_packages/system/ten_runtime/lib/libasan.so"
         },
         {
           "name": "NODE_PATH",
-          "value": "lib:$NODE_PATH"
+          "value": "ten_packages/system/ten_runtime_nodejs/lib:$NODE_PATH"
+        },
+        {
+          "name": "LD_LIBRARY_PATH",
+          "value": "ten_packages/system/ten_runtime/lib"
+        },
+        {
+          "name": "LSAN_OPTIONS",
+          "value": "verbosity=1:log_threads=1"
         }
       ],
       "sourceFileMap": {

build/ten_runtime/feature/build_pkg.py

@@ -10,6 +10,7 @@
 import sys
 import platform
 from datetime import datetime
+import time
 from build.scripts import fs_utils, cmd_exec, timestamp_proxy
 
 
@@ -157,6 +158,110 @@ def build_go_app(args: ArgumentInfo) -> int:
     return returncode
 
 
+def npm_install() -> int:
+    # 'npm install' might be failed because of the bad network connection, and
+    # it might be stuck in a situation where it cannot be recovered. Therefore,
+    # the best way is to delete the whole node_modules/, and try again.
+    returncode = 0
+
+    for i in range(1, 10):
+        returncode, output = cmd_exec.run_cmd(
+            [
+                "npm",
+                "install",
+            ]
+        )
+
+        if returncode == 0:
+            break
+        else:
+            # Delete node_modules/ and try again.
+            print(
+                (
+                    f"Failed to 'npm install', output: {output}, "
+                    "deleting node_modules/ and try again."
+                )
+            )
+            fs_utils.remove_tree("node_modules")
+            time.sleep(5)
+    if returncode != 0:
+        print("Failed to 'npm install' after 10 times retries")
+
+    return returncode
+
+
+def build_nodejs_extensions(app_root_path: str) -> int:
+    origin_wd = os.getcwd()
+
+    extension_dir = os.path.join(app_root_path, "ten_packages/extension")
+
+    if os.path.exists(extension_dir):
+        for extension in os.listdir(extension_dir):
+            extension_path = os.path.join(extension_dir, extension)
+            if os.path.isdir(extension_path):
+                # If the extension is a typescript extension, build it.
+                if not os.path.exists(
+                    os.path.join(extension_path, "tsconfig.json")
+                ):
+                    continue
+
+                # Change to extension directory.
+                os.chdir(extension_path)
+
+                now = datetime.now()
+                cmd = ["npm", "run", "build"]
+                returncode, output = cmd_exec.run_cmd(cmd)
+                if returncode:
+                    print(
+                        (
+                            f"Failed to build extension {extension_path}: "
+                            f"{output}"
+                        )
+                    )
+                    return 1
+
+                duration = (datetime.now() - now).seconds
+                print(
+                    (
+                        f"Profiling ====> build ts extension({extension}) "
+                        f"costs {duration} seconds."
+                    )
+                )
+
+    os.chdir(origin_wd)
+
+    return 0
+
+
+def build_nodejs_app(args: ArgumentInfo) -> int:
+    t1 = datetime.now()
+
+    status_code = npm_install()
+    if status_code != 0:
+        return status_code
+
+    cmd = ["npm", "run", "build"]
+    status_code, output = cmd_exec.run_cmd(cmd)
+    if status_code != 0:
+        print(f"Failed to npm run build, output: {output}")
+        return status_code
+
+    t2 = datetime.now()
+    duration = (t2 - t1).seconds
+    print(
+        (
+            f"Profiling ====> build node app({args.pkg_name}) "
+            f"costs {duration} seconds."
+        )
+    )
+
+    status_code = build_nodejs_extensions(os.getcwd())
+    if status_code != 0:
+        print(f"Failed to build ts extensions of app {args.pkg_name}")
+
+    return status_code
+
+
 def get_pkg_type(pkg_root: str) -> str:
     manifest_path = os.path.join(pkg_root, "manifest.json")
     with open(manifest_path, "r") as f:
@@ -173,6 +278,8 @@ def build_app(args: ArgumentInfo) -> int:
         returncode = build_go_app(args)
     elif args.pkg_language == "python":
         returncode = 0
+    elif args.pkg_language == "nodejs":
+        returncode = build_nodejs_app(args)
     else:
         print(f"Unknown app language: {args.pkg_language}")
         returncode = 1

build/ten_runtime/feature/cmake.py

@@ -345,7 +345,7 @@ def build(self):
             + f" --target {self.args.project_name}"
         )
 
-        if self.args.target_os != "win":
+        if self.args.target_os != "win" and self.args.target_os != "mac":
             cmd += " --parallel $(nproc)"
 
         if self.args.log_level > 1:

build/ten_runtime/feature/install_pkg.py

@@ -159,12 +159,11 @@ def process_possible_published_results(file_paths: list[str]) -> list[str]:
                 args.src_pkg += f"@{versions[0]}"
 
         cmd += [
-            "install",
+            "create",
             args.pkg_type,
+            generated_app_name,
+            "--template",
             args.src_pkg,
-            "--template-mode",
-            "--template-data",
-            f"package_name={generated_app_name}",
         ]
 
         if args.build_type is not None: