Merge branch 'main' into feature/arm64

.github/workflows/build_linux_ubuntu1804.yml

@@ -33,6 +33,12 @@ jobs:
           fetch-depth: "0"
           submodules: "true"
 
+      - name: Update version
+        run: |
+          git config --global --add safe.directory $(pwd)
+          python3 tools/version/update_version_in_ten_framework.py
+          python3 tools/version/check_version_in_ten_framework.py
+
       - name: Install cpulimit and taskset
         run: |
           apt-get update
@@ -44,7 +50,7 @@ jobs:
           echo $PATH
           go env -w GOFLAGS="-buildvcs=false"
           rustup default nightly
-          tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=false log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_enable_package_manager=false
+          tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=false log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_enable_package_manager=false ten_rust_enable_gen_cargo_config=false
           tgn build linux x64 ${{ matrix.build_type }}
           tree -I 'gen|obj' out
 
@@ -106,7 +112,6 @@ jobs:
             out/linux/x64/ten_packages/system/ten_runtime
             out/linux/x64/ten_packages/system/ten_runtime_go
             out/linux/x64/ten_packages/system/ten_runtime_python
-            out/linux/x64/ten_packages/extension_group
             out/linux/x64/ten_packages/extension/default_extension_cpp
             out/linux/x64/ten_packages/extension/default_extension_go
             out/linux/x64/ten_packages/extension/default_extension_python
@@ -120,13 +125,12 @@ jobs:
             ten_packages/system/ten_runtime \
             ten_packages/system/ten_runtime_go \
             ten_packages/system/ten_runtime_python \
-            ten_packages/extension_group \
             ten_packages/extension/default_extension_cpp \
             ten_packages/extension/default_extension_go \
             ten_packages/extension/default_extension_python \
             ten_packages/extension/py_init_extension_cpp
 
-      - name: Release assets
+      - name: Publish to release assets
         uses: softprops/action-gh-release@v1
         if: startsWith(github.ref, 'refs/tags/')
         with:

.github/workflows/build_linux_ubuntu2204.yml

@@ -32,6 +32,12 @@ jobs:
           fetch-depth: "0"
           submodules: "true"
 
+      - name: Update version
+        run: |
+          git config --global --add safe.directory $(pwd)
+          python3 tools/version/update_version_in_ten_framework.py
+          python3 tools/version/check_version_in_ten_framework.py
+
       - name: Install cpulimit and taskset
         run: |
           apt-get update
@@ -44,7 +50,7 @@ jobs:
           go env -w GOFLAGS="-buildvcs=false"
           go1.20.12 download
           rustup default nightly
-          tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=${{ matrix.compiler == 'gcc' && 'false' || 'true' }} log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_package_manager_enable_tests=false
+          tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=${{ matrix.compiler == 'gcc' && 'false' || 'true' }} log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_package_manager_enable_tests=false ten_rust_enable_gen_cargo_config=false
           tgn build linux x64 ${{ matrix.build_type }}
           tree -I 'gen|obj' out
 
@@ -112,7 +118,6 @@ jobs:
             out/linux/x64/ten_packages/system/ten_runtime
             out/linux/x64/ten_packages/system/ten_runtime_go
             out/linux/x64/ten_packages/system/ten_runtime_python
-            out/linux/x64/ten_packages/extension_group
             out/linux/x64/ten_packages/extension/default_extension_cpp
             out/linux/x64/ten_packages/extension/default_extension_go
             out/linux/x64/ten_packages/extension/default_extension_python
@@ -127,16 +132,39 @@ jobs:
             ten_packages/system/ten_runtime \
             ten_packages/system/ten_runtime_go \
             ten_packages/system/ten_runtime_python \
-            ten_packages/extension_group \
             ten_packages/extension/default_extension_cpp \
             ten_packages/extension/default_extension_go \
             ten_packages/extension/default_extension_python \
             ten_packages/extension/py_init_extension_cpp
 
-      - name: Release assets
+      - name: Publish to release assets
         uses: softprops/action-gh-release@v2
         if: startsWith(github.ref, 'refs/tags/')
         with:
           files: |
             out/linux/x64/tman-linux-x64-${{ matrix.compiler }}-${{ matrix.build_type }}.zip
             out/linux/x64/ten_packages-linux-x64-${{ matrix.compiler }}-${{ matrix.build_type }}.zip
+
+      - name: Publish release to TEN cloud store
+        if: ${{ startsWith(github.ref, 'refs/tags/') && matrix.compiler == 'gcc' && matrix.build_type == 'release' }}
+        run: |
+          TMAN_BIN=$(pwd)/out/linux/x64/ten_manager/bin/tman
+
+          cd out/linux/x64/ten_packages
+          ARRAY=(
+            "system/ten_runtime"
+            "system/ten_runtime_go"
+            "system/ten_runtime_python"
+            "extension/default_extension_cpp"
+            "extension/default_extension_go"
+            "extension/default_extension_python"
+            "extension/py_init_extension_cpp"
+          )
+
+          for item in "${ARRAY[@]}"; do
+            echo $item
+            cd $item
+            ${TMAN_BIN} --verbose --user-token ${{ secrets.TEN_CLOUD_STORE }} publish
+            cd -
+          done
+        shell: bash

.github/workflows/build_mac.yml

@@ -76,7 +76,7 @@ jobs:
           fi
           export PATH=$(pwd)/core/ten_gn:$PATH
           echo $PATH
-          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false
+          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_rust_enable_gen_cargo_config=false
           tgn build mac arm64 ${{ matrix.build_type }}
           tree -I 'gen|obj' out
 
@@ -173,7 +173,7 @@ jobs:
           fi
           export PATH=$(pwd)/core/ten_gn:$PATH
           echo $PATH
-          tgn gen mac x64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_package_manager_enable_tests=false
+          tgn gen mac x64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_package_manager_enable_tests=false ten_rust_enable_gen_cargo_config=false
           tgn build mac x64 ${{ matrix.build_type }}
           tree -I 'gen|obj' out
 

.github/workflows/build_win.yml

@@ -49,7 +49,7 @@ jobs:
       - name: Build
         run: |
           $ENV:PATH += ";$PWD/core/ten_gn"
-          tgn gen win x64 ${{ matrix.build_type }} -- vs_version=2022 log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_package_manager_enable_tests=false
+          tgn gen win x64 ${{ matrix.build_type }} -- vs_version=2022 log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_package_manager_enable_tests=false ten_rust_enable_gen_cargo_config=false
           tgn build win x64 ${{ matrix.build_type }}
 
       - name: Run Tests (ten_utils_unit_test)

.gitignore

@@ -75,6 +75,7 @@ package-lock.json
 core/src/ten_manager/target/
 core/src/ten_rust/src/schema/bindings.rs
 core/src/ten_rust/target/
+/.cargo
 
 # private modules
 packages/private_apps

.vscode/settings.json

@@ -7,8 +7,13 @@
     "editor.wordWrapColumn": 80
   },
   "[python]": {
-    "editor.tabSize": 4
+    "editor.tabSize": 4,
+    "editor.defaultFormatter": "ms-python.black-formatter"
   },
+  "black-formatter.args": [
+    "--line-length",
+    "80"
+  ],
   "debug.allowBreakpointsEverywhere": true,
   "doxdocgen.c.commentPrefix": "/// ",
   "doxdocgen.c.firstLine": "///",

build/common/rust/rust.gni

@@ -43,12 +43,31 @@ template("rust_target") {
       _rustflags = invoker._rustflags
     }
 
-    if (is_debug) {
-      if (ten_rust_enable_asan) {
-        if (enable_sanitizer) {
-          _rustflags = "${_rustflags} -Zsanitizer=address"
-        }
+    if (!ten_rust_enable_gen_cargo_config && enable_sanitizer &&
+        ten_rust_enable_asan) {
+      asan_args = [
+        "--action",
+        "print",
+        "--compiler",
+      ]
+      if (is_clang) {
+        asan_args += [ "clang" ]
+      } else {
+        asan_args += [ "gcc" ]
       }
+
+      asan_args += [
+        "--target-os",
+        target_os,
+        "--target-arch",
+        target_cpu,
+      ]
+
+      flags = exec_script("//build/common/rust/rust_gen_cargo_config.py",
+                          asan_args,
+                          "trim string")
+
+      _rustflags = "${_rustflags} ${flags}"
     }
 
     if (_rustflags != "") {
@@ -209,12 +228,31 @@ template("rust_test") {
       _rustflags = invoker._rustflags
     }
 
-    if (is_debug) {
-      if (ten_rust_enable_asan) {
-        if (enable_sanitizer) {
-          _rustflags = "${_rustflags} -Zsanitizer=address"
-        }
+    if (!ten_rust_enable_gen_cargo_config && enable_sanitizer &&
+        ten_rust_enable_asan) {
+      asan_args = [
+        "--action",
+        "print",
+        "--compiler",
+      ]
+      if (is_clang) {
+        asan_args += [ "clang" ]
+      } else {
+        asan_args += [ "gcc" ]
       }
+
+      asan_args += [
+        "--target-os",
+        target_os,
+        "--target-arch",
+        target_cpu,
+      ]
+
+      flags = exec_script("//build/common/rust/rust_gen_cargo_config.py",
+                          asan_args,
+                          "trim string")
+
+      _rustflags = "${_rustflags} ${flags}"
     }
 
     if (_rustflags != "") {
@@ -353,3 +391,91 @@ template("rust_cbindgen") {
     outputs = [ _output ]
   }
 }
+
+template("rust_gen_cargo_config") {
+  assert(defined(invoker.project_root), "project_root is not defined")
+
+  _project_root = rebase_path(invoker.project_root)
+  _target_name = target_name
+  _target_path = target_gen_dir
+
+  action("${_target_name}") {
+    script = "//build/common/rust/rust_gen_cargo_config.py"
+    _output = "${_target_path}/${_target_name}_gen_cargo_config"
+
+    args = [
+      "--project-root",
+      _project_root,
+      "--tg-timestamp-proxy-file",
+      rebase_path(_output),
+    ]
+
+    args += [ "--compiler" ]
+    if (is_clang) {
+      args += [ "clang" ]
+    } else {
+      args += [ "gcc" ]
+    }
+
+    if (is_win) {
+      if (target_cpu == "x86") {
+        target = "i686-pc-windows-msvc"
+      } else if (target_cpu == "x64") {
+        target = "x86_64-pc-windows-msvc"
+      } else if (target_cpu == "arm64") {
+        target = "aarch64-pc-windows-msvc"
+      }
+    } else if (is_linux) {
+      if (target_cpu == "arm64") {
+        target = "aarch64-unknown-linux-gnu"
+      } else if (target_cpu == "arm") {
+        target = "armv7-unknown-linux-gnueabi"
+      } else if (target_cpu == "x86") {
+        target = "i686-unknown-linux-gnu"
+      } else if (target_cpu == "x64") {
+        target = "x86_64-unknown-linux-gnu"
+      }
+    } else if (is_mac) {
+      if (target_cpu == "arm64") {
+        target = "aarch64-apple-darwin"
+      } else if (target_cpu == "x86") {
+        target = "i686-apple-darwin"
+      } else if (target_cpu == "x64") {
+        target = "x86_64-apple-darwin"
+      }
+    }
+
+    args += [
+      "--target",
+      target,
+      "--target-os",
+      target_os,
+      "--target-arch",
+      target_cpu,
+    ]
+
+    if (enable_sanitizer && ten_rust_enable_gen_cargo_config) {
+      args += [
+        "--action",
+        "gen",
+      ]
+    } else {
+      # Delete `.cargo/config.toml` file to remove all configurations related to
+      # Rust + ASan.
+      args += [
+        "--action",
+        "delete",
+      ]
+    }
+
+    forward_variables_from(invoker,
+                           [
+                             "deps",
+                             "public_deps",
+                             "data_deps",
+                             "public_configs",
+                           ])
+
+    outputs = [ _output ]
+  }
+}