From 8e68b759d76c90cabe7d4efed14ce0877de969d4 Mon Sep 17 00:00:00 2001 From: Hzj_jie Date: Tue, 24 Sep 2024 17:47:03 -0700 Subject: [PATCH 1/5] impl --- shell/platform/fuchsia/dart_runner/BUILD.gn | 6 - tools/fuchsia/copy_debug_symbols.py | 146 --------- tools/fuchsia/copy_path.py | 68 ----- tools/fuchsia/fuchsia_archive.gni | 313 ++++++-------------- tools/fuchsia/fuchsia_debug_symbols.gni | 73 ----- tools/fuchsia/gen_package.py | 132 --------- tools/fuchsia/gen_repo.py | 52 ---- 7 files changed, 86 insertions(+), 704 deletions(-) delete mode 100755 tools/fuchsia/copy_debug_symbols.py delete mode 100755 tools/fuchsia/copy_path.py delete mode 100644 tools/fuchsia/fuchsia_debug_symbols.gni delete mode 100755 tools/fuchsia/gen_package.py delete mode 100755 tools/fuchsia/gen_repo.py diff --git a/shell/platform/fuchsia/dart_runner/BUILD.gn b/shell/platform/fuchsia/dart_runner/BUILD.gn index e36ce92743002..1a626e1f7af47 100644 --- a/shell/platform/fuchsia/dart_runner/BUILD.gn +++ b/shell/platform/fuchsia/dart_runner/BUILD.gn @@ -190,12 +190,6 @@ template("aot_runner_package") { "//flutter/shell/platform/fuchsia/runtime/dart/profiler_symbols:dart_aot_runner", "target_gen_dir") + "/dart_aot_runner.dartprofilersymbols") - inputs = [ - vmservice_snapshot, - observatory_archive_file, - dart_profiler_symbols, - ] - resources += [ { path = vmservice_snapshot diff --git a/tools/fuchsia/copy_debug_symbols.py b/tools/fuchsia/copy_debug_symbols.py deleted file mode 100755 index cc5842f092ccb..0000000000000 --- a/tools/fuchsia/copy_debug_symbols.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright 2013 The Flutter Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -""" Gather the build_id, prefix_dir, and exec_name given the path to executable - also copies to the specified destination. - - The structure of debug symbols is as follows: - .build-id//[.debug] -""" - -import argparse -import errno -import hashlib -import json -import os -import re -import shutil -import subprocess -import sys -import time - - -def HashFile(filepath): - """Calculates the hash of a file without reading it all in memory at once.""" - digest = hashlib.sha1() - with open(filepath, 'rb') as f: - while True: - chunk = f.read(1024 * 1024) - if not chunk: - break - digest.update(chunk) - return digest.hexdigest() - - -def Touch(fname): - with open(fname, 'a'): - os.utime(fname, None) - - -def GetBuildIdParts(exec_path, read_elf): - sha1_pattern = re.compile(r'[0-9a-fA-F\-]+') - file_out = subprocess.check_output([read_elf, '-n', exec_path]) - build_id_line = file_out.splitlines()[-1].split() - if (build_id_line[0] != b'Build' or build_id_line[1] != b'ID:' or - not sha1_pattern.match(str(build_id_line[-1])) or not len(build_id_line[-1]) > 2): - raise Exception( - 'Expected the last line of llvm-readelf to match "Build ID " Got: %s' % file_out - ) - - build_id = build_id_line[-1] - return { - 'build_id': build_id.decode('utf-8'), 'prefix_dir': build_id[:2].decode('utf-8'), - 'exec_name': build_id[2:].decode('utf-8') - } - - -def main(): - parser = argparse.ArgumentParser() - - parser.add_argument( - '--executable-name', - dest='exec_name', - action='store', - required=True, - help='This is the name of the executable that we wish to layout debug symbols for.' - ) - parser.add_argument( - '--executable-path', - dest='exec_path', - action='store', - required=True, - help='Path to the executable on the filesystem.' - ) - parser.add_argument( - '--destination-base', - dest='dest', - action='store', - required=True, - help='Path to the base directory where the debug symbols are to be laid out.' - ) - parser.add_argument( - '--stripped', - dest='stripped', - action='store_true', - default=True, - help='Executable at the specified path is stripped.' - ) - parser.add_argument( - '--unstripped', - dest='stripped', - action='store_false', - help='Executable at the specified path is unstripped.' - ) - parser.add_argument( - '--read-elf', - dest='read_elf', - action='store', - required=True, - help='Path to read-elf executable.' - ) - - args = parser.parse_args() - assert os.path.exists(args.exec_path), ('exec_path "%s" does not exist' % args.exec_path) - assert os.path.exists(args.dest), ('dest "%s" does not exist' % args.dest) - assert os.path.exists(args.read_elf), ('read_elf "%s" does not exist' % args.read_elf) - - parts = GetBuildIdParts(args.exec_path, args.read_elf) - dbg_prefix_base = os.path.join(args.dest, parts['prefix_dir']) - - # Multiple processes may be trying to create the same directory. - # TODO(dnfield): use exist_ok when we upgrade to python 3, rather than try - try: - os.makedirs(dbg_prefix_base) - except OSError as e: - if e.errno != errno.EEXIST: - raise - - if not os.path.exists(dbg_prefix_base): - print('Unable to create directory: %s.' % dbg_prefix_base) - return 1 - - dbg_suffix = '' - if not args.stripped: - dbg_suffix = '.debug' - dbg_file_name = '%s%s' % (parts['exec_name'], dbg_suffix) - dbg_file_path = os.path.join(dbg_prefix_base, dbg_file_name) - - # If the debug file hasn't changed, don't rewrite the debug and completion - # file, speeding up incremental builds. - if os.path.exists(dbg_file_path) and HashFile(args.exec_path) == HashFile(dbg_file_path): - return 0 - - shutil.copyfile(args.exec_path, dbg_file_path) - - # Note this needs to be in sync with fuchsia_debug_symbols.gni - completion_file = os.path.join(args.dest, '.%s_dbg_success' % args.exec_name) - Touch(completion_file) - - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/tools/fuchsia/copy_path.py b/tools/fuchsia/copy_path.py deleted file mode 100755 index 7f46cb0918057..0000000000000 --- a/tools/fuchsia/copy_path.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright 2013 The Flutter Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -""" Copies paths, creates if they do not exist. -""" - -import argparse -import errno -import json -import os -import platform -import shutil -import subprocess -import sys - - -def EnsureParentExists(path): - dir_name, _ = os.path.split(path) - if not os.path.exists(dir_name): - os.makedirs(dir_name) - - -def SameStat(s1, s2): - return s1.st_ino == s2.st_ino and s1.st_dev == s2.st_dev - - -def SameFile(f1, f2): - if not os.path.exists(f2): - return False - s1 = os.stat(f1) - s2 = os.stat(f2) - return SameStat(s1, s2) - - -def CopyPath(src, dst): - try: - EnsureParentExists(dst) - shutil.copytree(src, dst) - except OSError as exc: - if exc.errno == errno.ENOTDIR: - if not SameFile(src, dst): - shutil.copyfile(src, dst) - else: - raise - - -def main(): - parser = argparse.ArgumentParser() - - parser.add_argument('--file-list', dest='file_list', action='store', required=True) - - args = parser.parse_args() - - files = open(args.file_list, 'r') - files_to_copy = files.read().split() - num_files = len(files_to_copy) // 2 - - for i in range(num_files): - CopyPath(files_to_copy[i], files_to_copy[num_files + i]) - - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/tools/fuchsia/fuchsia_archive.gni b/tools/fuchsia/fuchsia_archive.gni index d18a4c9ec8043..7d4d63a631cd4 100644 --- a/tools/fuchsia/fuchsia_archive.gni +++ b/tools/fuchsia/fuchsia_archive.gni @@ -2,247 +2,122 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import("//flutter/tools/fuchsia/fuchsia_debug_symbols.gni") import("//flutter/tools/fuchsia/fuchsia_libs.gni") import("//flutter/tools/fuchsia/gn-sdk/src/cmc.gni") +import("//flutter/tools/fuchsia/gn-sdk/src/component.gni") import("//flutter/tools/fuchsia/gn-sdk/src/gn_configs.gni") +import("//flutter/tools/fuchsia/gn-sdk/src/package.gni") -# Alias of cmc_compile in gn-sdk/src/cmc.gni -template("_compile_cml") { - assert(defined(invoker.manifest), "_compile_cml must define manifest") - - # Create an empty depfile, it's not used in flutter. - write_file("${target_gen_dir}/${target_name}/${target_name}.d", - [], - "list lines") - - cmc_compile(target_name) { - forward_variables_from(invoker, - [ - "deps", - "manifest", - "testonly", - ]) - output_file = invoker.output - } -} - -# TODO(zijiehe): May use fuchsia_package in gn-sdk if possible. - http://crbug.com/40935282 - -# Creates a Fuchsia archive (.far) file using PM from the Fuchsia SDK. +# Creates a Fuchsia archive (.far) file from the Fuchsia SDK and gn-sdk. # -# binary (required): -# The ELF binary for the archive's program. +# An archive combines an ELF binary and a component manifest to create +# a packaged Fuchsia program that can be deployed to a Fuchsia device. +# +# binary (optional): +# The ELF binary for the archive's program, or target_name if unspecified. # deps (optional): # The code dependencies for the archive. -# inputs (optional): -# When these files are changed, the package should be regenerated. # libraries (optional): # Paths to .so libraries that should be dynamically linked to the binary. +# manifest (optional): +# The component manifest cml file, or meta/binary.cml if unspecified. # resources (optional): # Files that should be placed into the `data/` directory of the archive. # testonly (optional): # Set this to true for archives that are only used for tests. -template("_fuchsia_archive") { - assert(defined(invoker.binary), "package must define binary") +template("fuchsia_archive") { + if (defined(invoker.binary)) { + _binary = invoker.binary + } else { + _binary = target_name + } + + if (defined(invoker.manifest)) { + _manifest = invoker.manifest + } else { + _manifest = rebase_path("meta/${_binary}.cml") + } - pkg_testonly = defined(invoker.testonly) && invoker.testonly - pkg_target_name = target_name - pkg = { - package_version = "0" # placeholder + _component_target = target_name + "__component" + fuchsia_component(_component_target) { forward_variables_from(invoker, [ - "binary", "deps", - "resources", - "libraries", + "testonly", ]) - if (!defined(package_name)) { - package_name = pkg_target_name - } - if (!defined(deps)) { - deps = [] - } - if (!defined(resources)) { - resources = [] - } - if (!defined(libraries)) { - libraries = [] - } - } - - far_base_dir = "$root_out_dir/${pkg_target_name}_far" - - copy_sources = [ "$root_out_dir/${invoker.binary}" ] - copy_outputs = [ "$far_base_dir/bin/app" ] - - foreach(res, pkg.resources) { - copy_sources += [ res.path ] - copy_outputs += [ "$far_base_dir/data/${res.dest}" ] - } + manifest = _manifest - foreach(lib, pkg.libraries) { - output_path = "" - - if (defined(lib.output_path)) { - output_path = lib.output_path + resources = [ + { + path = "$root_out_dir/$_binary" + dest = "bin/app" + }, + ] + if (defined(invoker.resources)) { + foreach(resource, invoker.resources) { + resources += [ + { + path = resource.path + dest = "data/${resource.dest}" + }, + ] + } } - copy_sources += [ "${lib.path}/${lib.name}" ] - copy_outputs += [ "$far_base_dir/lib/${output_path}${lib.name}" ] - } - - pkg_dir_deps = pkg.deps - - write_file("${far_base_dir}/meta/package", - { - name = pkg.package_name - version = pkg.package_version - }, - "json") - - _dbg_symbols_target = "${target_name}_dbg_symbols" - fuchsia_debug_symbols(_dbg_symbols_target) { - deps = pkg.deps - testonly = pkg_testonly - binary = invoker.binary - } - - action("${target_name}_dir") { - script = "//flutter/tools/fuchsia/copy_path.py" - sources = copy_sources - response_file_contents = rebase_path(copy_sources + copy_outputs) - deps = pkg_dir_deps - args = [ "--file-list={{response_file_name}}" ] - outputs = copy_outputs - testonly = pkg_testonly - } - - manifest_json_file = "${root_out_dir}/${target_name}_package_manifest.json" - action(target_name) { - script = "//flutter/tools/fuchsia/gen_package.py" - deps = pkg_dir_deps + [ - ":${target_name}_dir", - ":${_dbg_symbols_target}", - ] - - sources = copy_outputs - - inputs = [] - if (defined(invoker.inputs)) { - inputs += invoker.inputs + _libs = common_libs + if (defined(invoker.libraries)) { + _libs += invoker.libraries } - - args = [ - "--pm-bin", - rebase_path("$fuchsia_tool_dir/pm"), - "--package-dir", - rebase_path(far_base_dir), - "--far-name", - target_name, - "--manifest-json-file", - rebase_path(manifest_json_file, root_build_dir), - ] - - assert(fuchsia_target_api_level != -1, - "Must set a target api level when creating an archive") - if (fuchsia_target_api_level != -1) { - args += [ - "--api-level", - "${fuchsia_target_api_level}", + foreach(lib, _libs) { + output_path = "" + if (defined(lib.output_path)) { + output_path = lib.output_path + } + resources += [ + { + path = "${lib.path}/${lib.name}" + dest = "lib/${output_path}${lib.name}" + }, ] } - - outputs = [ - manifest_json_file, - "${far_base_dir}.manifest", - "$root_out_dir/${target_name}-0.far", - ] - testonly = pkg_testonly } -} -# Creates a Fuchsia archive. -# -# An archive combines an ELF binary and a component manifest to create -# a packaged Fuchsia program that can be deployed to a Fuchsia device. -# -# binary (optional): -# The ELF binary for the archive's program, or target_name if unspecified. -# deps (optional): -# The code dependencies for the archive. -# inputs (optional): -# When these files are changed, the package should be regenerated. -# libraries (optional): -# Paths to .so libraries that should be dynamically linked to the binary. -# resources (optional): -# Files that should be placed into the `data/` directory of the archive. -# testonly (optional): -# Set this to true for archives that are only used for tests. -template("fuchsia_archive") { - if (!defined(invoker.binary)) { - _binary = target_name - } else { - _binary = invoker.binary - } - _deps = [] - if (defined(invoker.deps)) { - _deps += invoker.deps - } - - _cml_file = rebase_path("meta/${_binary}.cml") - _far_base_dir = "$root_out_dir/${target_name}_far" - _cml_file_name = get_path_info(_cml_file, "name") - _compile_cml_target = "${target_name}_${_cml_file_name}_compile_cml" - - _compile_cml(_compile_cml_target) { + _package_target = target_name + "__package" + _package_name = target_name + fuchsia_package(_package_target) { forward_variables_from(invoker, [ "testonly" ]) - - manifest = _cml_file - output = "$_far_base_dir/meta/${_cml_file_name}.cm" + package_name = _package_name + deps = [ ":$_component_target" ] } - _deps += [ ":$_compile_cml_target" ] - _fuchsia_archive(target_name) { - deps = _deps - binary = _binary - forward_variables_from(invoker, - [ - "inputs", - "libraries", - "resources", - "testonly", - ]) + # TODO(zijiehe): http://crbug.com/368608542, copying the far files is not very + # necessary, try to remove the -0.far copy. + copy(target_name) { + forward_variables_from(invoker, [ "testonly" ]) + package_output_dir = get_label_info(":$_package_target", "target_gen_dir") + sources = [ "$package_output_dir/$_package_name/${_package_name}.far" ] + outputs = [ "$root_out_dir/${_package_name}-0.far" ] + deps = [ ":$_package_target" ] } } # Creates a Fuchsia archive (.far) file containing a generated test root -# component and test driver component, using PM from the Fuchsia SDK. +# component and test driver component. # -# binary (optional): -# The binary for the test, or target_name if unspecified. +# binary: +# Forward to fuchsia_archive # deps (required): # Dependencies for the test archive. # gen_cml_file (optional): # If is defined and true, an interpolate cml file will be generated. -# libraries (optional): -# Paths to .so libraries that should be dynamically linked to the binary. -# resources (optional): -# Files that should be placed into the `data/` directory of the archive. +# libraries: +# Forward to fuchsia_archive +# resources: +# Forward to fuchsia_archive template("fuchsia_test_archive") { assert(defined(invoker.deps), "package must define deps") - if (!defined(invoker.binary)) { - _binary = target_name - } else { - _binary = invoker.binary - } - _deps = [] - if (defined(invoker.deps)) { - _deps += invoker.deps - } - - _generated_cml = defined(invoker.gen_cml_file) && invoker.gen_cml_file - if (_generated_cml) { + _deps = invoker.deps + if (defined(invoker.gen_cml_file) && invoker.gen_cml_file) { _cml_file = "$root_out_dir/${target_name}.cml" _interpolate_cml_target = "${target_name}_interpolate_cml" action(_interpolate_cml_target) { @@ -259,36 +134,20 @@ template("fuchsia_test_archive") { ] outputs = [ _cml_file ] } - } else { - _cml_file = rebase_path("meta/${_binary}.cml") - } - - _far_base_dir = "$root_out_dir/${target_name}_far" - _cml_file_name = get_path_info(_cml_file, "name") - _compile_cml_target = "${target_name}_${_cml_file_name}_compile_cml" - - _compile_cml(_compile_cml_target) { - testonly = true - - manifest = _cml_file - output = "$_far_base_dir/meta/${_cml_file_name}.cm" - - if (_generated_cml) { - deps = [ ":$_interpolate_cml_target" ] - } + _deps += [ ":$_interpolate_cml_target" ] } - _deps += [ ":$_compile_cml_target" ] - _fuchsia_archive(target_name) { + fuchsia_archive(target_name) { + forward_variables_from(invoker, + [ + "binary", + "libraries", + "resources", + ]) testonly = true - binary = _binary - forward_variables_from(invoker, [ "resources" ]) - - libraries = common_libs - if (defined(invoker.libraries)) { - libraries += invoker.libraries + if (defined(_cml_file)) { + manifest = _cml_file } - deps = _deps } } diff --git a/tools/fuchsia/fuchsia_debug_symbols.gni b/tools/fuchsia/fuchsia_debug_symbols.gni deleted file mode 100644 index ad18d3d9d349b..0000000000000 --- a/tools/fuchsia/fuchsia_debug_symbols.gni +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2013 The Flutter Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import("//build/toolchain/toolchain.gni") -import("//flutter/common/fuchsia_config.gni") - -# The inputs to this template are 'binary_path' and a boolean 'unstripped'. -# If 'unstripped' is specified, we append '.debug' to the symbols name. -template("_copy_debug_symbols") { - assert(defined(invoker.binary_path), "'binary_path' needs to be defined.") - assert(defined(invoker.unstripped), "'unstripped' needs to be defined.") - - action(target_name) { - forward_variables_from(invoker, - [ - "deps", - "unstripped", - "binary_path", - "testonly", - ]) - - script = "//flutter/tools/fuchsia/copy_debug_symbols.py" - - sources = [ binary_path ] - - _dest_base = "${root_out_dir}/.build-id" - - args = [ - "--executable-name", - target_name, - "--executable-path", - rebase_path(binary_path), - "--destination-base", - rebase_path(_dest_base), - "--read-elf", - rebase_path( - "$buildtools_path/${host_os}-${host_cpu}/clang/bin/llvm-readelf"), - ] - - if (unstripped) { - args += [ "--unstripped" ] - } - - outputs = [ "${_dest_base}/.${target_name}_success" ] - } -} - -# Takes a binary and generates its debug symbols following -# the Fuchsia packaging convention. -template("fuchsia_debug_symbols") { - assert(defined(invoker.binary), "'binary' needs to be defined.") - - _copy_debug_symbols("_${target_name}_stripped") { - forward_variables_from(invoker, "*") - binary_path = rebase_path("${root_out_dir}/$binary") - unstripped = false - } - - _copy_debug_symbols("_${target_name}_unstripped") { - forward_variables_from(invoker, "*") - binary_path = "${root_out_dir}/exe.unstripped/$binary" - unstripped = true - } - - group(target_name) { - forward_variables_from(invoker, [ "testonly" ]) - deps = [ - ":_${target_name}_stripped", - ":_${target_name}_unstripped", - ] - } -} diff --git a/tools/fuchsia/gen_package.py b/tools/fuchsia/gen_package.py deleted file mode 100755 index 5ca757d845121..0000000000000 --- a/tools/fuchsia/gen_package.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright 2013 The Flutter Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -""" Generate a Fuchsia FAR Archive from an asset manifest. -""" - -import argparse -import collections -import json -import os -import subprocess -import sys - -from gather_flutter_runner_artifacts import CreateMetaPackage - - -# Generates the manifest and returns the file. -def GenerateManifest(package_dir): - full_paths = [] - for root, dirs, files in os.walk(package_dir): - for f in files: - common_prefix = os.path.commonprefix([root, package_dir]) - rel_path = os.path.relpath(os.path.join(root, f), common_prefix) - from_package = os.path.abspath(os.path.join(package_dir, rel_path)) - assert from_package, 'Failed to create from_package for %s' % os.path.join(root, f) - full_paths.append('%s=%s' % (rel_path, from_package)) - - parent_dir = os.path.abspath(os.path.join(package_dir, os.pardir)) - manifest_file_name = os.path.basename(package_dir) + '.manifest' - manifest_path = os.path.join(parent_dir, manifest_file_name) - with open(manifest_path, 'w') as f: - for item in full_paths: - f.write("%s\n" % item) - return manifest_path - - -def CreateFarPackage(pm_bin, package_dir, signing_key, dst_dir, api_level): - manifest_path = GenerateManifest(package_dir) - - pm_command_base = [ - pm_bin, '-m', manifest_path, '-k', signing_key, '-o', dst_dir, '--api-level', api_level - ] - - # Build the package - subprocess.check_output(pm_command_base + ['build']) - - # Archive the package - subprocess.check_output(pm_command_base + ['archive']) - - return 0 - - -def main(): - parser = argparse.ArgumentParser() - - parser.add_argument('--pm-bin', dest='pm_bin', action='store', required=True) - parser.add_argument('--package-dir', dest='package_dir', action='store', required=True) - parser.add_argument('--manifest-file', dest='manifest_file', action='store', required=False) - parser.add_argument( - '--manifest-json-file', dest='manifest_json_file', action='store', required=True - ) - parser.add_argument('--far-name', dest='far_name', action='store', required=False) - parser.add_argument('--api-level', dest='api_level', action='store', required=False) - - args = parser.parse_args() - - assert os.path.exists(args.pm_bin) - assert os.path.exists(args.package_dir) - pkg_dir = args.package_dir - - if not os.path.exists(os.path.join(pkg_dir, 'meta', 'package')): - CreateMetaPackage(pkg_dir, args.far_name) - - output_dir = os.path.abspath(pkg_dir + '_out') - if not os.path.exists(output_dir): - os.makedirs(output_dir) - - manifest_file = None - if args.manifest_file is not None: - assert os.path.exists(args.manifest_file) - manifest_file = args.manifest_file - else: - manifest_file = GenerateManifest(args.package_dir) - - pm_command_base = [ - args.pm_bin, - '-o', - output_dir, - '-n', - args.far_name, - '-m', - manifest_file, - ] - - # Build and then archive the package - # Use check_output so if anything goes wrong we get the output. - try: - - build_command = ['build', '--output-package-manifest', args.manifest_json_file] - - if args.api_level is not None: - build_command = ['--api-level', args.api_level] + build_command - - archive_command = [ - 'archive', '--output=' + os.path.join(os.path.dirname(output_dir), args.far_name + "-0") - ] - - pm_commands = [build_command, archive_command] - - for pm_command in pm_commands: - subprocess.check_output(pm_command_base + pm_command) - except subprocess.CalledProcessError as e: - print('==================== Manifest contents =========================================') - with open(manifest_file, 'r') as manifest: - sys.stdout.write(manifest.read()) - print('==================== End manifest contents =====================================') - meta_contents_path = os.path.join(output_dir, 'meta', 'contents') - if os.path.exists(meta_contents_path): - print('==================== meta/contents =============================================') - with open(meta_contents_path, 'r') as meta_contents: - sys.stdout.write(meta_contents.read()) - print('==================== End meta/contents =========================================') - raise - - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/tools/fuchsia/gen_repo.py b/tools/fuchsia/gen_repo.py deleted file mode 100755 index b795254b8df06..0000000000000 --- a/tools/fuchsia/gen_repo.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright 2013 The Flutter Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -""" Generate a Fuchsia repo capable of serving Fuchsia archives over the -network. -""" -import argparse -import collections -import json -import os -import subprocess -import sys - - -def main(): - parser = argparse.ArgumentParser() - - parser.add_argument('--pm-bin', dest='pm_bin', action='store', required=True) - parser.add_argument('--repo-dir', dest='repo_dir', action='store', required=True) - parser.add_argument('--archive', dest='archives', action='append', required=True) - - args = parser.parse_args() - - assert os.path.exists(args.pm_bin) - - if not os.path.exists(args.repo_dir): - pm_newrepo_command = [args.pm_bin, 'newrepo', '-repo', args.repo_dir] - subprocess.check_call(pm_newrepo_command) - - pm_publish_command = [ - args.pm_bin, - 'publish', - '-C', # Remove all previous registrations. - '-a', # Publish archives from an archive (mode). - '-repo', - args.repo_dir - ] - - for archive in args.archives: - pm_publish_command.append('-f') - pm_publish_command.append(archive) - - subprocess.check_call(pm_publish_command) - - return 0 - - -if __name__ == '__main__': - sys.exit(main()) From 319e85853b6303cfa93784b044b1d69eafa2b4b7 Mon Sep 17 00:00:00 2001 From: Hzj_jie Date: Thu, 10 Oct 2024 10:15:05 -0700 Subject: [PATCH 2/5] TODO --- tools/fuchsia/build_fuchsia_artifacts.py | 2 + tools/fuchsia/gen_package.py | 132 +++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100755 tools/fuchsia/gen_package.py diff --git a/tools/fuchsia/build_fuchsia_artifacts.py b/tools/fuchsia/build_fuchsia_artifacts.py index 5258466145387..3a30ad47bcd39 100755 --- a/tools/fuchsia/build_fuchsia_artifacts.py +++ b/tools/fuchsia/build_fuchsia_artifacts.py @@ -146,6 +146,8 @@ def CopyZirconFFILibIfExists(source, destination): FindFileAndCopyTo('libzircon_ffi.so', source_root, destination_base) +# TODO(zijiehe): http://crbug.com/368608542, avoid using pm or building far +# packages here, packages should be built by ninja. def CopyToBucketWithMode(source, destination, aot, product, runner_type, api_level): mode = 'aot' if aot else 'jit' product_suff = '_product' if product else '' diff --git a/tools/fuchsia/gen_package.py b/tools/fuchsia/gen_package.py new file mode 100755 index 0000000000000..5ca757d845121 --- /dev/null +++ b/tools/fuchsia/gen_package.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" Generate a Fuchsia FAR Archive from an asset manifest. +""" + +import argparse +import collections +import json +import os +import subprocess +import sys + +from gather_flutter_runner_artifacts import CreateMetaPackage + + +# Generates the manifest and returns the file. +def GenerateManifest(package_dir): + full_paths = [] + for root, dirs, files in os.walk(package_dir): + for f in files: + common_prefix = os.path.commonprefix([root, package_dir]) + rel_path = os.path.relpath(os.path.join(root, f), common_prefix) + from_package = os.path.abspath(os.path.join(package_dir, rel_path)) + assert from_package, 'Failed to create from_package for %s' % os.path.join(root, f) + full_paths.append('%s=%s' % (rel_path, from_package)) + + parent_dir = os.path.abspath(os.path.join(package_dir, os.pardir)) + manifest_file_name = os.path.basename(package_dir) + '.manifest' + manifest_path = os.path.join(parent_dir, manifest_file_name) + with open(manifest_path, 'w') as f: + for item in full_paths: + f.write("%s\n" % item) + return manifest_path + + +def CreateFarPackage(pm_bin, package_dir, signing_key, dst_dir, api_level): + manifest_path = GenerateManifest(package_dir) + + pm_command_base = [ + pm_bin, '-m', manifest_path, '-k', signing_key, '-o', dst_dir, '--api-level', api_level + ] + + # Build the package + subprocess.check_output(pm_command_base + ['build']) + + # Archive the package + subprocess.check_output(pm_command_base + ['archive']) + + return 0 + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument('--pm-bin', dest='pm_bin', action='store', required=True) + parser.add_argument('--package-dir', dest='package_dir', action='store', required=True) + parser.add_argument('--manifest-file', dest='manifest_file', action='store', required=False) + parser.add_argument( + '--manifest-json-file', dest='manifest_json_file', action='store', required=True + ) + parser.add_argument('--far-name', dest='far_name', action='store', required=False) + parser.add_argument('--api-level', dest='api_level', action='store', required=False) + + args = parser.parse_args() + + assert os.path.exists(args.pm_bin) + assert os.path.exists(args.package_dir) + pkg_dir = args.package_dir + + if not os.path.exists(os.path.join(pkg_dir, 'meta', 'package')): + CreateMetaPackage(pkg_dir, args.far_name) + + output_dir = os.path.abspath(pkg_dir + '_out') + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + manifest_file = None + if args.manifest_file is not None: + assert os.path.exists(args.manifest_file) + manifest_file = args.manifest_file + else: + manifest_file = GenerateManifest(args.package_dir) + + pm_command_base = [ + args.pm_bin, + '-o', + output_dir, + '-n', + args.far_name, + '-m', + manifest_file, + ] + + # Build and then archive the package + # Use check_output so if anything goes wrong we get the output. + try: + + build_command = ['build', '--output-package-manifest', args.manifest_json_file] + + if args.api_level is not None: + build_command = ['--api-level', args.api_level] + build_command + + archive_command = [ + 'archive', '--output=' + os.path.join(os.path.dirname(output_dir), args.far_name + "-0") + ] + + pm_commands = [build_command, archive_command] + + for pm_command in pm_commands: + subprocess.check_output(pm_command_base + pm_command) + except subprocess.CalledProcessError as e: + print('==================== Manifest contents =========================================') + with open(manifest_file, 'r') as manifest: + sys.stdout.write(manifest.read()) + print('==================== End manifest contents =====================================') + meta_contents_path = os.path.join(output_dir, 'meta', 'contents') + if os.path.exists(meta_contents_path): + print('==================== meta/contents =============================================') + with open(meta_contents_path, 'r') as meta_contents: + sys.stdout.write(meta_contents.read()) + print('==================== End meta/contents =========================================') + raise + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) From 6f99e2fe059ad591420d6917cd1e67f0279a1a51 Mon Sep 17 00:00:00 2001 From: Hzj_jie Date: Thu, 10 Oct 2024 14:41:22 -0700 Subject: [PATCH 3/5] go --- tools/fuchsia/fuchsia_archive.gni | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/fuchsia/fuchsia_archive.gni b/tools/fuchsia/fuchsia_archive.gni index 7d4d63a631cd4..68d76c86eb6c7 100644 --- a/tools/fuchsia/fuchsia_archive.gni +++ b/tools/fuchsia/fuchsia_archive.gni @@ -3,6 +3,7 @@ # found in the LICENSE file. import("//flutter/tools/fuchsia/fuchsia_libs.gni") +import("//flutter/tools/fuchsia/gn-sdk/src/build_id_dir.gni") import("//flutter/tools/fuchsia/gn-sdk/src/cmc.gni") import("//flutter/tools/fuchsia/gn-sdk/src/component.gni") import("//flutter/tools/fuchsia/gn-sdk/src/gn_configs.gni") @@ -90,6 +91,13 @@ template("fuchsia_archive") { deps = [ ":$_component_target" ] } + _build_dir_target = target_name + "__build_id" + build_id_dir(_build_dir_target) { + forward_variables_from(invoker, [ "testonly" ]) + output_path = "$target_gen_dir/$_package_name/.build_id" + deps = [ ":$_package_target" ] + } + # TODO(zijiehe): http://crbug.com/368608542, copying the far files is not very # necessary, try to remove the -0.far copy. copy(target_name) { @@ -97,7 +105,8 @@ template("fuchsia_archive") { package_output_dir = get_label_info(":$_package_target", "target_gen_dir") sources = [ "$package_output_dir/$_package_name/${_package_name}.far" ] outputs = [ "$root_out_dir/${_package_name}-0.far" ] - deps = [ ":$_package_target" ] + deps = [ ":$_package_target", + ":$_build_dir_target", ] } } From ed7aad861e99e9c27343e6f30b637dbf81022f48 Mon Sep 17 00:00:00 2001 From: Hzj_jie Date: Thu, 10 Oct 2024 15:23:19 -0700 Subject: [PATCH 4/5] debug --- tools/fuchsia/copy_debug_symbols.py | 146 ++++++++++++++++++++++++ tools/fuchsia/fuchsia_archive.gni | 22 ++-- tools/fuchsia/fuchsia_debug_symbols.gni | 73 ++++++++++++ 3 files changed, 234 insertions(+), 7 deletions(-) create mode 100755 tools/fuchsia/copy_debug_symbols.py create mode 100644 tools/fuchsia/fuchsia_debug_symbols.gni diff --git a/tools/fuchsia/copy_debug_symbols.py b/tools/fuchsia/copy_debug_symbols.py new file mode 100755 index 0000000000000..cc5842f092ccb --- /dev/null +++ b/tools/fuchsia/copy_debug_symbols.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +# +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" Gather the build_id, prefix_dir, and exec_name given the path to executable + also copies to the specified destination. + + The structure of debug symbols is as follows: + .build-id//[.debug] +""" + +import argparse +import errno +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import time + + +def HashFile(filepath): + """Calculates the hash of a file without reading it all in memory at once.""" + digest = hashlib.sha1() + with open(filepath, 'rb') as f: + while True: + chunk = f.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest() + + +def Touch(fname): + with open(fname, 'a'): + os.utime(fname, None) + + +def GetBuildIdParts(exec_path, read_elf): + sha1_pattern = re.compile(r'[0-9a-fA-F\-]+') + file_out = subprocess.check_output([read_elf, '-n', exec_path]) + build_id_line = file_out.splitlines()[-1].split() + if (build_id_line[0] != b'Build' or build_id_line[1] != b'ID:' or + not sha1_pattern.match(str(build_id_line[-1])) or not len(build_id_line[-1]) > 2): + raise Exception( + 'Expected the last line of llvm-readelf to match "Build ID " Got: %s' % file_out + ) + + build_id = build_id_line[-1] + return { + 'build_id': build_id.decode('utf-8'), 'prefix_dir': build_id[:2].decode('utf-8'), + 'exec_name': build_id[2:].decode('utf-8') + } + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument( + '--executable-name', + dest='exec_name', + action='store', + required=True, + help='This is the name of the executable that we wish to layout debug symbols for.' + ) + parser.add_argument( + '--executable-path', + dest='exec_path', + action='store', + required=True, + help='Path to the executable on the filesystem.' + ) + parser.add_argument( + '--destination-base', + dest='dest', + action='store', + required=True, + help='Path to the base directory where the debug symbols are to be laid out.' + ) + parser.add_argument( + '--stripped', + dest='stripped', + action='store_true', + default=True, + help='Executable at the specified path is stripped.' + ) + parser.add_argument( + '--unstripped', + dest='stripped', + action='store_false', + help='Executable at the specified path is unstripped.' + ) + parser.add_argument( + '--read-elf', + dest='read_elf', + action='store', + required=True, + help='Path to read-elf executable.' + ) + + args = parser.parse_args() + assert os.path.exists(args.exec_path), ('exec_path "%s" does not exist' % args.exec_path) + assert os.path.exists(args.dest), ('dest "%s" does not exist' % args.dest) + assert os.path.exists(args.read_elf), ('read_elf "%s" does not exist' % args.read_elf) + + parts = GetBuildIdParts(args.exec_path, args.read_elf) + dbg_prefix_base = os.path.join(args.dest, parts['prefix_dir']) + + # Multiple processes may be trying to create the same directory. + # TODO(dnfield): use exist_ok when we upgrade to python 3, rather than try + try: + os.makedirs(dbg_prefix_base) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + if not os.path.exists(dbg_prefix_base): + print('Unable to create directory: %s.' % dbg_prefix_base) + return 1 + + dbg_suffix = '' + if not args.stripped: + dbg_suffix = '.debug' + dbg_file_name = '%s%s' % (parts['exec_name'], dbg_suffix) + dbg_file_path = os.path.join(dbg_prefix_base, dbg_file_name) + + # If the debug file hasn't changed, don't rewrite the debug and completion + # file, speeding up incremental builds. + if os.path.exists(dbg_file_path) and HashFile(args.exec_path) == HashFile(dbg_file_path): + return 0 + + shutil.copyfile(args.exec_path, dbg_file_path) + + # Note this needs to be in sync with fuchsia_debug_symbols.gni + completion_file = os.path.join(args.dest, '.%s_dbg_success' % args.exec_name) + Touch(completion_file) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/fuchsia/fuchsia_archive.gni b/tools/fuchsia/fuchsia_archive.gni index 68d76c86eb6c7..e42aef06905f9 100644 --- a/tools/fuchsia/fuchsia_archive.gni +++ b/tools/fuchsia/fuchsia_archive.gni @@ -2,6 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("//flutter/tools/fuchsia/fuchsia_debug_symbols.gni") import("//flutter/tools/fuchsia/fuchsia_libs.gni") import("//flutter/tools/fuchsia/gn-sdk/src/build_id_dir.gni") import("//flutter/tools/fuchsia/gn-sdk/src/cmc.gni") @@ -91,13 +92,18 @@ template("fuchsia_archive") { deps = [ ":$_component_target" ] } - _build_dir_target = target_name + "__build_id" - build_id_dir(_build_dir_target) { - forward_variables_from(invoker, [ "testonly" ]) - output_path = "$target_gen_dir/$_package_name/.build_id" - deps = [ ":$_package_target" ] + _build_id_target = target_name + "__build_id" + fuchsia_debug_symbols(_build_id_target) { + forward_variables_from(invoker, [ "deps", "testonly", ]) + binary = _binary } + #build_id_dir(_build_id_target) { + # forward_variables_from(invoker, [ "testonly" ]) + # output_path = "$target_gen_dir/$_package_name/.build-id" + # deps = [ ":$_package_target" ] + #} + # TODO(zijiehe): http://crbug.com/368608542, copying the far files is not very # necessary, try to remove the -0.far copy. copy(target_name) { @@ -105,8 +111,10 @@ template("fuchsia_archive") { package_output_dir = get_label_info(":$_package_target", "target_gen_dir") sources = [ "$package_output_dir/$_package_name/${_package_name}.far" ] outputs = [ "$root_out_dir/${_package_name}-0.far" ] - deps = [ ":$_package_target", - ":$_build_dir_target", ] + deps = [ + ":$_build_id_target", + ":$_package_target", + ] } } diff --git a/tools/fuchsia/fuchsia_debug_symbols.gni b/tools/fuchsia/fuchsia_debug_symbols.gni new file mode 100644 index 0000000000000..ad18d3d9d349b --- /dev/null +++ b/tools/fuchsia/fuchsia_debug_symbols.gni @@ -0,0 +1,73 @@ +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build/toolchain/toolchain.gni") +import("//flutter/common/fuchsia_config.gni") + +# The inputs to this template are 'binary_path' and a boolean 'unstripped'. +# If 'unstripped' is specified, we append '.debug' to the symbols name. +template("_copy_debug_symbols") { + assert(defined(invoker.binary_path), "'binary_path' needs to be defined.") + assert(defined(invoker.unstripped), "'unstripped' needs to be defined.") + + action(target_name) { + forward_variables_from(invoker, + [ + "deps", + "unstripped", + "binary_path", + "testonly", + ]) + + script = "//flutter/tools/fuchsia/copy_debug_symbols.py" + + sources = [ binary_path ] + + _dest_base = "${root_out_dir}/.build-id" + + args = [ + "--executable-name", + target_name, + "--executable-path", + rebase_path(binary_path), + "--destination-base", + rebase_path(_dest_base), + "--read-elf", + rebase_path( + "$buildtools_path/${host_os}-${host_cpu}/clang/bin/llvm-readelf"), + ] + + if (unstripped) { + args += [ "--unstripped" ] + } + + outputs = [ "${_dest_base}/.${target_name}_success" ] + } +} + +# Takes a binary and generates its debug symbols following +# the Fuchsia packaging convention. +template("fuchsia_debug_symbols") { + assert(defined(invoker.binary), "'binary' needs to be defined.") + + _copy_debug_symbols("_${target_name}_stripped") { + forward_variables_from(invoker, "*") + binary_path = rebase_path("${root_out_dir}/$binary") + unstripped = false + } + + _copy_debug_symbols("_${target_name}_unstripped") { + forward_variables_from(invoker, "*") + binary_path = "${root_out_dir}/exe.unstripped/$binary" + unstripped = true + } + + group(target_name) { + forward_variables_from(invoker, [ "testonly" ]) + deps = [ + ":_${target_name}_stripped", + ":_${target_name}_unstripped", + ] + } +} From 1b3d3cc675a868d07d668f90c9b624502ad0cfee Mon Sep 17 00:00:00 2001 From: Hzj_jie Date: Thu, 10 Oct 2024 15:26:06 -0700 Subject: [PATCH 5/5] build-id --- tools/fuchsia/fuchsia_archive.gni | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tools/fuchsia/fuchsia_archive.gni b/tools/fuchsia/fuchsia_archive.gni index e42aef06905f9..911c30ac3da03 100644 --- a/tools/fuchsia/fuchsia_archive.gni +++ b/tools/fuchsia/fuchsia_archive.gni @@ -4,7 +4,6 @@ import("//flutter/tools/fuchsia/fuchsia_debug_symbols.gni") import("//flutter/tools/fuchsia/fuchsia_libs.gni") -import("//flutter/tools/fuchsia/gn-sdk/src/build_id_dir.gni") import("//flutter/tools/fuchsia/gn-sdk/src/cmc.gni") import("//flutter/tools/fuchsia/gn-sdk/src/component.gni") import("//flutter/tools/fuchsia/gn-sdk/src/gn_configs.gni") @@ -92,18 +91,18 @@ template("fuchsia_archive") { deps = [ ":$_component_target" ] } + # TODO(zijiehe): http://crbug.com/368608542, prefer using + # gn-sdk/build_id_dir.gni _build_id_target = target_name + "__build_id" fuchsia_debug_symbols(_build_id_target) { - forward_variables_from(invoker, [ "deps", "testonly", ]) + forward_variables_from(invoker, + [ + "deps", + "testonly", + ]) binary = _binary } - #build_id_dir(_build_id_target) { - # forward_variables_from(invoker, [ "testonly" ]) - # output_path = "$target_gen_dir/$_package_name/.build-id" - # deps = [ ":$_package_target" ] - #} - # TODO(zijiehe): http://crbug.com/368608542, copying the far files is not very # necessary, try to remove the -0.far copy. copy(target_name) {