From 7ba93f9a24b3479bb271dcc33acb5b09ab1d8add Mon Sep 17 00:00:00 2001 From: Chun-Heng Tai Date: Mon, 25 Nov 2024 14:37:17 -0800 Subject: [PATCH 1/3] Rewire ios accessibility bridge message channel to receive semantics generating event --- ci/licenses_golden/excluded_files | 1 + ci/licenses_golden/licenses_flutter | 2 + runtime/BUILD.gn | 2 + runtime/fixtures/runtime_test.dart | 99 ++++++++++++ runtime/runtime_controller.cc | 4 +- runtime/runtime_controller.h | 5 + runtime/runtime_controller_unittests.cc | 143 ++++++++++++++++++ shell/platform/darwin/ios/BUILD.gn | 1 + .../framework/Source/accessibility_bridge.h | 3 +- .../framework/Source/accessibility_bridge.mm | 27 ++-- .../Source/accessibility_bridge_test.mm | 51 +------ shell/platform/darwin/ios/platform_view_ios.h | 36 +++-- .../platform/darwin/ios/platform_view_ios.mm | 90 ++++++----- .../darwin/ios/platform_view_ios_test.mm | 106 +++++++++++++ 14 files changed, 440 insertions(+), 130 deletions(-) create mode 100644 runtime/runtime_controller_unittests.cc create mode 100644 shell/platform/darwin/ios/platform_view_ios_test.mm diff --git a/ci/licenses_golden/excluded_files b/ci/licenses_golden/excluded_files index 1883a2b7c4b00..1db744f18c000 100644 --- a/ci/licenses_golden/excluded_files +++ b/ci/licenses_golden/excluded_files @@ -271,6 +271,7 @@ ../../../flutter/runtime/fixtures ../../../flutter/runtime/no_dart_plugin_registrant_unittests.cc ../../../flutter/runtime/platform_isolate_manager_unittests.cc +../../../flutter/runtime/runtime_controller_unittests.cc ../../../flutter/runtime/type_conversions_unittests.cc ../../../flutter/shell/common/animator_unittests.cc ../../../flutter/shell/common/base64_unittests.cc diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index bc1a32764f98d..dd21d2087d8d4 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -44667,6 +44667,7 @@ ORIGIN: ../../../flutter/shell/platform/darwin/ios/platform_message_handler_ios. ORIGIN: ../../../flutter/shell/platform/darwin/ios/platform_message_handler_ios_test.mm + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/platform_view_ios.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/platform_view_ios.mm + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/darwin/ios/platform_view_ios_test.mm + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/rendering_api_selection.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/rendering_api_selection.mm + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/macos/framework/Headers/FlutterAppDelegate.h + ../../../flutter/LICENSE @@ -47611,6 +47612,7 @@ FILE: ../../../flutter/shell/platform/darwin/ios/platform_message_handler_ios.mm FILE: ../../../flutter/shell/platform/darwin/ios/platform_message_handler_ios_test.mm FILE: ../../../flutter/shell/platform/darwin/ios/platform_view_ios.h FILE: ../../../flutter/shell/platform/darwin/ios/platform_view_ios.mm +FILE: ../../../flutter/shell/platform/darwin/ios/platform_view_ios_test.mm FILE: ../../../flutter/shell/platform/darwin/ios/rendering_api_selection.h FILE: ../../../flutter/shell/platform/darwin/ios/rendering_api_selection.mm FILE: ../../../flutter/shell/platform/darwin/macos/framework/Headers/FlutterAppDelegate.h diff --git a/runtime/BUILD.gn b/runtime/BUILD.gn index 5c75d2f4cb2bc..701f7f802dca9 100644 --- a/runtime/BUILD.gn +++ b/runtime/BUILD.gn @@ -140,6 +140,7 @@ if (enable_unittests) { "dart_service_isolate_unittests.cc", "dart_vm_unittests.cc", "platform_isolate_manager_unittests.cc", + "runtime_controller_unittests.cc", "type_conversions_unittests.cc", ] @@ -153,6 +154,7 @@ if (enable_unittests) { "//flutter/common", "//flutter/fml", "//flutter/lib/snapshot", + "//flutter/shell/common:shell_test_fixture_sources", "//flutter/skia", "//flutter/testing", "//flutter/testing:dart", diff --git a/runtime/fixtures/runtime_test.dart b/runtime/fixtures/runtime_test.dart index 435c00430a1af..0c62b040c03cf 100644 --- a/runtime/fixtures/runtime_test.dart +++ b/runtime/fixtures/runtime_test.dart @@ -6,6 +6,8 @@ import 'dart:async'; import 'dart:isolate'; +import 'dart:typed_data'; +import 'dart:ui'; import 'split_lib_test.dart' deferred as splitlib; @@ -219,3 +221,100 @@ Function createEntryPointForPlatIsoSendAndRecvTest() { void mainForPlatformIsolatesThrowError() { throw AssertionError('Error from platform isolate'); } + +@pragma('vm:entry-point') +void sendSemanticsUpdate() { + final SemanticsUpdateBuilder builder = SemanticsUpdateBuilder(); + const String identifier = 'identifier'; + const String label = 'label'; + final List labelAttributes = [ + SpellOutStringAttribute(range: const TextRange(start: 1, end: 2)), + ]; + + const String value = 'value'; + final List valueAttributes = [ + SpellOutStringAttribute(range: const TextRange(start: 2, end: 3)), + ]; + + const String increasedValue = 'increasedValue'; + final List increasedValueAttributes = [ + SpellOutStringAttribute(range: const TextRange(start: 4, end: 5)), + ]; + + const String decreasedValue = 'decreasedValue'; + final List decreasedValueAttributes = [ + SpellOutStringAttribute(range: const TextRange(start: 5, end: 6)), + ]; + + const String hint = 'hint'; + final List hintAttributes = [ + LocaleStringAttribute( + locale: const Locale('en', 'MX'), range: const TextRange(start: 0, end: 1), + ), + ]; + + const String tooltip = 'tooltip'; + + final Float64List transform = Float64List(16); + final Int32List childrenInTraversalOrder = Int32List(0); + final Int32List childrenInHitTestOrder = Int32List(0); + final Int32List additionalActions = Int32List(0); + transform[0] = 1; + transform[1] = 0; + transform[2] = 0; + transform[3] = 0; + + transform[4] = 0; + transform[5] = 1; + transform[6] = 0; + transform[7] = 0; + + transform[8] = 0; + transform[9] = 0; + transform[10] = 1; + transform[11] = 0; + + transform[12] = 0; + transform[13] = 0; + transform[14] = 0; + transform[15] = 0; + builder.updateNode( + id: 0, + flags: 0, + actions: 0, + maxValueLength: 0, + currentValueLength: 0, + textSelectionBase: -1, + textSelectionExtent: -1, + platformViewId: -1, + scrollChildren: 0, + scrollIndex: 0, + scrollPosition: 0, + scrollExtentMax: 0, + scrollExtentMin: 0, + rect: const Rect.fromLTRB(0, 0, 10, 10), + elevation: 0, + thickness: 0, + identifier: identifier, + label: label, + labelAttributes: labelAttributes, + value: value, + valueAttributes: valueAttributes, + increasedValue: increasedValue, + increasedValueAttributes: increasedValueAttributes, + decreasedValue: decreasedValue, + decreasedValueAttributes: decreasedValueAttributes, + hint: hint, + hintAttributes: hintAttributes, + tooltip: tooltip, + textDirection: TextDirection.ltr, + transform: transform, + childrenInTraversalOrder: childrenInTraversalOrder, + childrenInHitTestOrder: childrenInHitTestOrder, + additionalActions: additionalActions, + ); + _semanticsUpdate(builder.build()); +} + +@pragma('vm:external-name', 'SemanticsUpdate') +external void _semanticsUpdate(SemanticsUpdate update); diff --git a/runtime/runtime_controller.cc b/runtime/runtime_controller.cc index 27193cdf73db3..d1161ffbfe7af 100644 --- a/runtime/runtime_controller.cc +++ b/runtime/runtime_controller.cc @@ -440,9 +440,7 @@ void RuntimeController::CheckIfAllViewsRendered() { // |PlatformConfigurationClient| void RuntimeController::UpdateSemantics(SemanticsUpdate* update) { - if (platform_data_.semantics_enabled) { - client_.UpdateSemantics(update->takeNodes(), update->takeActions()); - } + client_.UpdateSemantics(update->takeNodes(), update->takeActions()); } // |PlatformConfigurationClient| diff --git a/runtime/runtime_controller.h b/runtime/runtime_controller.h index 3ed05aa945461..e83c38bdabd23 100644 --- a/runtime/runtime_controller.h +++ b/runtime/runtime_controller.h @@ -31,6 +31,10 @@ class RuntimeDelegate; class View; class Window; +namespace testing { +class RuntimeControllerTester; +} + //------------------------------------------------------------------------------ /// Represents an instance of a running root isolate with window bindings. In /// normal operation, a single instance of this object is owned by the engine @@ -779,6 +783,7 @@ class RuntimeController : public PlatformConfigurationClient, double GetScaledFontSize(double unscaled_font_size, int configuration_id) const override; + friend class testing::RuntimeControllerTester; FML_DISALLOW_COPY_AND_ASSIGN(RuntimeController); }; diff --git a/runtime/runtime_controller_unittests.cc b/runtime/runtime_controller_unittests.cc new file mode 100644 index 0000000000000..d056412e7b867 --- /dev/null +++ b/runtime/runtime_controller_unittests.cc @@ -0,0 +1,143 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/runtime/runtime_controller.h" +#include "flutter/runtime/runtime_delegate.h" + +#include "flutter/lib/ui/semantics/semantics_update.h" +#include "flutter/shell/common/shell_test.h" +#include "flutter/testing/testing.h" + +namespace flutter::testing { + +fml::AutoResetWaitableEvent message_latch; +using RuntimeControllerTest = ShellTest; + +class MockRuntimeDelegate : public RuntimeDelegate { + public: + FontCollection font; + std::vector updates; + std::vector actions; + std::string DefaultRouteName() override { return ""; } + + void ScheduleFrame(bool regenerate_layer_trees = true) override {} + + void OnAllViewsRendered() override {} + + void Render(int64_t view_id, + std::unique_ptr layer_tree, + float device_pixel_ratio) override {} + + void UpdateSemantics(SemanticsNodeUpdates update, + CustomAccessibilityActionUpdates actions) override { + this->updates.push_back(update); + this->actions.push_back(actions); + } + + void HandlePlatformMessage( + std::unique_ptr message) override {} + + FontCollection& GetFontCollection() override { return font; } + + std::shared_ptr GetAssetManager() override { return nullptr; } + + void OnRootIsolateCreated() override {}; + + void UpdateIsolateDescription(const std::string isolate_name, + int64_t isolate_port) override {}; + + void SetNeedsReportTimings(bool value) override {}; + + std::unique_ptr> ComputePlatformResolvedLocale( + const std::vector& supported_locale_data) override { + return nullptr; + } + + void RequestDartDeferredLibrary(intptr_t loading_unit_id) override {} + + std::weak_ptr GetPlatformMessageHandler() + const override { + return {}; + } + + void SendChannelUpdate(std::string name, bool listening) override {} + + double GetScaledFontSize(double unscaled_font_size, + int configuration_id) const override { + return 0.0; + } +}; + +class RuntimeControllerTester { + public: + explicit RuntimeControllerTester(UIDartState::Context& context) + : context_(context), + runtime_controller_(delegate_, + nullptr, + {}, + {}, + {}, + {}, + {}, + nullptr, + context_) {} + + void CanUpdateSemantics(SemanticsUpdate* update) { + ASSERT_TRUE(delegate_.updates.empty()); + ASSERT_TRUE(delegate_.actions.empty()); + runtime_controller_.UpdateSemantics(update); + ASSERT_FALSE(delegate_.updates.empty()); + ASSERT_FALSE(delegate_.actions.empty()); + } + + private: + MockRuntimeDelegate delegate_; + UIDartState::Context& context_; + RuntimeController runtime_controller_; +}; + +TEST_F(RuntimeControllerTest, CanUpdateSemantics) { + // The test is mostly setup code to get a SemanticsUpdate object. + // The real test is in RuntimeControllerTester::CanUpdateSemantics. + TaskRunners task_runners("test", // label + GetCurrentTaskRunner(), // platform + CreateNewThread(), // raster + CreateNewThread(), // ui + CreateNewThread() // io + ); + UIDartState::Context context(task_runners); + std::shared_ptr tester = + std::make_shared(context); + + auto native_semantics_update = [tester](Dart_NativeArguments args) { + auto handle = Dart_GetNativeArgument(args, 0); + intptr_t peer = 0; + Dart_Handle result = Dart_GetNativeInstanceField( + handle, tonic::DartWrappable::kPeerIndex, &peer); + ASSERT_FALSE(Dart_IsError(result)); + SemanticsUpdate* update = reinterpret_cast(peer); + + tester->CanUpdateSemantics(update); + message_latch.Signal(); + }; + + Settings settings = CreateSettingsForFixture(); + AddNativeCallback("SemanticsUpdate", + CREATE_NATIVE_ENTRY(native_semantics_update)); + + std::unique_ptr shell = CreateShell(settings, task_runners); + + ASSERT_TRUE(shell->IsSetup()); + auto configuration = RunConfiguration::InferFromSettings(settings); + configuration.SetEntrypoint("sendSemanticsUpdate"); + + shell->RunEngine(std::move(configuration), [](auto result) { + ASSERT_EQ(result, Engine::RunStatus::Success); + }); + + message_latch.Wait(); + DestroyShell(std::move(shell), task_runners); +} + +} // namespace flutter::testing diff --git a/shell/platform/darwin/ios/BUILD.gn b/shell/platform/darwin/ios/BUILD.gn index 4a0eeac21c854..806c2c373886f 100644 --- a/shell/platform/darwin/ios/BUILD.gn +++ b/shell/platform/darwin/ios/BUILD.gn @@ -249,6 +249,7 @@ shared_library("ios_test_flutter") { "ios_context_noop_unittests.mm", "ios_surface_noop_unittests.mm", "platform_message_handler_ios_test.mm", + "platform_view_ios_test.mm", ] deps = [ ":flutter_framework", diff --git a/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h b/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h index e808b88d20f8a..d1589c7538ca9 100644 --- a/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h +++ b/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h @@ -56,7 +56,7 @@ class AccessibilityBridge final : public AccessibilityBridgeIos { void UpdateSemantics(flutter::SemanticsNodeUpdates nodes, const flutter::CustomAccessibilityActionUpdates& actions); - void HandleEvent(NSDictionary* annotatedEvent); + void HandleMessage(NSDictionary* message, FlutterReply reply); void DispatchSemanticsAction(int32_t id, flutter::SemanticsAction action) override; void DispatchSemanticsAction(int32_t id, flutter::SemanticsAction action, @@ -98,7 +98,6 @@ class AccessibilityBridge final : public AccessibilityBridgeIos { int32_t last_focused_semantics_object_id_; NSMutableDictionary* objects_; - FlutterBasicMessageChannel* accessibility_channel_; int32_t previous_route_id_ = 0; std::unordered_map actions_; std::vector previous_routes_; diff --git a/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm b/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm index daf6389834c81..98b3c6ed5cadd 100644 --- a/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm +++ b/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm @@ -52,18 +52,9 @@ void PostAccessibilityNotification(UIAccessibilityNotifications notification, previous_routes_({}), ios_delegate_(ios_delegate ? std::move(ios_delegate) : std::make_unique()), - weak_factory_(this) { - accessibility_channel_ = [[FlutterBasicMessageChannel alloc] - initWithName:@"flutter/accessibility" - binaryMessenger:platform_view->GetOwnerViewController().engine.binaryMessenger - codec:[FlutterStandardMessageCodec sharedInstance]]; - [accessibility_channel_ setMessageHandler:^(id message, FlutterReply reply) { - HandleEvent((NSDictionary*)message); - }]; -} + weak_factory_(this) {} AccessibilityBridge::~AccessibilityBridge() { - [accessibility_channel_ setMessageHandler:nil]; clearState(); view_controller_.viewIfLoaded.accessibilityElements = nil; } @@ -74,7 +65,7 @@ void PostAccessibilityNotification(UIAccessibilityNotifications notification, void AccessibilityBridge::AccessibilityObjectDidBecomeFocused(int32_t id) { last_focused_semantics_object_id_ = id; - [accessibility_channel_ sendMessage:@{@"type" : @"didGainFocus", @"nodeId" : @(id)}]; + platform_view_->SendAccessibilityMessage(@{@"type" : @"didGainFocus", @"nodeId" : @(id)}); } void AccessibilityBridge::AccessibilityObjectDidLoseFocus(int32_t id) { @@ -354,16 +345,20 @@ static bool DidFlagChange(const flutter::SemanticsNode& oldNode, return nil; } -void AccessibilityBridge::HandleEvent(NSDictionary* annotatedEvent) { - NSString* type = annotatedEvent[@"type"]; +void AccessibilityBridge::HandleMessage(NSDictionary* message, FlutterReply reply) { + NSString* type = message[@"type"]; if ([type isEqualToString:@"announce"]) { - NSString* message = annotatedEvent[@"data"][@"message"]; - ios_delegate_->PostAccessibilityNotification(UIAccessibilityAnnouncementNotification, message); + NSString* message_to_announce = message[@"data"][@"message"]; + ios_delegate_->PostAccessibilityNotification(UIAccessibilityAnnouncementNotification, + message_to_announce); } if ([type isEqualToString:@"focus"]) { - SemanticsObject* node = objects_[annotatedEvent[@"nodeId"]]; + SemanticsObject* node = objects_[message[@"nodeId"]]; ios_delegate_->PostAccessibilityNotification(UIAccessibilityLayoutChangedNotification, node); } + if (reply) { + reply(nil); + } } fml::WeakPtr AccessibilityBridge::GetWeakPtr() { diff --git a/shell/platform/darwin/ios/framework/Source/accessibility_bridge_test.mm b/shell/platform/darwin/ios/framework/Source/accessibility_bridge_test.mm index 343ed99eae983..960a5f85ddb98 100644 --- a/shell/platform/darwin/ios/framework/Source/accessibility_bridge_test.mm +++ b/shell/platform/darwin/ios/framework/Source/accessibility_bridge_test.mm @@ -1372,9 +1372,9 @@ - (void)testHandleEvent { /*platform_views_controller=*/nil, /*ios_delegate=*/std::move(ios_delegate)); - NSDictionary* annotatedEvent = @{@"type" : @"focus", @"nodeId" : @123}; + NSDictionary* message = @{@"type" : @"focus", @"nodeId" : @123}; - bridge->HandleEvent(annotatedEvent); + bridge->HandleMessage(message, nil); XCTAssertEqual([accessibility_notifications count], 1ul); XCTAssertEqual([accessibility_notifications[0][@"notification"] unsignedIntValue], @@ -2068,53 +2068,6 @@ - (void)testAnnouncesIgnoresScrollChangeWhenModal { XCTAssertEqual([accessibility_notifications count], 0ul); } -- (void)testAccessibilityMessageAfterDeletion { - flutter::MockDelegate mock_delegate; - auto thread = std::make_unique("AccessibilityBridgeTest"); - auto thread_task_runner = thread->GetTaskRunner(); - flutter::TaskRunners runners(/*label=*/self.name.UTF8String, - /*platform=*/thread_task_runner, - /*raster=*/thread_task_runner, - /*ui=*/thread_task_runner, - /*io=*/thread_task_runner); - id messenger = OCMProtocolMock(@protocol(FlutterBinaryMessenger)); - id engine = OCMClassMock([FlutterEngine class]); - id flutterViewController = OCMClassMock([FlutterViewController class]); - - OCMStub([flutterViewController engine]).andReturn(engine); - OCMStub([engine binaryMessenger]).andReturn(messenger); - FlutterBinaryMessengerConnection connection = 123; - OCMStub([messenger setMessageHandlerOnChannel:@"flutter/accessibility" - binaryMessageHandler:[OCMArg any]]) - .andReturn(connection); - - auto platform_view = std::make_unique( - /*delegate=*/mock_delegate, - /*rendering_api=*/mock_delegate.settings_.enable_impeller - ? flutter::IOSRenderingAPI::kMetal - : flutter::IOSRenderingAPI::kSoftware, - /*platform_views_controller=*/nil, - /*task_runners=*/runners, - /*worker_task_runner=*/nil, - /*is_gpu_disabled_sync_switch=*/std::make_shared()); - fml::AutoResetWaitableEvent latch; - thread_task_runner->PostTask([&] { - platform_view->SetOwnerViewController(flutterViewController); - auto bridge = - std::make_unique(/*view=*/nil, - /*platform_view=*/platform_view.get(), - /*platform_views_controller=*/nil); - XCTAssertTrue(bridge.get()); - OCMVerify([messenger setMessageHandlerOnChannel:@"flutter/accessibility" - binaryMessageHandler:[OCMArg isNotNil]]); - bridge.reset(); - latch.Signal(); - }); - latch.Wait(); - OCMVerify([messenger cleanUpConnection:connection]); - [engine stopMocking]; -} - - (void)testFlutterSemanticsScrollViewManagedObjectLifecycleCorrectly { flutter::MockDelegate mock_delegate; auto thread_task_runner = CreateNewThread("AccessibilityBridgeTest"); diff --git a/shell/platform/darwin/ios/platform_view_ios.h b/shell/platform/darwin/ios/platform_view_ios.h index 4e1da467d22bd..ae4dffc7ef536 100644 --- a/shell/platform/darwin/ios/platform_view_ios.h +++ b/shell/platform/darwin/ios/platform_view_ios.h @@ -66,6 +66,11 @@ class PlatformViewIOS final : public PlatformView { */ void SetOwnerViewController(__weak FlutterViewController* owner_controller); + /** + * Send accessibility message to accessibility channel. + */ + void SendAccessibilityMessage(__weak id message); + /** * Called one time per `FlutterViewController` when the `FlutterViewController`'s * UIView is first loaded. @@ -98,6 +103,16 @@ class PlatformViewIOS final : public PlatformView { return platform_message_handler_; } + /** + * Gets the accessibility bridge created in this platform view. + */ + AccessibilityBridge* GetAccessibilityBridge() { return accessibility_bridge_.get(); } + + /** + * Handles accessibility message from accessibility channel. + */ + void HandleAccessibilityMessage(__weak id message, FlutterReply reply); + private: /// Smart pointer for use with objective-c observers. /// This guarantees we remove the observer. @@ -113,24 +128,6 @@ class PlatformViewIOS final : public PlatformView { id observer_ = nil; }; - /// Wrapper that guarantees we communicate clearing Accessibility - /// information to Dart. - class AccessibilityBridgeManager { - public: - explicit AccessibilityBridgeManager(const std::function& set_semantics_enabled); - AccessibilityBridgeManager(const std::function& set_semantics_enabled, - AccessibilityBridge* bridge); - explicit operator bool() const noexcept { return static_cast(accessibility_bridge_); } - AccessibilityBridge* get() const noexcept { return accessibility_bridge_.get(); } - void Set(std::unique_ptr bridge); - void Clear(); - - private: - FML_DISALLOW_COPY_AND_ASSIGN(AccessibilityBridgeManager); - std::unique_ptr accessibility_bridge_; - std::function set_semantics_enabled_; - }; - __weak FlutterViewController* owner_controller_; // Since the `ios_surface_` is created on the platform thread but // used on the raster thread we need to protect it with a mutex. @@ -138,10 +135,11 @@ class PlatformViewIOS final : public PlatformView { std::unique_ptr ios_surface_; std::shared_ptr ios_context_; __weak FlutterPlatformViewsController* platform_views_controller_; - AccessibilityBridgeManager accessibility_bridge_; + std::unique_ptr accessibility_bridge_; ScopedObserver dealloc_view_controller_observer_; std::vector platform_resolved_locale_; std::shared_ptr platform_message_handler_; + FlutterBasicMessageChannel* accessibility_channel_; // |PlatformView| void HandlePlatformMessage(std::unique_ptr message) override; diff --git a/shell/platform/darwin/ios/platform_view_ios.mm b/shell/platform/darwin/ios/platform_view_ios.mm index b726fd8fc1ee4..b9641c71776f0 100644 --- a/shell/platform/darwin/ios/platform_view_ios.mm +++ b/shell/platform/darwin/ios/platform_view_ios.mm @@ -18,29 +18,6 @@ namespace flutter { -PlatformViewIOS::AccessibilityBridgeManager::AccessibilityBridgeManager( - const std::function& set_semantics_enabled) - : AccessibilityBridgeManager(set_semantics_enabled, nullptr) {} - -PlatformViewIOS::AccessibilityBridgeManager::AccessibilityBridgeManager( - const std::function& set_semantics_enabled, - AccessibilityBridge* bridge) - : accessibility_bridge_(bridge), set_semantics_enabled_(set_semantics_enabled) { - if (bridge) { - set_semantics_enabled_(true); - } -} - -void PlatformViewIOS::AccessibilityBridgeManager::Set(std::unique_ptr bridge) { - accessibility_bridge_ = std::move(bridge); - set_semantics_enabled_(true); -} - -void PlatformViewIOS::AccessibilityBridgeManager::Clear() { - set_semantics_enabled_(false); - accessibility_bridge_.reset(); -} - PlatformViewIOS::PlatformViewIOS(PlatformView::Delegate& delegate, const std::shared_ptr& context, __weak FlutterPlatformViewsController* platform_views_controller, @@ -48,7 +25,6 @@ : PlatformView(delegate, task_runners), ios_context_(context), platform_views_controller_(platform_views_controller), - accessibility_bridge_([this](bool enabled) { PlatformView::SetSemanticsEnabled(enabled); }), platform_message_handler_( new PlatformMessageHandlerIos(task_runners.GetPlatformTaskRunner())) {} @@ -85,7 +61,19 @@ new PlatformMessageHandlerIos(task_runners.GetPlatformTaskRunner())) {} if (ios_surface_ || !owner_controller) { NotifyDestroyed(); ios_surface_.reset(); - accessibility_bridge_.Clear(); + accessibility_bridge_.reset(); + } + if (owner_controller) { + accessibility_channel_ = [[FlutterBasicMessageChannel alloc] + initWithName:@"flutter/accessibility" + binaryMessenger:owner_controller.engine.binaryMessenger + codec:[FlutterStandardMessageCodec sharedInstance]]; + [accessibility_channel_ setMessageHandler:^(id message, FlutterReply reply) { + HandleAccessibilityMessage(message, reply); + }]; + } else { + [accessibility_channel_ setMessageHandler:nil]; + accessibility_channel_ = nil; } owner_controller_ = owner_controller; @@ -97,7 +85,7 @@ new PlatformMessageHandlerIos(task_runners.GetPlatformTaskRunner())) {} queue:[NSOperationQueue mainQueue] usingBlock:^(NSNotification* note) { // Implicit copy of 'this' is fine. - accessibility_bridge_.Clear(); + accessibility_bridge_.reset(); owner_controller_ = nil; }]); @@ -120,8 +108,8 @@ new PlatformMessageHandlerIos(task_runners.GetPlatformTaskRunner())) {} FML_DCHECK(ios_surface_ != nullptr); if (accessibility_bridge_) { - accessibility_bridge_.Set(std::make_unique( - owner_controller_, this, owner_controller_.platformViewsController)); + accessibility_bridge_ = std::make_unique( + owner_controller_, this, owner_controller_.platformViewsController); } } @@ -131,6 +119,37 @@ new PlatformMessageHandlerIos(task_runners.GetPlatformTaskRunner())) {} }; } +void PlatformViewIOS::HandleAccessibilityMessage(__weak id message, FlutterReply reply) { + if (!owner_controller_) { + FML_LOG(WARNING) << "Could not accept accessibility message, this " + "PlatformViewIOS has no ViewController."; + } + NSString* type = message[@"type"]; + if ([type isEqualToString:@"generatingSemanticsTree"]) { + BOOL generating = [message[@"data"][@"generating"] boolValue]; + if (generating) { + if (!accessibility_bridge_) { + accessibility_bridge_ = std::make_unique(owner_controller_, this, + platform_views_controller_); + } + } else { + accessibility_bridge_.reset(); + } + if (reply) { + reply(nil); + } + return; + } + + if (accessibility_bridge_) { + accessibility_bridge_->HandleMessage(message, reply); + } +} + +void PlatformViewIOS::SendAccessibilityMessage(__weak id message) { + [accessibility_channel_ sendMessage:message]; +} + void PlatformViewIOS::RegisterExternalTexture(int64_t texture_id, NSObject* texture) { RegisterTexture(ios_context_->CreateExternalTexture(texture_id, texture)); @@ -165,19 +184,7 @@ new PlatformMessageHandlerIos(task_runners.GetPlatformTaskRunner())) {} // |PlatformView| void PlatformViewIOS::SetSemanticsEnabled(bool enabled) { - if (!owner_controller_) { - FML_LOG(WARNING) << "Could not set semantics to enabled, this " - "PlatformViewIOS has no ViewController."; - return; - } - if (enabled && !accessibility_bridge_) { - accessibility_bridge_.Set(std::make_unique( - owner_controller_, this, owner_controller_.platformViewsController)); - } else if (!enabled && accessibility_bridge_) { - accessibility_bridge_.Clear(); - } else { - PlatformView::SetSemanticsEnabled(enabled); - } + PlatformView::SetSemanticsEnabled(enabled); } // |shell:PlatformView| @@ -189,6 +196,7 @@ new PlatformMessageHandlerIos(task_runners.GetPlatformTaskRunner())) {} void PlatformViewIOS::UpdateSemantics(flutter::SemanticsNodeUpdates update, flutter::CustomAccessibilityActionUpdates actions) { FML_DCHECK(owner_controller_); + FML_DCHECK(accessibility_bridge_); if (accessibility_bridge_) { accessibility_bridge_.get()->UpdateSemantics(std::move(update), actions); [[NSNotificationCenter defaultCenter] postNotificationName:FlutterSemanticsUpdateNotification diff --git a/shell/platform/darwin/ios/platform_view_ios_test.mm b/shell/platform/darwin/ios/platform_view_ios_test.mm new file mode 100644 index 0000000000000..97ab2a987ff1a --- /dev/null +++ b/shell/platform/darwin/ios/platform_view_ios_test.mm @@ -0,0 +1,106 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#import +#import + +#import "flutter/fml/thread.h" +#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h" +#import "flutter/shell/platform/darwin/ios/platform_view_ios.h" + +FLUTTER_ASSERT_ARC + +namespace flutter { + +namespace { + +class MockDelegate : public PlatformView::Delegate { + public: + void OnPlatformViewCreated(std::unique_ptr surface) override {} + void OnPlatformViewDestroyed() override {} + void OnPlatformViewScheduleFrame() override {} + void OnPlatformViewAddView(int64_t view_id, + const ViewportMetrics& viewport_metrics, + AddViewCallback callback) override {} + void OnPlatformViewRemoveView(int64_t view_id, RemoveViewCallback callback) override {} + void OnPlatformViewSetNextFrameCallback(const fml::closure& closure) override {} + void OnPlatformViewSetViewportMetrics(int64_t view_id, const ViewportMetrics& metrics) override {} + const flutter::Settings& OnPlatformViewGetSettings() const override { return settings_; } + void OnPlatformViewDispatchPlatformMessage(std::unique_ptr message) override {} + void OnPlatformViewDispatchPointerDataPacket(std::unique_ptr packet) override { + } + void OnPlatformViewDispatchSemanticsAction(int32_t id, + SemanticsAction action, + fml::MallocMapping args) override {} + void OnPlatformViewSetSemanticsEnabled(bool enabled) override {} + void OnPlatformViewSetAccessibilityFeatures(int32_t flags) override {} + void OnPlatformViewRegisterTexture(std::shared_ptr texture) override {} + void OnPlatformViewUnregisterTexture(int64_t texture_id) override {} + void OnPlatformViewMarkTextureFrameAvailable(int64_t texture_id) override {} + + void LoadDartDeferredLibrary(intptr_t loading_unit_id, + std::unique_ptr snapshot_data, + std::unique_ptr snapshot_instructions) override { + } + void LoadDartDeferredLibraryError(intptr_t loading_unit_id, + const std::string error_message, + bool transient) override {} + void UpdateAssetResolverByType(std::unique_ptr updated_asset_resolver, + flutter::AssetResolver::AssetResolverType type) override {} + + flutter::Settings settings_; +}; +} // namespace +} // namespace flutter + +@interface PlatformViewIOSTest : XCTestCase +@end + +@implementation PlatformViewIOSTest + +- (void)testCreate { + flutter::MockDelegate mock_delegate; + auto thread = std::make_unique("PlatformViewIOSTest"); + auto thread_task_runner = thread->GetTaskRunner(); + flutter::TaskRunners runners(/*label=*/self.name.UTF8String, + /*platform=*/thread_task_runner, + /*raster=*/thread_task_runner, + /*ui=*/thread_task_runner, + /*io=*/thread_task_runner); + id messenger = OCMProtocolMock(@protocol(FlutterBinaryMessenger)); + id engine = OCMClassMock([FlutterEngine class]); + + id flutterViewController = OCMClassMock([FlutterViewController class]); + + OCMStub([flutterViewController isViewLoaded]).andReturn(NO); + OCMStub([flutterViewController engine]).andReturn(engine); + OCMStub([engine binaryMessenger]).andReturn(messenger); + + auto platform_view = std::make_unique( + /*delegate=*/mock_delegate, + /*rendering_api=*/mock_delegate.settings_.enable_impeller + ? flutter::IOSRenderingAPI::kMetal + : flutter::IOSRenderingAPI::kSoftware, + /*platform_views_controller=*/nil, + /*task_runners=*/runners, + /*worker_task_runner=*/nil, + /*is_gpu_disabled_sync_switch=*/std::make_shared()); + fml::AutoResetWaitableEvent latch; + thread_task_runner->PostTask([&] { + platform_view->SetOwnerViewController(flutterViewController); + XCTAssertFalse(platform_view->GetAccessibilityBridge()); + platform_view->HandleAccessibilityMessage( + @{@"type" : @"generatingSemanticsTree", @"data" : @{@"generating" : @(YES)}}, nil); + XCTAssertTrue(platform_view->GetAccessibilityBridge()); + platform_view->HandleAccessibilityMessage( + @{@"type" : @"generatingSemanticsTree", @"data" : @{@"generating" : @(NO)}}, nil); + XCTAssertFalse(platform_view->GetAccessibilityBridge()); + latch.Signal(); + }); + latch.Wait(); + + [engine stopMocking]; +} + +@end From c91b832860edb459db56fbd256faa64cc561f915 Mon Sep 17 00:00:00 2001 From: Chun-Heng Tai Date: Tue, 26 Nov 2024 15:23:27 -0800 Subject: [PATCH 2/3] update --- testing/scenario_app/lib/main.dart | 8 + .../scenario_app/lib/src/standard_codec.dart | 286 ++++++++++++++++++ 2 files changed, 294 insertions(+) create mode 100644 testing/scenario_app/lib/src/standard_codec.dart diff --git a/testing/scenario_app/lib/main.dart b/testing/scenario_app/lib/main.dart index 636cd3b1521de..f6d34625194b6 100644 --- a/testing/scenario_app/lib/main.dart +++ b/testing/scenario_app/lib/main.dart @@ -7,6 +7,7 @@ import 'dart:typed_data'; import 'dart:ui'; import 'src/scenarios.dart'; +import 'src/standard_codec.dart'; void main() { // TODO(goderbauer): Create a window if embedder doesn't provide an implicit @@ -33,6 +34,13 @@ void main() { final ByteData data = ByteData(1); data.setUint8(0, 1); PlatformDispatcher.instance.sendPlatformMessage('waiting_for_status', data, null); + final Map enableSemantics = { + 'type': 'generatingSemanticsTree', + 'data': { + 'generating': true, + }, + }; + PlatformDispatcher.instance.sendPlatformMessage('flutter/accessibility', const StandardMessageCodec().encodeMessage(enableSemantics), null); } /// The FlutterView into which the [Scenario]s will be rendered. diff --git a/testing/scenario_app/lib/src/standard_codec.dart b/testing/scenario_app/lib/src/standard_codec.dart new file mode 100644 index 0000000000000..1e697ee38b23f --- /dev/null +++ b/testing/scenario_app/lib/src/standard_codec.dart @@ -0,0 +1,286 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:math' as math; +import 'dart:typed_data'; + +const int __writeBufferStartCapacity = 64; + +/// This is mirroring the standard codec from framework +class StandardMessageCodec { + /// Creates a [MessageCodec] using the Flutter standard binary encoding. + const StandardMessageCodec(); + static const int _valueNull = 0; + static const int _valueTrue = 1; + static const int _valueFalse = 2; + static const int _valueInt32 = 3; + static const int _valueInt64 = 4; + // Unused + // static const int _valueLargeInt = 5; + static const int _valueFloat64 = 6; + static const int _valueString = 7; + static const int _valueUint8List = 8; + static const int _valueInt32List = 9; + static const int _valueInt64List = 10; + static const int _valueFloat64List = 11; + static const int _valueList = 12; + static const int _valueMap = 13; + static const int _valueFloat32List = 14; + + // Encode the message. + ByteData? encodeMessage(Object? message) { + if (message == null) { + return null; + } + final _WriteBuffer buffer = _WriteBuffer(startCapacity: __writeBufferStartCapacity); + _writeValue(buffer, message); + return buffer.done(); + } + + void _writeValue(_WriteBuffer buffer, Object? value) { + if (value == null) { + buffer.putUint8(_valueNull); + } else if (value is bool) { + buffer.putUint8(value ? _valueTrue : _valueFalse); + } else if (value is double) { // Double precedes int because in JS everything is a double. + // Therefore in JS, both `is int` and `is double` always + // return `true`. If we check int first, we'll end up treating + // all numbers as ints and attempt the int32/int64 conversion, + // which is wrong. This precedence rule is irrelevant when + // decoding because we use tags to detect the type of value. + buffer.putUint8(_valueFloat64); + buffer.putFloat64(value); + } else if (value is int) { // ignore: avoid_double_and_int_checks, JS code always goes through the `double` path above + if (-0x7fffffff - 1 <= value && value <= 0x7fffffff) { + buffer.putUint8(_valueInt32); + buffer.putInt32(value); + } else { + buffer.putUint8(_valueInt64); + buffer.putInt64(value); + } + } else if (value is String) { + buffer.putUint8(_valueString); + final Uint8List asciiBytes = Uint8List(value.length); + Uint8List? utf8Bytes; + int utf8Offset = 0; + // Only do utf8 encoding if we encounter non-ascii characters. + for (int i = 0; i < value.length; i += 1) { + final int char = value.codeUnitAt(i); + if (char <= 0x7f) { + asciiBytes[i] = char; + } else { + utf8Bytes = utf8.encode(value.substring(i)); + utf8Offset = i; + break; + } + } + if (utf8Bytes != null) { + _writeSize(buffer, utf8Offset + utf8Bytes.length); + buffer.putUint8List(Uint8List.sublistView(asciiBytes, 0, utf8Offset)); + buffer.putUint8List(utf8Bytes); + } else { + _writeSize(buffer, asciiBytes.length); + buffer.putUint8List(asciiBytes); + } + } else if (value is Uint8List) { + buffer.putUint8(_valueUint8List); + _writeSize(buffer, value.length); + buffer.putUint8List(value); + } else if (value is Int32List) { + buffer.putUint8(_valueInt32List); + _writeSize(buffer, value.length); + buffer.putInt32List(value); + } else if (value is Int64List) { + buffer.putUint8(_valueInt64List); + _writeSize(buffer, value.length); + buffer.putInt64List(value); + } else if (value is Float32List) { + buffer.putUint8(_valueFloat32List); + _writeSize(buffer, value.length); + buffer.putFloat32List(value); + } else if (value is Float64List) { + buffer.putUint8(_valueFloat64List); + _writeSize(buffer, value.length); + buffer.putFloat64List(value); + } else if (value is List) { + buffer.putUint8(_valueList); + _writeSize(buffer, value.length); + for (final Object? item in value) { + _writeValue(buffer, item); + } + } else if (value is Map) { + buffer.putUint8(_valueMap); + _writeSize(buffer, value.length); + value.forEach((Object? key, Object? value) { + _writeValue(buffer, key); + _writeValue(buffer, value); + }); + } else { + throw ArgumentError.value(value); + } + } + + void _writeSize(_WriteBuffer buffer, int value) { + assert(0 <= value && value <= 0xffffffff); + if (value < 254) { + buffer.putUint8(value); + } else if (value <= 0xffff) { + buffer.putUint8(254); + buffer.putUint16(value); + } else { + buffer.putUint8(255); + buffer.putUint32(value); + } + } +} + + +class _WriteBuffer { + factory _WriteBuffer({int startCapacity = 8}) { + assert(startCapacity > 0); + final ByteData eightBytes = ByteData(8); + final Uint8List eightBytesAsList = eightBytes.buffer.asUint8List(); + return _WriteBuffer._(Uint8List(startCapacity), eightBytes, eightBytesAsList); + } + + _WriteBuffer._(this._buffer, this._eightBytes, this._eightBytesAsList); + + Uint8List _buffer; + int _currentSize = 0; + bool _isDone = false; + final ByteData _eightBytes; + final Uint8List _eightBytesAsList; + static final Uint8List _zeroBuffer = Uint8List(8); + + void _add(int byte) { + if (_currentSize == _buffer.length) { + _resize(); + } + _buffer[_currentSize] = byte; + _currentSize += 1; + } + + void _append(Uint8List other) { + final int newSize = _currentSize + other.length; + if (newSize >= _buffer.length) { + _resize(newSize); + } + _buffer.setRange(_currentSize, newSize, other); + _currentSize += other.length; + } + + void _addAll(Uint8List data, [int start = 0, int? end]) { + final int newEnd = end ?? _eightBytesAsList.length; + final int newSize = _currentSize + (newEnd - start); + if (newSize >= _buffer.length) { + _resize(newSize); + } + _buffer.setRange(_currentSize, newSize, data); + _currentSize = newSize; + } + + void _resize([int? requiredLength]) { + final int doubleLength = _buffer.length * 2; + final int newLength = math.max(requiredLength ?? 0, doubleLength); + final Uint8List newBuffer = Uint8List(newLength); + newBuffer.setRange(0, _buffer.length, _buffer); + _buffer = newBuffer; + } + + /// Write a Uint8 into the buffer. + void putUint8(int byte) { + assert(!_isDone); + _add(byte); + } + + /// Write a Uint16 into the buffer. + void putUint16(int value, {Endian? endian}) { + assert(!_isDone); + _eightBytes.setUint16(0, value, endian ?? Endian.host); + _addAll(_eightBytesAsList, 0, 2); + } + + /// Write a Uint32 into the buffer. + void putUint32(int value, {Endian? endian}) { + assert(!_isDone); + _eightBytes.setUint32(0, value, endian ?? Endian.host); + _addAll(_eightBytesAsList, 0, 4); + } + + /// Write an Int32 into the buffer. + void putInt32(int value, {Endian? endian}) { + assert(!_isDone); + _eightBytes.setInt32(0, value, endian ?? Endian.host); + _addAll(_eightBytesAsList, 0, 4); + } + + /// Write an Int64 into the buffer. + void putInt64(int value, {Endian? endian}) { + assert(!_isDone); + _eightBytes.setInt64(0, value, endian ?? Endian.host); + _addAll(_eightBytesAsList, 0, 8); + } + + /// Write an Float64 into the buffer. + void putFloat64(double value, {Endian? endian}) { + assert(!_isDone); + _alignTo(8); + _eightBytes.setFloat64(0, value, endian ?? Endian.host); + _addAll(_eightBytesAsList); + } + + /// Write all the values from a [Uint8List] into the buffer. + void putUint8List(Uint8List list) { + assert(!_isDone); + _append(list); + } + + /// Write all the values from an [Int32List] into the buffer. + void putInt32List(Int32List list) { + assert(!_isDone); + _alignTo(4); + _append(list.buffer.asUint8List(list.offsetInBytes, 4 * list.length)); + } + + /// Write all the values from an [Int64List] into the buffer. + void putInt64List(Int64List list) { + assert(!_isDone); + _alignTo(8); + _append(list.buffer.asUint8List(list.offsetInBytes, 8 * list.length)); + } + + /// Write all the values from a [Float32List] into the buffer. + void putFloat32List(Float32List list) { + assert(!_isDone); + _alignTo(4); + _append(list.buffer.asUint8List(list.offsetInBytes, 4 * list.length)); + } + + /// Write all the values from a [Float64List] into the buffer. + void putFloat64List(Float64List list) { + assert(!_isDone); + _alignTo(8); + _append(list.buffer.asUint8List(list.offsetInBytes, 8 * list.length)); + } + + void _alignTo(int alignment) { + assert(!_isDone); + final int mod = _currentSize % alignment; + if (mod != 0) { + _addAll(_zeroBuffer, 0, alignment - mod); + } + } + + /// Finalize and return the written [ByteData]. + ByteData done() { + if (_isDone) { + throw StateError('done() must not be called more than once on the same $runtimeType.'); + } + final ByteData result = _buffer.buffer.asByteData(0, _currentSize); + _buffer = Uint8List(0); + _isDone = true; + return result; + } +} From af16d0e866809a0912984a6d5658500bd5486b11 Mon Sep 17 00:00:00 2001 From: Chun-Heng Tai Date: Wed, 27 Nov 2024 13:23:36 -0800 Subject: [PATCH 3/3] lint --- testing/scenario_app/lib/src/standard_codec.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/scenario_app/lib/src/standard_codec.dart b/testing/scenario_app/lib/src/standard_codec.dart index 1e697ee38b23f..09687e8535054 100644 --- a/testing/scenario_app/lib/src/standard_codec.dart +++ b/testing/scenario_app/lib/src/standard_codec.dart @@ -29,7 +29,7 @@ class StandardMessageCodec { static const int _valueMap = 13; static const int _valueFloat32List = 14; - // Encode the message. + /// Encode the message. ByteData? encodeMessage(Object? message) { if (message == null) { return null;