Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rewire ios accessibility bridge message channel to receive semantics generating event #56691

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ci/licenses_golden/excluded_files
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@
../../../flutter/runtime/fixtures
../../../flutter/runtime/no_dart_plugin_registrant_unittests.cc
../../../flutter/runtime/platform_isolate_manager_unittests.cc
../../../flutter/runtime/runtime_controller_unittests.cc
../../../flutter/runtime/type_conversions_unittests.cc
../../../flutter/shell/common/animator_unittests.cc
../../../flutter/shell/common/base64_unittests.cc
Expand Down
2 changes: 2 additions & 0 deletions ci/licenses_golden/licenses_flutter
Original file line number Diff line number Diff line change
Expand Up @@ -44667,6 +44667,7 @@ ORIGIN: ../../../flutter/shell/platform/darwin/ios/platform_message_handler_ios.
ORIGIN: ../../../flutter/shell/platform/darwin/ios/platform_message_handler_ios_test.mm + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/darwin/ios/platform_view_ios.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/darwin/ios/platform_view_ios.mm + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/darwin/ios/platform_view_ios_test.mm + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/darwin/ios/rendering_api_selection.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/darwin/ios/rendering_api_selection.mm + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/darwin/macos/framework/Headers/FlutterAppDelegate.h + ../../../flutter/LICENSE
Expand Down Expand Up @@ -47611,6 +47612,7 @@ FILE: ../../../flutter/shell/platform/darwin/ios/platform_message_handler_ios.mm
FILE: ../../../flutter/shell/platform/darwin/ios/platform_message_handler_ios_test.mm
FILE: ../../../flutter/shell/platform/darwin/ios/platform_view_ios.h
FILE: ../../../flutter/shell/platform/darwin/ios/platform_view_ios.mm
FILE: ../../../flutter/shell/platform/darwin/ios/platform_view_ios_test.mm
FILE: ../../../flutter/shell/platform/darwin/ios/rendering_api_selection.h
FILE: ../../../flutter/shell/platform/darwin/ios/rendering_api_selection.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Headers/FlutterAppDelegate.h
Expand Down
2 changes: 2 additions & 0 deletions runtime/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ if (enable_unittests) {
"dart_service_isolate_unittests.cc",
"dart_vm_unittests.cc",
"platform_isolate_manager_unittests.cc",
"runtime_controller_unittests.cc",
"type_conversions_unittests.cc",
]

Expand All @@ -153,6 +154,7 @@ if (enable_unittests) {
"//flutter/common",
"//flutter/fml",
"//flutter/lib/snapshot",
"//flutter/shell/common:shell_test_fixture_sources",
"//flutter/skia",
"//flutter/testing",
"//flutter/testing:dart",
Expand Down
99 changes: 99 additions & 0 deletions runtime/fixtures/runtime_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@

import 'dart:async';
import 'dart:isolate';
import 'dart:typed_data';
import 'dart:ui';

import 'split_lib_test.dart' deferred as splitlib;

Expand Down Expand Up @@ -219,3 +221,100 @@ Function createEntryPointForPlatIsoSendAndRecvTest() {
void mainForPlatformIsolatesThrowError() {
throw AssertionError('Error from platform isolate');
}

@pragma('vm:entry-point')
void sendSemanticsUpdate() {
final SemanticsUpdateBuilder builder = SemanticsUpdateBuilder();
const String identifier = 'identifier';
const String label = 'label';
final List<StringAttribute> labelAttributes = <StringAttribute> [
SpellOutStringAttribute(range: const TextRange(start: 1, end: 2)),
];

const String value = 'value';
final List<StringAttribute> valueAttributes = <StringAttribute> [
SpellOutStringAttribute(range: const TextRange(start: 2, end: 3)),
];

const String increasedValue = 'increasedValue';
final List<StringAttribute> increasedValueAttributes = <StringAttribute> [
SpellOutStringAttribute(range: const TextRange(start: 4, end: 5)),
];

const String decreasedValue = 'decreasedValue';
final List<StringAttribute> decreasedValueAttributes = <StringAttribute> [
SpellOutStringAttribute(range: const TextRange(start: 5, end: 6)),
];

const String hint = 'hint';
final List<StringAttribute> hintAttributes = <StringAttribute> [
LocaleStringAttribute(
locale: const Locale('en', 'MX'), range: const TextRange(start: 0, end: 1),
),
];

const String tooltip = 'tooltip';

final Float64List transform = Float64List(16);
final Int32List childrenInTraversalOrder = Int32List(0);
final Int32List childrenInHitTestOrder = Int32List(0);
final Int32List additionalActions = Int32List(0);
transform[0] = 1;
transform[1] = 0;
transform[2] = 0;
transform[3] = 0;

transform[4] = 0;
transform[5] = 1;
transform[6] = 0;
transform[7] = 0;

transform[8] = 0;
transform[9] = 0;
transform[10] = 1;
transform[11] = 0;

transform[12] = 0;
transform[13] = 0;
transform[14] = 0;
transform[15] = 0;
builder.updateNode(
id: 0,
flags: 0,
actions: 0,
maxValueLength: 0,
currentValueLength: 0,
textSelectionBase: -1,
textSelectionExtent: -1,
platformViewId: -1,
scrollChildren: 0,
scrollIndex: 0,
scrollPosition: 0,
scrollExtentMax: 0,
scrollExtentMin: 0,
rect: const Rect.fromLTRB(0, 0, 10, 10),
elevation: 0,
thickness: 0,
identifier: identifier,
label: label,
labelAttributes: labelAttributes,
value: value,
valueAttributes: valueAttributes,
increasedValue: increasedValue,
increasedValueAttributes: increasedValueAttributes,
decreasedValue: decreasedValue,
decreasedValueAttributes: decreasedValueAttributes,
hint: hint,
hintAttributes: hintAttributes,
tooltip: tooltip,
textDirection: TextDirection.ltr,
transform: transform,
childrenInTraversalOrder: childrenInTraversalOrder,
childrenInHitTestOrder: childrenInHitTestOrder,
additionalActions: additionalActions,
);
_semanticsUpdate(builder.build());
}

@pragma('vm:external-name', 'SemanticsUpdate')
external void _semanticsUpdate(SemanticsUpdate update);
4 changes: 1 addition & 3 deletions runtime/runtime_controller.cc
Original file line number Diff line number Diff line change
Expand Up @@ -440,9 +440,7 @@ void RuntimeController::CheckIfAllViewsRendered() {

// |PlatformConfigurationClient|
void RuntimeController::UpdateSemantics(SemanticsUpdate* update) {
if (platform_data_.semantics_enabled) {
client_.UpdateSemantics(update->takeNodes(), update->takeActions());
}
client_.UpdateSemantics(update->takeNodes(), update->takeActions());
}

// |PlatformConfigurationClient|
Expand Down
5 changes: 5 additions & 0 deletions runtime/runtime_controller.h
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ class RuntimeDelegate;
class View;
class Window;

namespace testing {
class RuntimeControllerTester;
}

//------------------------------------------------------------------------------
/// Represents an instance of a running root isolate with window bindings. In
/// normal operation, a single instance of this object is owned by the engine
Expand Down Expand Up @@ -779,6 +783,7 @@ class RuntimeController : public PlatformConfigurationClient,
double GetScaledFontSize(double unscaled_font_size,
int configuration_id) const override;

friend class testing::RuntimeControllerTester;
FML_DISALLOW_COPY_AND_ASSIGN(RuntimeController);
};

Expand Down
143 changes: 143 additions & 0 deletions runtime/runtime_controller_unittests.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "flutter/runtime/runtime_controller.h"
#include "flutter/runtime/runtime_delegate.h"

#include "flutter/lib/ui/semantics/semantics_update.h"
#include "flutter/shell/common/shell_test.h"
#include "flutter/testing/testing.h"

namespace flutter::testing {

fml::AutoResetWaitableEvent message_latch;
using RuntimeControllerTest = ShellTest;

class MockRuntimeDelegate : public RuntimeDelegate {
public:
FontCollection font;
std::vector<SemanticsNodeUpdates> updates;
std::vector<CustomAccessibilityActionUpdates> actions;
std::string DefaultRouteName() override { return ""; }

void ScheduleFrame(bool regenerate_layer_trees = true) override {}

void OnAllViewsRendered() override {}

void Render(int64_t view_id,
std::unique_ptr<flutter::LayerTree> layer_tree,
float device_pixel_ratio) override {}

void UpdateSemantics(SemanticsNodeUpdates update,
CustomAccessibilityActionUpdates actions) override {
this->updates.push_back(update);
this->actions.push_back(actions);
}

void HandlePlatformMessage(
std::unique_ptr<PlatformMessage> message) override {}

FontCollection& GetFontCollection() override { return font; }

std::shared_ptr<AssetManager> GetAssetManager() override { return nullptr; }

void OnRootIsolateCreated() override {};

void UpdateIsolateDescription(const std::string isolate_name,
int64_t isolate_port) override {};

void SetNeedsReportTimings(bool value) override {};

std::unique_ptr<std::vector<std::string>> ComputePlatformResolvedLocale(
const std::vector<std::string>& supported_locale_data) override {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nothing to do in your patch, but I'm going to take a look at the base class. Really curious why we're handing back a std::unique_ptr<std::vector<std::string>> here rather than just a std::vector<std::string>. I doubt these arrays are ever more than 5-10 entries even in extreme cases.

return nullptr;
}

void RequestDartDeferredLibrary(intptr_t loading_unit_id) override {}

std::weak_ptr<PlatformMessageHandler> GetPlatformMessageHandler()
const override {
return {};
}

void SendChannelUpdate(std::string name, bool listening) override {}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similarly, nothing to do here, but we should almost certainly make this a const std::string& or a std::string_view in the base class.


double GetScaledFontSize(double unscaled_font_size,
int configuration_id) const override {
return 0.0;
}
};

class RuntimeControllerTester {
public:
explicit RuntimeControllerTester(UIDartState::Context& context)
: context_(context),
runtime_controller_(delegate_,
nullptr,
{},
{},
{},
{},
{},
nullptr,
context_) {}

void CanUpdateSemantics(SemanticsUpdate* update) {
ASSERT_TRUE(delegate_.updates.empty());
ASSERT_TRUE(delegate_.actions.empty());
runtime_controller_.UpdateSemantics(update);
ASSERT_FALSE(delegate_.updates.empty());
ASSERT_FALSE(delegate_.actions.empty());
}

private:
MockRuntimeDelegate delegate_;
UIDartState::Context& context_;
RuntimeController runtime_controller_;
};

TEST_F(RuntimeControllerTest, CanUpdateSemantics) {
// The test is mostly setup code to get a SemanticsUpdate object.
// The real test is in RuntimeControllerTester::CanUpdateSemantics.
TaskRunners task_runners("test", // label
GetCurrentTaskRunner(), // platform
CreateNewThread(), // raster
CreateNewThread(), // ui
CreateNewThread() // io
);
UIDartState::Context context(task_runners);
std::shared_ptr<RuntimeControllerTester> tester =
std::make_shared<RuntimeControllerTester>(context);

auto native_semantics_update = [tester](Dart_NativeArguments args) {
auto handle = Dart_GetNativeArgument(args, 0);
intptr_t peer = 0;
Dart_Handle result = Dart_GetNativeInstanceField(
handle, tonic::DartWrappable::kPeerIndex, &peer);
ASSERT_FALSE(Dart_IsError(result));
SemanticsUpdate* update = reinterpret_cast<SemanticsUpdate*>(peer);

tester->CanUpdateSemantics(update);
message_latch.Signal();
};

Settings settings = CreateSettingsForFixture();
AddNativeCallback("SemanticsUpdate",
CREATE_NATIVE_ENTRY(native_semantics_update));

std::unique_ptr<Shell> shell = CreateShell(settings, task_runners);

ASSERT_TRUE(shell->IsSetup());
auto configuration = RunConfiguration::InferFromSettings(settings);
configuration.SetEntrypoint("sendSemanticsUpdate");

shell->RunEngine(std::move(configuration), [](auto result) {
ASSERT_EQ(result, Engine::RunStatus::Success);
});

message_latch.Wait();
DestroyShell(std::move(shell), task_runners);
}

} // namespace flutter::testing
1 change: 1 addition & 0 deletions shell/platform/darwin/ios/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,7 @@ shared_library("ios_test_flutter") {
"ios_context_noop_unittests.mm",
"ios_surface_noop_unittests.mm",
"platform_message_handler_ios_test.mm",
"platform_view_ios_test.mm",
]
deps = [
":flutter_framework",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ class AccessibilityBridge final : public AccessibilityBridgeIos {

void UpdateSemantics(flutter::SemanticsNodeUpdates nodes,
const flutter::CustomAccessibilityActionUpdates& actions);
void HandleEvent(NSDictionary<NSString*, id>* annotatedEvent);
void HandleMessage(NSDictionary<NSString*, id>* message, FlutterReply reply);
void DispatchSemanticsAction(int32_t id, flutter::SemanticsAction action) override;
void DispatchSemanticsAction(int32_t id,
flutter::SemanticsAction action,
Expand Down Expand Up @@ -98,7 +98,6 @@ class AccessibilityBridge final : public AccessibilityBridgeIos {
int32_t last_focused_semantics_object_id_;

NSMutableDictionary<NSNumber*, SemanticsObject*>* objects_;
FlutterBasicMessageChannel* accessibility_channel_;
int32_t previous_route_id_ = 0;
std::unordered_map<int32_t, flutter::CustomAccessibilityAction> actions_;
std::vector<int32_t> previous_routes_;
Expand Down
Loading