Insert a new commit on branch creation (#3005)

Issue: flutter/flutter#132066 This is based on comments from #2991, but the code was significantly different so it made more sense to create a new branch. When a branch is created, get the associated commit and push it to the datastore with the new branch. This will allow dart-internal builders, which run on a new candidate branch creation, to have a commit to save their task results to.
flutter · Aug 23, 2023 · c79008f · c79008f
1 parent 536a174
commit c79008f
Show file tree

Hide file tree

Showing 7 changed files with 2,005 additions and 1,661 deletions.
diff --git a/app_dart/bin/server.dart b/app_dart/bin/server.dart
@@ -8,6 +8,7 @@ import 'dart:math';
 import 'package:appengine/appengine.dart';
 import 'package:cocoon_service/cocoon_service.dart';
 import 'package:cocoon_service/src/request_handlers/github/branch_subscription.dart';
+import 'package:cocoon_service/src/service/commit_service.dart';
 import 'package:gcloud/db.dart';
 
 /// For local development, you might want to set this to true.
@@ -56,6 +57,8 @@ Future<void> main() async {
       gerritService: gerritService,
     );
 
+    final CommitService commitService = CommitService(config: config);
+
     final Map<String, RequestHandler<dynamic>> handlers = <String, RequestHandler<dynamic>>{
       '/api/check_flaky_builders': CheckFlakyBuilders(
         config: config,
@@ -99,6 +102,7 @@ Future<void> main() async {
         config: config,
         cache: cache,
         branchService: branchService,
+        commitService: commitService,
       ),
 
       /// API to run authenticated graphql queries. It requires to pass the graphql query as the body

diff --git a/app_dart/lib/src/request_handlers/github/branch_subscription.dart b/app_dart/lib/src/request_handlers/github/branch_subscription.dart
@@ -4,6 +4,7 @@
 
 import 'dart:convert';
 
+import 'package:cocoon_service/src/service/commit_service.dart';
 import 'package:github/hooks.dart';
 import 'package:meta/meta.dart';
 
@@ -25,9 +26,11 @@ class GithubBranchWebhookSubscription extends SubscriptionHandler {
     required super.cache,
     required super.config,
     required this.branchService,
+    required this.commitService,
   }) : super(subscriptionName: 'github-webhook-branches');
 
   final BranchService branchService;
+  final CommitService commitService;
 
   @override
   Future<Body> post() async {
@@ -45,6 +48,11 @@ class GithubBranchWebhookSubscription extends SubscriptionHandler {
     final CreateEvent createEvent = CreateEvent.fromJson(json.decode(webhook.payload) as Map<String, dynamic>);
     await branchService.handleCreateRequest(createEvent);
 
+    final RegExp candidateBranchRegex = RegExp(r'flutter-\d+\.\d+-candidate\.\d+');
+    if (candidateBranchRegex.hasMatch(createEvent.ref!)) {
+      await commitService.handleCreateGithubRequest(createEvent);
+    }
+
     return Body.empty;
   }
 }
diff --git a/app_dart/lib/src/service/commit_service.dart b/app_dart/lib/src/service/commit_service.dart
@@ -0,0 +1,65 @@
+// Copyright 2021 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:cocoon_service/src/model/appengine/commit.dart';
+import 'package:cocoon_service/src/service/config.dart';
+import 'package:cocoon_service/src/service/github_service.dart';
+import 'package:github/github.dart';
+import 'package:meta/meta.dart';
+import 'package:truncate/truncate.dart';
+
+import 'logging.dart';
+import 'package:cocoon_service/src/service/datastore.dart';
+import 'package:gcloud/db.dart';
+import 'package:github/hooks.dart';
+
+/// A class for doing various actions related to Github commits.
+class CommitService {
+  CommitService({
+    required this.config,
+    @visibleForTesting this.datastoreProvider = DatastoreService.defaultProvider,
+  });
+
+  final Config config;
+  final DatastoreServiceProvider datastoreProvider;
+
+  /// Add a commit based on a [CreateEvent] to the Datastore.
+  Future<void> handleCreateGithubRequest(CreateEvent createEvent) async {
+    final DatastoreService datastore = datastoreProvider(config.db);
+    final RepositorySlug slug = RepositorySlug.full(createEvent.repository!.fullName);
+    final String branch = createEvent.ref!;
+    final Commit commit = await _createCommitFromBranchEvent(datastore, slug, branch);
+
+    try {
+      await datastore.lookupByValue<Commit>(commit.key);
+    } on KeyNotFoundException {
+      log.info('commit does not exist in datastore, inserting into datastore');
+      await datastore.insert(<Commit>[commit]);
+    }
+  }
+
+  Future<Commit> _createCommitFromBranchEvent(DatastoreService datastore, RepositorySlug slug, String branch) async {
+    final GithubService githubService = await config.createDefaultGitHubService();
+    final GitReference gitRef = await githubService.getReference(slug, 'heads/$branch');
+    final String sha = gitRef.object!.sha!;
+    final RepositoryCommit commit = await githubService.github.repositories.getCommit(slug, sha);
+
+    final String id = '${slug.fullName}/$branch/$sha';
+    final Key<String> key = datastore.db.emptyKey.append<String>(Commit, id: id);
+    return Commit(
+      key: key,
+      timestamp: commit.author?.createdAt?.millisecondsSinceEpoch,
+      repository: slug.fullName,
+      sha: commit.sha,
+      author: commit.author?.login,
+      authorAvatarUrl: commit.author?.avatarUrl,
+      // The field has a size of 1500 we need to ensure the commit message
+      // is at most 1500 chars long.
+      message: truncate(commit.commit!.message!, 1490, omission: '...'),
+      branch: branch,
+    );
+  }
+}
diff --git a/app_dart/test/request_handlers/github/branch_subscription_test.dart b/app_dart/test/request_handlers/github/branch_subscription_test.dart
@@ -0,0 +1,81 @@
+// Copyright 2019 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:cocoon_service/cocoon_service.dart';
+import 'package:cocoon_service/src/model/luci/push_message.dart';
+import 'package:cocoon_service/src/request_handlers/github/branch_subscription.dart';
+
+import 'package:mockito/mockito.dart';
+import 'package:test/test.dart';
+
+import '../../src/request_handling/fake_http.dart';
+import '../../src/request_handling/subscription_tester.dart';
+import '../../src/utilities/mocks.dart';
+import '../../src/utilities/webhook_generators.dart';
+
+void main() {
+  late GithubBranchWebhookSubscription webhook;
+  late SubscriptionTester tester;
+  late MockBranchService branchService;
+  late MockCommitService commitService;
+
+  setUp(() {
+    branchService = MockBranchService();
+    commitService = MockCommitService();
+    webhook = GithubBranchWebhookSubscription(
+      config: MockConfig(),
+      cache: CacheService(inMemory: true),
+      commitService: commitService,
+      branchService: branchService,
+    );
+    tester = SubscriptionTester(request: FakeHttpRequest());
+  });
+
+  group('branch subscription', () {
+    test('Ignores empty message', () async {
+      tester.message = const PushMessage();
+
+      await tester.post(webhook);
+
+      verifyNever(branchService.handleCreateRequest(any)).called(0);
+      verifyNever(commitService.handleCreateGithubRequest(any)).called(0);
+    });
+
+    test('Ignores webhook message from event that is not "create"', () async {
+      tester.message = generateGithubWebhookMessage(
+        event: 'pull_request',
+      );
+
+      await tester.post(webhook);
+
+      verifyNever(branchService.handleCreateRequest(any)).called(0);
+      verifyNever(commitService.handleCreateGithubRequest(any)).called(0);
+    });
+
+    test('Successfully stores branch in datastore and does not create a new commit due to not being a candidate branch',
+        () async {
+      tester.message = generateCreateBranchMessage(
+        'cool-branch',
+        'flutter/flutter',
+      );
+
+      await tester.post(webhook);
+
+      verify(branchService.handleCreateRequest(any)).called(1);
+      verifyNever(commitService.handleCreateGithubRequest(any)).called(0);
+    });
+
+    test('Successfully stores branch in datastore and creates a new commit due to being a candidate branch', () async {
+      tester.message = generateCreateBranchMessage(
+        'flutter-1.2-candidate.3',
+        'flutter/flutter',
+      );
+
+      await tester.post(webhook);
+
+      verify(branchService.handleCreateRequest(any)).called(1);
+      verify(commitService.handleCreateGithubRequest(any)).called(1);
+    });
+  });
+}
diff --git a/app_dart/test/service/commit_service_test.dart b/app_dart/test/service/commit_service_test.dart
@@ -0,0 +1,130 @@
+// Copyright 2021 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:cocoon_service/src/model/appengine/commit.dart';
+import 'package:cocoon_service/src/service/commit_service.dart';
+import 'package:github/github.dart';
+
+import 'package:mockito/mockito.dart';
+import 'package:test/test.dart';
+import 'package:github/hooks.dart';
+
+import '../src/datastore/fake_datastore.dart';
+import '../src/utilities/entity_generators.dart';
+import '../src/utilities/mocks.mocks.dart';
+import '../src/utilities/webhook_generators.dart';
+
+void main() {
+  late MockConfig config;
+  late FakeDatastoreDB db;
+  late CommitService commitService;
+  late MockGithubService githubService;
+  late MockRepositoriesService repositories;
+  late MockGitHub github;
+  const String owner = "flutter";
+  const String repository = "engine";
+  const String branch = "coolest-branch";
+  const String sha = "1234";
+  const String message = "Adding null safety";
+  const String avatarUrl = "https://avatars.githubusercontent.com/u/fake-user-num";
+  const String username = "AwesomeGithubUser";
+  const String dateTimeAsString = "2023-08-18T19:27:00Z";
+
+  setUp(() {
+    db = FakeDatastoreDB();
+    github = MockGitHub();
+    githubService = MockGithubService();
+    when(githubService.github).thenReturn(github);
+    repositories = MockRepositoriesService();
+    when(github.repositories).thenReturn(repositories);
+    config = MockConfig();
+    commitService = CommitService(config: config);
+
+    when(config.createDefaultGitHubService()).thenAnswer((_) async => githubService);
+    when(config.db).thenReturn(db);
+  });
+
+  group('handleCreateRequest', () {
+    test('adds commit to db if it does not exist in the datastore', () async {
+      expect(db.values.values.whereType<Commit>().length, 0);
+
+      when(githubService.getReference(RepositorySlug(owner, repository), 'heads/$branch'))
+          .thenAnswer((Invocation invocation) {
+        return Future<GitReference>.value(
+          GitReference(ref: 'refs/$branch', object: GitObject('', sha, '')),
+        );
+      });
+
+      when(repositories.getCommit(RepositorySlug(owner, repository), sha)).thenAnswer((Invocation invocation) {
+        return Future<RepositoryCommit>.value(
+          RepositoryCommit(
+            sha: sha,
+            author: User(
+              createdAt: DateTime.parse(dateTimeAsString),
+              login: username,
+              avatarUrl: avatarUrl,
+            ),
+            commit: GitCommit(message: message),
+          ),
+        );
+      });
+
+      final CreateEvent createEvent = generateCreateBranchEvent(branch, '$owner/$repository');
+      await commitService.handleCreateGithubRequest(createEvent);
+
+      expect(db.values.values.whereType<Commit>().length, 1);
+      final Commit commit = db.values.values.whereType<Commit>().single;
+      expect(commit.repository, "$owner/$repository");
+      expect(commit.message, message);
+      expect(commit.key.id, "$owner/$repository/$branch/$sha");
+      expect(commit.timestamp, DateTime.parse(dateTimeAsString).millisecondsSinceEpoch);
+      expect(commit.sha, sha);
+      expect(commit.author, username);
+      expect(commit.authorAvatarUrl, avatarUrl);
+      expect(commit.branch, branch);
+    });
+
+    test('does not add commit to db if it exists in the datastore', () async {
+      final Commit existingCommit = generateCommit(
+        1,
+        sha: sha,
+        branch: branch,
+        owner: owner,
+        repo: repository,
+        timestamp: 0,
+      );
+      final List<Commit> datastoreCommit = <Commit>[existingCommit];
+      await config.db.commit(inserts: datastoreCommit);
+      expect(db.values.values.whereType<Commit>().length, 1);
+
+      when(githubService.getReference(RepositorySlug(owner, repository), 'heads/$branch'))
+          .thenAnswer((Invocation invocation) {
+        return Future<GitReference>.value(
+          GitReference(ref: 'refs/$branch', object: GitObject('', sha, '')),
+        );
+      });
+
+      when(repositories.getCommit(RepositorySlug(owner, repository), sha)).thenAnswer((Invocation invocation) {
+        return Future<RepositoryCommit>.value(
+          RepositoryCommit(
+            sha: sha,
+            author: User(
+              createdAt: DateTime.parse(dateTimeAsString),
+              login: username,
+              avatarUrl: avatarUrl,
+            ),
+            commit: GitCommit(message: message),
+          ),
+        );
+      });
+
+      final CreateEvent createEvent = generateCreateBranchEvent(branch, '$owner/$repository');
+      await commitService.handleCreateGithubRequest(createEvent);
+
+      expect(db.values.values.whereType<Commit>().length, 1);
+      final Commit commit = db.values.values.whereType<Commit>().single;
+      expect(commit, existingCommit);
+    });
+  });
+}
diff --git a/app_dart/test/src/utilities/mocks.dart b/app_dart/test/src/utilities/mocks.dart
@@ -12,6 +12,7 @@ import 'package:cocoon_service/src/service/access_token_provider.dart';
 import 'package:cocoon_service/src/service/bigquery.dart';
 import 'package:cocoon_service/src/service/branch_service.dart';
 import 'package:cocoon_service/src/service/buildbucket.dart';
+import 'package:cocoon_service/src/service/commit_service.dart';
 import 'package:cocoon_service/src/service/config.dart';
 import 'package:cocoon_service/src/service/datastore.dart';
 import 'package:cocoon_service/src/service/github_checks_service.dart';
@@ -64,6 +65,7 @@ Future<AutoRefreshingAuthClient> authClientProviderShim({
     BigqueryService,
     BranchService,
     BuildBucketClient,
+    CommitService,
     Config,
     DatastoreService,
     FakeEntry,