[dashboard + branch] reimplement branch fetching logic (#2924)

For branches on cocoon dashboard, fetch any branch that has recent activities, AND, all the release branches. Sorry for the delay, it took me a huge amount of time to restudy the branching logic and dart syntax. The mocks and stubs in tests took me quite a while to get right.
flutter · Jul 27, 2023 · 8eb603e · 8eb603e
1 parent 07c2214
commit 8eb603e
Show file tree

Hide file tree

Showing 7 changed files with 540 additions and 201 deletions.
diff --git a/app_dart/bin/server.dart b/app_dart/bin/server.dart
@@ -260,7 +260,7 @@ Future<void> main() async {
       '/api/public/get-branches': CacheRequestHandler<Body>(
         cache: cache,
         config: config,
-        delegate: GetBranches(config: config),
+        delegate: GetBranches(config: config, branchService: branchService),
         ttl: const Duration(minutes: 15),
       ),
 

diff --git a/app_dart/lib/src/request_handlers/get_branches.dart b/app_dart/lib/src/request_handlers/get_branches.dart
@@ -6,9 +6,11 @@ import 'dart:async';
 
 import 'package:cocoon_service/src/model/appengine/branch.dart';
 import 'package:process_runner/process_runner.dart';
+import 'package:github/github.dart' as gh;
 
 import '../../cocoon_service.dart';
 import '../service/datastore.dart';
+import '../service/github_service.dart';
 
 /// Return currently active branches across all repos.
 ///
@@ -39,30 +41,43 @@ import '../service/datastore.dart';
 class GetBranches extends RequestHandler<Body> {
   GetBranches({
     required super.config,
+    required this.branchService,
     this.datastoreProvider = DatastoreService.defaultProvider,
     this.processRunner,
   });
 
+  final BranchService branchService;
   final DatastoreServiceProvider datastoreProvider;
   ProcessRunner? processRunner;
 
   static const Duration kActiveBranchActivity = Duration(days: 60);
 
+  bool isRecent(Branch b) {
+    return DateTime.now().millisecondsSinceEpoch - b.lastActivity! < kActiveBranchActivity.inMilliseconds ||
+        config.supportedRepos.map((repo) => Config.defaultBranch(repo)).toSet().contains(b.name);
+  }
+
   @override
   Future<Body> get() async {
     final DatastoreService datastore = datastoreProvider(config.db);
+    List<Branch> branches = await datastore.queryBranches().toList();
 
-    List<Branch> branches = await datastore
-        .queryBranches()
+    // From the dashboard point of view, these are the subset of branches we care about.
+    final RegExp branchRegex = RegExp(r'^main|^master|^flutter-.+|^fuchsia.+');
+
+    // Fetch release branches too.
+    final gh.GitHub github = await config.createGitHubClient(slug: Config.flutterSlug);
+    final GithubService githubService = GithubService(github);
+    final List<Map<String, String>> branchNamesMap =
+        await branchService.getReleaseBranches(githubService: githubService, slug: Config.flutterSlug);
+    final List<String?> releaseBranchNames = branchNamesMap.map((branchMap) => branchMap["branch"]).toList();
+    // Retrieve branches with recent activities and release branches.
+    branches = branches
         .where(
-          (Branch b) =>
-              DateTime.now().millisecondsSinceEpoch - b.lastActivity! < kActiveBranchActivity.inMilliseconds ||
-              <String>['main', 'master'].contains(b.name),
+          (branch) =>
+              (isRecent(branch) && branch.name.contains(branchRegex)) || releaseBranchNames.contains(branch.name),
         )
         .toList();
-    // From the dashboard point of view, these are the subset of branches we care about.
-    final RegExp branchRegex = RegExp(r'^beta|^stable|^main|^master|^flutter-.+|^fuchsia.+');
-    branches = branches.where((branch) => branch.name.contains(branchRegex)).toList();
     return Body.forJson(branches);
   }
 }
diff --git a/app_dart/lib/src/request_handlers/github/branch_subscription.dart b/app_dart/lib/src/request_handlers/github/branch_subscription.dart
@@ -0,0 +1,50 @@
+// Copyright 2019 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+
+import 'package:github/hooks.dart';
+import 'package:meta/meta.dart';
+
+import '../../../protos.dart' as pb;
+import '../../request_handling/body.dart';
+import '../../request_handling/subscription_handler.dart';
+import '../../service/branch_service.dart';
+import '../../service/logging.dart';
+
+const String kWebhookCreateEvent = 'create';
+
+/// Subscription for processing GitHub webhooks relating to branches.
+///
+/// This subscription processes branch events on GitHub into Cocoon.
+@immutable
+class GithubBranchWebhookSubscription extends SubscriptionHandler {
+  /// Creates a subscription for processing GitHub webhooks.
+  const GithubBranchWebhookSubscription({
+    required super.cache,
+    required super.config,
+    required this.branchService,
+  }) : super(subscriptionName: 'github-webhook-branches');
+
+  final BranchService branchService;
+
+  @override
+  Future<Body> post() async {
+    if (message.data == null || message.data!.isEmpty) {
+      log.warning('GitHub webhook message was empty. No-oping');
+      return Body.empty;
+    }
+
+    final pb.GithubWebhookMessage webhook = pb.GithubWebhookMessage.fromJson(message.data!);
+    if (webhook.event != kWebhookCreateEvent) {
+      return Body.empty;
+    }
+
+    log.fine('Processing ${webhook.event}');
+    final CreateEvent createEvent = CreateEvent.fromJson(json.decode(webhook.payload) as Map<String, dynamic>);
+    await branchService.handleCreateRequest(createEvent);
+
+    return Body.empty;
+  }
+}
diff --git a/app_dart/lib/src/service/branch_service.dart b/app_dart/lib/src/service/branch_service.dart
@@ -9,10 +9,14 @@ import 'package:cocoon_service/src/service/github_service.dart';
 import 'package:github/github.dart' as gh;
 import 'package:retry/retry.dart';
 
+import '../model/appengine/branch.dart';
 import '../model/gerrit/commit.dart';
 import '../request_handling/exceptions.dart';
 import 'gerrit_service.dart';
 import 'logging.dart';
+import 'package:cocoon_service/src/service/datastore.dart';
+import 'package:gcloud/db.dart';
+import 'package:github/hooks.dart';
 
 class RetryException implements Exception {}
 
@@ -30,6 +34,42 @@ class BranchService {
   final GerritService gerritService;
   final RetryOptions retryOptions;
 
+  /// Add a [CreateEvent] branch to Datastore.
+  Future<void> handleCreateRequest(CreateEvent createEvent) async {
+    log.info('the branch parsed from string request is ${createEvent.ref}');
+
+    final String? refType = createEvent.refType;
+    if (refType == 'tag') {
+      log.info('create branch event was rejected because it is a tag');
+      return;
+    }
+    final String? branch = createEvent.ref;
+    if (branch == null) {
+      log.fine('Branch is null, exiting early');
+      return;
+    }
+    final gh.RepositorySlug slug = createEvent.repository!.slug();
+    final int lastActivity = createEvent.repository!.pushedAt!.millisecondsSinceEpoch;
+    final bool forked = createEvent.repository!.isFork;
+
+    if (forked) {
+      log.info('create branch event was rejected because the branch is a fork');
+      return;
+    }
+
+    final String id = '${slug.fullName}/$branch';
+    log.info('the id used to create branch key was $id');
+    final DatastoreService datastore = DatastoreService.defaultProvider(config.db);
+    final Key<String> key = datastore.db.emptyKey.append<String>(Branch, id: id);
+    final Branch currentBranch = Branch(key: key, lastActivity: lastActivity);
+    try {
+      await datastore.lookupByValue<Branch>(currentBranch.key);
+    } on KeyNotFoundException {
+      log.info('create branch event was successful since the key is unique');
+      await datastore.insert(<Branch>[currentBranch]);
+    }
+  }
+
   /// Creates a flutter/recipes branch that aligns to a flutter/engine commit.
   ///
   /// Take the example repo history:

diff --git a/app_dart/test/request_handlers/get_branches_test.dart b/app_dart/test/request_handlers/get_branches_test.dart
@@ -5,17 +5,40 @@
 import 'dart:convert';
 
 import 'package:cocoon_service/src/model/appengine/branch.dart';
+import 'package:cocoon_service/src/service/branch_service.dart';
 import 'package:cocoon_service/src/request_handlers/get_branches.dart';
 import 'package:cocoon_service/src/request_handling/body.dart';
+import 'package:cocoon_service/src/service/config.dart';
 import 'package:cocoon_service/src/service/datastore.dart';
 import 'package:gcloud/db.dart';
 import 'package:test/test.dart';
+import 'package:mockito/mockito.dart';
+import 'package:retry/retry.dart';
+import 'package:github/github.dart' as gh;
 
+import '../src/utilities/mocks.mocks.dart';
 import '../src/datastore/fake_config.dart';
 import '../src/datastore/fake_datastore.dart';
 import '../src/request_handling/fake_authentication.dart';
 import '../src/request_handling/fake_http.dart';
 import '../src/request_handling/request_handler_tester.dart';
+import '../src/service/fake_gerrit_service.dart';
+
+String gitHubEncode(String source) {
+  final List<int> utf8Characters = utf8.encode(source);
+  final String base64encoded = base64Encode(utf8Characters);
+  return base64encoded;
+}
+
+void addBranch(String id, FakeDatastoreDB db) {
+  final int lastActivity = DateTime(2019, 5, 15).millisecondsSinceEpoch;
+  final Key<String> branchKey = db.emptyKey.append<String>(Branch, id: id);
+  final Branch currentBranch = Branch(
+    key: branchKey,
+    lastActivity: lastActivity,
+  );
+  db.values[currentBranch.key] = currentBranch;
+}
 
 void main() {
   group('GetBranches', () {
@@ -24,8 +47,13 @@ void main() {
     late GetBranches handler;
     late FakeHttpRequest request;
     late FakeDatastoreDB db;
+    late BranchService branchService;
+    late FakeGerritService gerritService;
+    late MockRepositoriesService repositories;
     FakeClientContext clientContext;
     FakeKeyHelper keyHelper;
+    const String betaBranchName = "flutter-3.5-candidate.1";
+    const String stableBranchName = "flutter-3.4-candidate.5";
 
     Future<T?> decodeHandlerBody<T>() async {
       final Body body = await tester.get(handler);
@@ -38,15 +66,52 @@ void main() {
       request = FakeHttpRequest();
       keyHelper = FakeKeyHelper(applicationContext: clientContext.applicationContext);
       tester = RequestHandlerTester(request: request);
+
+      final MockGitHub github = MockGitHub();
       config = FakeConfig(
+        githubClient: github,
         dbValue: db,
         keyHelperValue: keyHelper,
       );
+
+      repositories = MockRepositoriesService();
+      when(github.repositories).thenReturn(repositories);
+
+      gerritService = FakeGerritService();
+      branchService = BranchService(
+        config: config,
+        gerritService: gerritService,
+        retryOptions: const RetryOptions(maxDelay: Duration.zero),
+      );
+
       handler = GetBranches(
+        branchService: branchService,
         config: config,
         datastoreProvider: (DatastoreDB db) => DatastoreService(config.db, 5),
       );
 
+      when(
+        repositories.listBranches(Config.flutterSlug),
+      ).thenAnswer((Invocation invocation) {
+        return Stream<gh.Branch>.value(gh.Branch("flutter-9.9-candidate.9", null));
+      });
+
+      when(
+        repositories.getContents(any, 'bin/internal/release-candidate-branch.version', ref: "beta"),
+      ).thenAnswer((Invocation invocation) {
+        return Future<gh.RepositoryContents>.value(
+          gh.RepositoryContents(file: gh.GitHubFile(content: gitHubEncode(betaBranchName))),
+        );
+      });
+
+      when(
+        repositories.getContents(any, 'bin/internal/release-candidate-branch.version', ref: "stable"),
+      ).thenAnswer((Invocation invocation) {
+        return Future<gh.RepositoryContents>.value(
+          gh.RepositoryContents(file: gh.GitHubFile(content: gitHubEncode(stableBranchName))),
+        );
+      });
+
       const String id = 'flutter/flutter/branch-created-old';
       final int lastActivity = DateTime.tryParse("2019-05-15T15:20:56Z")!.millisecondsSinceEpoch;
       final Key<String> branchKey = db.emptyKey.append<String>(Branch, id: id);
@@ -162,5 +227,30 @@ void main() {
       expect((result.single)['branch']['branch'], 'flutter-branch-created-now');
       expect((result.single)['id'].runtimeType, String);
     });
+
+    test('should always retrieve release branches', () async {
+      expect(db.values.values.whereType<Branch>().length, 1);
+
+      final String id = '${Config.flutterSlug}/$betaBranchName';
+      addBranch(id, db);
+
+      final String stableId = '${Config.flutterSlug}/$stableBranchName';
+      addBranch(stableId, db);
+
+      expect(db.values.values.whereType<Branch>().length, 3);
+
+      final List<dynamic> result = (await decodeHandlerBody())!;
+      final List<dynamic> expected = [
+        {
+          'id': 'flutter/flutter/$betaBranchName',
+          'branch': {'branch': betaBranchName, 'repository': 'flutter/flutter'},
+        },
+        {
+          'id': 'flutter/flutter/$stableBranchName',
+          'branch': {'branch': stableBranchName, 'repository': 'flutter/flutter'},
+        }
+      ];
+      expect(result, expected);
+    });
   });
 }