diff --git a/app_dart/bin/gae_server.dart b/app_dart/bin/gae_server.dart index 4758bcd36..29ff48d38 100644 --- a/app_dart/bin/gae_server.dart +++ b/app_dart/bin/gae_server.dart @@ -9,9 +9,9 @@ import 'package:cocoon_service/cocoon_service.dart'; import 'package:cocoon_service/server.dart'; import 'package:cocoon_service/src/service/build_bucket_v2_client.dart'; import 'package:cocoon_service/src/service/commit_service.dart'; -// import 'package:cocoon_service/src/service/github_checks_service_v2.dart'; -// import 'package:cocoon_service/src/service/luci_build_service_v2.dart'; -// import 'package:cocoon_service/src/service/scheduler_v2.dart'; +import 'package:cocoon_service/src/service/github_checks_service_v2.dart'; +import 'package:cocoon_service/src/service/luci_build_service_v2.dart'; +import 'package:cocoon_service/src/service/scheduler_v2.dart'; import 'package:gcloud/db.dart'; Future main() async { @@ -40,21 +40,21 @@ Future main() async { pubsub: const PubSub(), ); - // final LuciBuildServiceV2 luciBuildServiceV2 = LuciBuildServiceV2( - // config: config, - // cache: cache, - // buildBucketV2Client: buildBucketV2Client, - // pubsub: const PubSub(), - // ); + final LuciBuildServiceV2 luciBuildServiceV2 = LuciBuildServiceV2( + config: config, + cache: cache, + buildBucketV2Client: buildBucketV2Client, + pubsub: const PubSub(), + ); /// Github checks api service used to provide luci test execution status on the Github UI. final GithubChecksService githubChecksService = GithubChecksService( config, ); - // final GithubChecksServiceV2 githubChecksServiceV2 = GithubChecksServiceV2( - // config, - // ); + final GithubChecksServiceV2 githubChecksServiceV2 = GithubChecksServiceV2( + config, + ); // Gerrit service class to communicate with GoB. final GerritService gerritService = GerritService(config: config); @@ -67,12 +67,12 @@ Future main() async { luciBuildService: luciBuildService, ); - // final SchedulerV2 schedulerV2 = SchedulerV2( - // cache: cache, - // config: config, - // githubChecksService: githubChecksServiceV2, - // luciBuildService: luciBuildServiceV2, - // ); + final SchedulerV2 schedulerV2 = SchedulerV2( + cache: cache, + config: config, + githubChecksService: githubChecksServiceV2, + luciBuildService: luciBuildServiceV2, + ); final BranchService branchService = BranchService( config: config, @@ -87,10 +87,14 @@ Future main() async { authProvider: authProvider, branchService: branchService, buildBucketClient: buildBucketClient, + buildBucketV2Client: buildBucketV2Client, gerritService: gerritService, scheduler: scheduler, + schedulerV2: schedulerV2, luciBuildService: luciBuildService, + luciBuildServiceV2: luciBuildServiceV2, githubChecksService: githubChecksService, + githubChecksServiceV2: githubChecksServiceV2, commitService: commitService, swarmingAuthProvider: swarmingAuthProvider, ); diff --git a/app_dart/bin/local_server.dart b/app_dart/bin/local_server.dart index 4021d7a43..db9cff1fe 100644 --- a/app_dart/bin/local_server.dart +++ b/app_dart/bin/local_server.dart @@ -11,9 +11,9 @@ import 'package:cocoon_service/src/model/appengine/cocoon_config.dart'; import 'package:cocoon_service/src/service/build_bucket_v2_client.dart'; import 'package:cocoon_service/src/service/commit_service.dart'; import 'package:cocoon_service/src/service/datastore.dart'; -// import 'package:cocoon_service/src/service/github_checks_service_v2.dart'; -// import 'package:cocoon_service/src/service/luci_build_service_v2.dart'; -// import 'package:cocoon_service/src/service/scheduler_v2.dart'; +import 'package:cocoon_service/src/service/github_checks_service_v2.dart'; +import 'package:cocoon_service/src/service/luci_build_service_v2.dart'; +import 'package:cocoon_service/src/service/scheduler_v2.dart'; import 'package:gcloud/db.dart'; import '../test/src/datastore/fake_datastore.dart'; @@ -47,21 +47,21 @@ Future main() async { pubsub: const PubSub(), ); - // final LuciBuildServiceV2 luciBuildServiceV2 = LuciBuildServiceV2( - // config: config, - // cache: cache, - // buildBucketV2Client: buildBucketV2Client, - // pubsub: const PubSub(), - // ); + final LuciBuildServiceV2 luciBuildServiceV2 = LuciBuildServiceV2( + config: config, + cache: cache, + buildBucketV2Client: buildBucketV2Client, + pubsub: const PubSub(), + ); /// Github checks api service used to provide luci test execution status on the Github UI. final GithubChecksService githubChecksService = GithubChecksService( config, ); - // final GithubChecksServiceV2 githubChecksServiceV2 = GithubChecksServiceV2( - // config, - // ); + final GithubChecksServiceV2 githubChecksServiceV2 = GithubChecksServiceV2( + config, + ); // Gerrit service class to communicate with GoB. final GerritService gerritService = GerritService(config: config); @@ -74,12 +74,12 @@ Future main() async { luciBuildService: luciBuildService, ); - // final SchedulerV2 schedulerV2 = SchedulerV2( - // cache: cache, - // config: config, - // githubChecksService: githubChecksServiceV2, - // luciBuildService: luciBuildServiceV2, - // ); + final SchedulerV2 schedulerV2 = SchedulerV2( + cache: cache, + config: config, + githubChecksService: githubChecksServiceV2, + luciBuildService: luciBuildServiceV2, + ); final BranchService branchService = BranchService( config: config, @@ -94,10 +94,14 @@ Future main() async { authProvider: authProvider, branchService: branchService, buildBucketClient: buildBucketClient, + buildBucketV2Client: buildBucketV2Client, gerritService: gerritService, scheduler: scheduler, + schedulerV2: schedulerV2, luciBuildService: luciBuildService, + luciBuildServiceV2: luciBuildServiceV2, githubChecksService: githubChecksService, + githubChecksServiceV2: githubChecksServiceV2, commitService: commitService, swarmingAuthProvider: swarmingAuthProvider, ); diff --git a/app_dart/lib/server.dart b/app_dart/lib/server.dart index 540a3e44b..fa5495fe3 100644 --- a/app_dart/lib/server.dart +++ b/app_dart/lib/server.dart @@ -6,7 +6,18 @@ import 'dart:io'; import 'dart:math'; import 'package:cocoon_service/cocoon_service.dart'; +import 'package:cocoon_service/src/request_handlers/postsubmit_luci_subscription_v2.dart'; +import 'package:cocoon_service/src/request_handlers/presubmit_luci_subscription_v2.dart'; +import 'package:cocoon_service/src/request_handlers/reset_prod_task_v2.dart'; +import 'package:cocoon_service/src/request_handlers/reset_try_task_v2.dart'; +import 'package:cocoon_service/src/request_handlers/scheduler/batch_backfiller_v2.dart'; +import 'package:cocoon_service/src/request_handlers/scheduler/scheduler_request_subscription.dart'; +import 'package:cocoon_service/src/request_handlers/vacuum_github_commits_v2.dart'; +import 'package:cocoon_service/src/service/build_bucket_v2_client.dart'; import 'package:cocoon_service/src/service/commit_service.dart'; +import 'package:cocoon_service/src/service/github_checks_service_v2.dart'; +import 'package:cocoon_service/src/service/luci_build_service_v2.dart'; +import 'package:cocoon_service/src/service/scheduler_v2.dart'; typedef Server = Future Function(HttpRequest); @@ -18,11 +29,15 @@ Server createServer({ required AuthenticationProvider swarmingAuthProvider, required BranchService branchService, required BuildBucketClient buildBucketClient, + required BuildBucketV2Client buildBucketV2Client, required LuciBuildService luciBuildService, + required LuciBuildServiceV2 luciBuildServiceV2, required GithubChecksService githubChecksService, + required GithubChecksServiceV2 githubChecksServiceV2, required CommitService commitService, required GerritService gerritService, required Scheduler scheduler, + required SchedulerV2 schedulerV2, }) { final Map> handlers = >{ '/api/check_flaky_builders': CheckFlakyBuilders( @@ -37,7 +52,7 @@ Server createServer({ '/api/dart-internal-subscription': DartInternalSubscription( cache: cache, config: config, - buildBucketClient: buildBucketClient, + buildBucketV2Client: buildBucketV2Client, ), '/api/file_flaky_issue_and_pr': FileFlakyIssueAndPR( config: config, @@ -65,24 +80,36 @@ Server createServer({ config: config, cache: cache, gerritService: gerritService, - githubChecksService: githubChecksService, scheduler: scheduler, + schedulerV2: schedulerV2, commitService: commitService, ), '/api/presubmit-luci-subscription': PresubmitLuciSubscription( cache: cache, config: config, - buildBucketClient: buildBucketClient, luciBuildService: luciBuildService, githubChecksService: githubChecksService, scheduler: scheduler, ), + '/api/v2/presubmit-luci-subscription': PresubmitLuciSubscriptionV2( + cache: cache, + config: config, + luciBuildService: luciBuildServiceV2, + githubChecksService: githubChecksServiceV2, + scheduler: schedulerV2, + ), '/api/postsubmit-luci-subscription': PostsubmitLuciSubscription( cache: cache, config: config, scheduler: scheduler, githubChecksService: githubChecksService, ), + '/api/v2/postsubmit-luci-subscription': PostsubmitLuciSubscriptionV2( + cache: cache, + config: config, + scheduler: schedulerV2, + githubChecksService: githubChecksServiceV2, + ), '/api/push-build-status-to-github': PushBuildStatusToGithub( config: config, authenticationProvider: authProvider, @@ -91,26 +118,47 @@ Server createServer({ config: config, authenticationProvider: authProvider, ), + // I do not believe these recieve a build message. '/api/reset-prod-task': ResetProdTask( config: config, authenticationProvider: authProvider, luciBuildService: luciBuildService, scheduler: scheduler, ), + '/api/v2/reset-prod-task': ResetProdTaskV2( + config: config, + authenticationProvider: authProvider, + luciBuildService: luciBuildServiceV2, + scheduler: schedulerV2, + ), '/api/reset-try-task': ResetTryTask( config: config, authenticationProvider: authProvider, scheduler: scheduler, ), + '/api/v2/reset-try-task': ResetTryTaskV2( + config: config, + authenticationProvider: authProvider, + scheduler: schedulerV2, + ), '/api/scheduler/batch-backfiller': BatchBackfiller( config: config, scheduler: scheduler, ), + '/api/v2/scheduler/batch-backfiller': BatchBackfillerV2( + config: config, + scheduler: schedulerV2, + ), '/api/scheduler/batch-request-subscription': SchedulerRequestSubscription( cache: cache, config: config, buildBucketClient: buildBucketClient, ), + '/api/v2/scheduler/batch-request-subscription': SchedulerRequestSubscriptionV2( + cache: cache, + config: config, + buildBucketClient: buildBucketV2Client, + ), '/api/scheduler/vacuum-stale-tasks': VacuumStaleTasks( config: config, ), @@ -144,6 +192,11 @@ Server createServer({ authenticationProvider: authProvider, scheduler: scheduler, ), + '/api/v2/vacuum-github-commits': VacuumGithubCommitsV2( + config: config, + authenticationProvider: authProvider, + scheduler: schedulerV2, + ), /// Returns status of the framework tree. /// diff --git a/app_dart/lib/src/request_handlers/dart_internal_subscription.dart b/app_dart/lib/src/request_handlers/dart_internal_subscription.dart index 4c12be269..35a12e547 100644 --- a/app_dart/lib/src/request_handlers/dart_internal_subscription.dart +++ b/app_dart/lib/src/request_handlers/dart_internal_subscription.dart @@ -2,16 +2,18 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// import 'package:cocoon_service/src/model/luci/buildbucket.dart'; import 'dart:convert'; - -import 'package:cocoon_service/src/model/luci/buildbucket.dart'; +import 'package:buildbucket/buildbucket_pb.dart' as bbv2; +import 'package:fixnum/fixnum.dart'; import 'package:googleapis/firestore/v1.dart'; import 'package:meta/meta.dart'; import '../../cocoon_service.dart'; import '../model/appengine/task.dart'; import '../model/firestore/task.dart' as firestore; -import '../request_handling/subscription_handler.dart'; +import '../request_handling/subscription_handler_v2.dart'; +import '../service/build_bucket_v2_client.dart'; import '../service/datastore.dart'; import '../service/logging.dart'; @@ -24,18 +26,18 @@ import '../service/logging.dart'; /// The PubSub subscription is set up here: /// https://console.cloud.google.com/cloudpubsub/subscription/detail/dart-internal-build-results-sub?project=flutter-dashboard @immutable -class DartInternalSubscription extends SubscriptionHandler { +class DartInternalSubscription extends SubscriptionHandlerV2 { /// Creates an endpoint for listening for dart-internal build results. /// The message should contain a single buildbucket id const DartInternalSubscription({ required super.cache, required super.config, super.authProvider, - required this.buildBucketClient, + required this.buildBucketV2Client, @visibleForTesting this.datastoreProvider = DatastoreService.defaultProvider, }) : super(subscriptionName: 'dart-internal-build-results-sub'); - final BuildBucketClient buildBucketClient; + final BuildBucketV2Client buildBucketV2Client; final DatastoreServiceProvider datastoreProvider; @override @@ -47,17 +49,19 @@ class DartInternalSubscription extends SubscriptionHandler { return Body.empty; } - final dynamic buildData = json.decode(message.data!); - log.info('Build data json: $buildData'); + // This looks to be like we are simply getting the build and not the top level + // buildsPubSub message. + final Map jsonBuildMap = json.decode(message.data!); - if (buildData['build'] == null) { + if (jsonBuildMap['build'] == null) { log.info('no build information in message'); return Body.empty; } - final String project = buildData['build']['builder']['project']; - final String bucket = buildData['build']['builder']['bucket']; - final String builder = buildData['build']['builder']['builder']; + final String project = jsonBuildMap['build']['builder']['project']; + final String bucket = jsonBuildMap['build']['builder']['bucket']; + final String builder = jsonBuildMap['build']['builder']['builder']; + final Int64 buildId = Int64.parseInt(jsonBuildMap['build']['id']); // This should already be covered by the pubsub filter, but adding an additional check // to ensure we don't process builds that aren't from dart-internal/flutter. @@ -76,28 +80,31 @@ class DartInternalSubscription extends SubscriptionHandler { return Body.empty; } - final String buildbucketId = buildData['build']['id']; - log.info('Creating build request object with build id $buildbucketId'); - final GetBuildRequest request = GetBuildRequest( - id: buildbucketId, + log.info('Creating build request object with build id $buildId'); + + final bbv2.GetBuildRequest getBuildRequest = bbv2.GetBuildRequest( + id: buildId, ); log.info( - 'Calling buildbucket api to get build data for build $buildbucketId', + 'Calling buildbucket api to get build data for build $buildId', ); - final Build build = await buildBucketClient.getBuild(request); + + final bbv2.Build existingBuild = await buildBucketV2Client.getBuild(getBuildRequest); + + log.info('Got back existing builder with name: ${existingBuild.builder.builder}'); log.info('Checking for existing task in datastore'); - final Task? existingTask = await datastore.getTaskFromBuildbucketBuild(build); + final Task? existingTask = await datastore.getTaskFromBuildbucketV2Build(existingBuild); late Task taskToInsert; if (existingTask != null) { - log.info('Updating Task from existing Task'); - existingTask.updateFromBuildbucketBuild(build); + log.info('Updating Task from existing Build'); + existingTask.updateFromBuildbucketV2Build(existingBuild); taskToInsert = existingTask; } else { log.info('Creating Task from Buildbucket result'); - taskToInsert = await Task.fromBuildbucketBuild(build, datastore); + taskToInsert = await Task.fromBuildbucketV2Build(existingBuild, datastore); } log.info('Inserting Task into the datastore: ${taskToInsert.toString()}'); diff --git a/app_dart/lib/src/request_handlers/github/webhook_subscription.dart b/app_dart/lib/src/request_handlers/github/webhook_subscription.dart index 1b94f14b7..24704e760 100644 --- a/app_dart/lib/src/request_handlers/github/webhook_subscription.dart +++ b/app_dart/lib/src/request_handlers/github/webhook_subscription.dart @@ -5,6 +5,8 @@ import 'dart:convert'; import 'package:cocoon_service/src/service/commit_service.dart'; +import 'package:cocoon_service/src/service/scheduler.dart'; +import 'package:cocoon_service/src/service/scheduler_v2.dart'; import 'package:github/github.dart'; import 'package:github/hooks.dart'; import 'package:meta/meta.dart'; @@ -18,9 +20,7 @@ import '../../request_handling/subscription_handler.dart'; import '../../service/config.dart'; import '../../service/datastore.dart'; import '../../service/gerrit_service.dart'; -import '../../service/github_checks_service.dart'; import '../../service/logging.dart'; -import '../../service/scheduler.dart'; // Filenames which are not actually tests. const List kNotActuallyATest = [ @@ -68,14 +68,17 @@ class GithubWebhookSubscription extends SubscriptionHandler { required super.cache, required super.config, required this.scheduler, + required this.schedulerV2, required this.gerritService, required this.commitService, - this.githubChecksService, this.datastoreProvider = DatastoreService.defaultProvider, super.authProvider, + // Gets the initial github events from this sub after the webhook uploads them. }) : super(subscriptionName: 'github-webhooks-sub'); /// Cocoon scheduler to trigger tasks against changes from GitHub. + final SchedulerV2 schedulerV2; + final Scheduler scheduler; /// To verify whether a commit was mirrored to GoB. @@ -84,9 +87,6 @@ class GithubWebhookSubscription extends SubscriptionHandler { /// Used to handle push events and create commits based on those events. final CommitService commitService; - /// To provide build status updates to GitHub pull requests. - final GithubChecksService? githubChecksService; - final DatastoreServiceProvider datastoreProvider; @override @@ -107,7 +107,7 @@ class GithubWebhookSubscription extends SubscriptionHandler { case 'check_run': final Map event = jsonDecode(webhook.payload) as Map; final cocoon_checks.CheckRunEvent checkRunEvent = cocoon_checks.CheckRunEvent.fromJson(event); - if (await scheduler.processCheckRun(checkRunEvent) == false) { + if (await schedulerV2.processCheckRun(checkRunEvent) == false) { throw InternalServerError('Failed to process check_run event. checkRunEvent: $checkRunEvent'); } break; @@ -164,7 +164,7 @@ class GithubWebhookSubscription extends SubscriptionHandler { // If it was closed without merging, cancel any outstanding tryjobs. // We'll leave unfinished jobs if it was merged since we care about those // results. - await scheduler.cancelPreSubmitTargets( + await schedulerV2.cancelPreSubmitTargets( pullRequest: pr, reason: (!pr.merged!) ? 'Pull request closed' : 'Pull request merged', ); @@ -173,7 +173,7 @@ class GithubWebhookSubscription extends SubscriptionHandler { log.fine('Pull request ${pr.number} was closed and merged.'); if (await _commitExistsInGob(pr)) { log.fine('Merged commit was found on GoB mirror. Scheduling postsubmit tasks...'); - return scheduler.addPullRequest(pr); + return schedulerV2.addPullRequest(pr); } throw InternalServerError( '${pr.mergeCommitSha!} was not found on GoB. Failing so this event can be retried...', @@ -246,7 +246,7 @@ class GithubWebhookSubscription extends SubscriptionHandler { return; } - await scheduler.triggerPresubmitTargets(pullRequest: pr); + await schedulerV2.triggerPresubmitTargets(pullRequest: pr); } /// Release tooling generates cherrypick pull requests that should be granted an approval. diff --git a/app_dart/lib/src/request_handlers/postsubmit_luci_subscription_v2.dart b/app_dart/lib/src/request_handlers/postsubmit_luci_subscription_v2.dart new file mode 100644 index 000000000..866187d8e --- /dev/null +++ b/app_dart/lib/src/request_handlers/postsubmit_luci_subscription_v2.dart @@ -0,0 +1,174 @@ +// Copyright 2019 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; + +import 'package:buildbucket/buildbucket_pb.dart' as bbv2; +import 'package:cocoon_service/ci_yaml.dart'; +import 'package:cocoon_service/src/model/luci/user_data.dart'; +import 'package:gcloud/db.dart'; +import 'package:googleapis/firestore/v1.dart' hide Status; +import 'package:meta/meta.dart'; + +import '../model/appengine/commit.dart'; +import '../model/appengine/task.dart'; +import '../model/firestore/task.dart' as firestore; +import '../request_handling/body.dart'; +import '../request_handling/exceptions.dart'; +import '../request_handling/subscription_handler_v2.dart'; +import '../service/datastore.dart'; +import '../service/firestore.dart'; +import '../service/logging.dart'; +import '../service/github_checks_service_v2.dart'; +import '../service/scheduler_v2.dart'; + +/// An endpoint for listening to build updates for postsubmit builds. +/// +/// The PubSub subscription is set up here: +/// https://cloud.google.com/cloudpubsub/subscription/detail/build-bucket-postsubmit-sub?project=flutter-dashboard&tab=overview +/// +/// This endpoint is responsible for updating Datastore with the result of builds from LUCI. +@immutable +class PostsubmitLuciSubscriptionV2 extends SubscriptionHandlerV2 { + /// Creates an endpoint for listening to LUCI status updates. + const PostsubmitLuciSubscriptionV2({ + required super.cache, + required super.config, + super.authProvider, + @visibleForTesting this.datastoreProvider = DatastoreService.defaultProvider, + required this.scheduler, + required this.githubChecksService, + }) : super(subscriptionName: 'build-bucket-postsubmit-sub'); + + final DatastoreServiceProvider datastoreProvider; + final SchedulerV2 scheduler; + final GithubChecksServiceV2 githubChecksService; + + @override + Future post() async { + if (message.data == null) { + log.info('no data in message'); + return Body.empty; + } + + final DatastoreService datastore = datastoreProvider(config.db); + final FirestoreService firestoreService = await config.createFirestoreService(); + + final bbv2.PubSubCallBack pubSubCallBack = bbv2.PubSubCallBack(); + pubSubCallBack.mergeFromProto3Json(jsonDecode(message.data!) as Map); + final bbv2.BuildsV2PubSub buildsV2PubSub = pubSubCallBack.buildPubsub; + + Map userDataMap = {}; + try { + userDataMap = json.decode(String.fromCharCodes(pubSubCallBack.userData)); + log.info('User data was not base64 encoded.'); + } on FormatException { + userDataMap = UserData.decodeUserDataBytes(pubSubCallBack.userData); + log.info('Decoding base64 encoded user data.'); + } + + // collect userData + if (userDataMap.isEmpty) { + log.info('User data is empty'); + return Body.empty; + } + + log.fine('userData=$userDataMap'); + + if (!buildsV2PubSub.hasBuild()) { + log.warning('No build was found in message.'); + return Body.empty; + } + + final bbv2.Build build = buildsV2PubSub.build; + // Note that result is no longer present in the output. + log.fine('Updating buildId=${build.id} for result=${build.status}'); + + log.info('build ${build.toProto3Json()}'); + + final String? rawTaskKey = userDataMap['task_key'] as String?; + final String? rawCommitKey = userDataMap['commit_key'] as String?; + final String? taskDocumentName = userDataMap['firestore_task_document_name'] as String?; + if (taskDocumentName == null) { + throw const BadRequestException('userData does not contain firestore_task_document_name'); + } + + final Key commitKey = Key(Key.emptyKey(Partition(null)), Commit, rawCommitKey); + Task? task; + firestore.Task? firestoreTask; + log.info('Looking up task document $kDatabase/documents/${firestore.kTaskCollectionId}/$taskDocumentName...'); + final int taskId = int.parse(rawTaskKey!); + final Key taskKey = Key(commitKey, Task, taskId); + task = await datastore.lookupByValue(taskKey); + firestoreTask = await firestore.Task.fromFirestore( + firestoreService: firestoreService, + documentName: '$kDatabase/documents/${firestore.kTaskCollectionId}/$taskDocumentName', + ); + log.info('Found $firestoreTask'); + + if (_shouldUpdateTask(build, firestoreTask)) { + final String oldTaskStatus = firestoreTask.status; + firestoreTask.updateFromBuildV2(build); + + log.info('Updated firestore task $firestoreTask'); + + task.updateFromBuildbucketV2Build(build); + await datastore.insert([task]); + final List writes = documentsToWrites([firestoreTask], exists: true); + await firestoreService.batchWriteDocuments(BatchWriteRequest(writes: writes), kDatabase); + log.fine('Updated datastore from $oldTaskStatus to ${firestoreTask.status}'); + } else { + log.fine('skip processing for build with status scheduled or task with status finished.'); + } + + final Commit commit = await datastore.lookupByValue(commitKey); + final CiYaml ciYaml = await scheduler.getCiYaml(commit); + final List postsubmitTargets = ciYaml.postsubmitTargets; + if (!postsubmitTargets.any((element) => element.value.name == firestoreTask!.taskName)) { + log.warning('Target ${firestoreTask.taskName} has been deleted from TOT. Skip updating.'); + return Body.empty; + } + final Target target = + postsubmitTargets.singleWhere((Target target) => target.value.name == firestoreTask!.taskName); + if (firestoreTask.status == firestore.Task.statusFailed || + firestoreTask.status == firestore.Task.statusInfraFailure || + firestoreTask.status == firestore.Task.statusCancelled) { + log.fine('Trying to auto-retry...'); + final bool retried = await scheduler.luciBuildService.checkRerunBuilder( + commit: commit, + target: target, + task: task, + datastore: datastore, + taskDocument: firestoreTask, + firestoreService: firestoreService, + ); + log.info('Retried: $retried'); + } + + // Only update GitHub checks if target is not bringup + if (target.value.bringup == false && config.postsubmitSupportedRepos.contains(target.slug)) { + log.info('Updating check status for ${target.getTestName}'); + await githubChecksService.updateCheckStatus( + build: build, + userDataMap: userDataMap, + luciBuildService: scheduler.luciBuildService, + slug: commit.slug, + ); + } + + return Body.empty; + } + + // No need to update task in datastore if + // 1) the build is `scheduled`. Task is marked as `In Progress` + // whenever scheduled, either from scheduler/backfiller/rerun. We need to update + // task in datastore only for + // a) `started`: update info like builder number. + // b) `completed`: update info like status. + // 2) the task is already completed. + // The task may have been marked as completed from test framework via update-task-status API. + bool _shouldUpdateTask(bbv2.Build build, firestore.Task task) { + return build.status != bbv2.Status.SCHEDULED && !firestore.Task.finishedStatusValues.contains(task.status); + } +} diff --git a/app_dart/lib/src/request_handlers/presubmit_luci_subscription.dart b/app_dart/lib/src/request_handlers/presubmit_luci_subscription.dart index 8049eb282..ecdba75a3 100644 --- a/app_dart/lib/src/request_handlers/presubmit_luci_subscription.dart +++ b/app_dart/lib/src/request_handlers/presubmit_luci_subscription.dart @@ -12,7 +12,6 @@ import '../model/luci/push_message.dart'; import '../request_handling/authentication.dart'; import '../request_handling/body.dart'; import '../request_handling/subscription_handler.dart'; -import '../service/buildbucket.dart'; import '../service/config.dart'; import '../service/github_checks_service.dart'; import '../service/logging.dart'; @@ -37,14 +36,12 @@ class PresubmitLuciSubscription extends SubscriptionHandler { const PresubmitLuciSubscription({ required super.cache, required super.config, - required this.buildBucketClient, required this.scheduler, required this.luciBuildService, required this.githubChecksService, AuthenticationProvider? authProvider, }) : super(subscriptionName: 'github-updater'); - final BuildBucketClient buildBucketClient; final LuciBuildService luciBuildService; final GithubChecksService githubChecksService; final Scheduler scheduler; @@ -96,13 +93,6 @@ class PresubmitLuciSubscription extends SubscriptionHandler { return Body.empty; } - /// Gets target's allowed reschedule attempt. - /// - /// Each target can define their own allowed max number of reschedule attemp, and it - /// is defined as a property `presubmit_max_attempts`. - /// - /// If not property is defined, the target doesn't allow a reschedule after failures. - /// Typically the property will be used for targets that are likely flaky. Future _getMaxAttempt( BuildPushMessage buildPushMessage, RepositorySlug slug, diff --git a/app_dart/lib/src/request_handlers/presubmit_luci_subscription_v2.dart b/app_dart/lib/src/request_handlers/presubmit_luci_subscription_v2.dart new file mode 100644 index 000000000..5964a011f --- /dev/null +++ b/app_dart/lib/src/request_handlers/presubmit_luci_subscription_v2.dart @@ -0,0 +1,168 @@ +// Copyright 2019 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; + +import 'package:buildbucket/buildbucket_pb.dart' as bbv2; +import 'package:cocoon_service/src/model/luci/user_data.dart'; +import 'package:cocoon_service/src/request_handling/subscription_handler_v2.dart'; +import 'package:cocoon_service/src/service/github_checks_service_v2.dart'; +import 'package:cocoon_service/src/service/luci_build_service_v2.dart'; +import 'package:cocoon_service/src/service/scheduler_v2.dart'; +import 'package:github/github.dart'; +import 'package:meta/meta.dart'; + +import '../model/appengine/commit.dart'; +import '../model/ci_yaml/ci_yaml.dart'; +import '../model/ci_yaml/target.dart'; +import '../request_handling/authentication.dart'; +import '../request_handling/body.dart'; +import '../service/config.dart'; +import '../service/logging.dart'; + +/// An endpoint for listening to LUCI status updates for scheduled builds. +/// +/// [ScheduleBuildRequest.notify] property is set to tell LUCI to use this +/// PubSub topic. LUCI then publishes updates about build status to that topic, +/// which we listen to on the github-updater subscription. When new messages +/// arrive, they are posted to this web service. +/// +/// The PubSub subscription is set up here: +/// https://console.cloud.google.com/cloudpubsub/subscription/detail/build-bucket-presubmit-sub?project=flutter-dashboard +/// +/// This endpoint is responsible for updating GitHub with the status of +/// completed builds from LUCI. +@immutable +class PresubmitLuciSubscriptionV2 extends SubscriptionHandlerV2 { + /// Creates an endpoint for listening to LUCI status updates. + const PresubmitLuciSubscriptionV2({ + required super.cache, + required super.config, + required this.scheduler, + required this.luciBuildService, + required this.githubChecksService, + AuthenticationProvider? authProvider, + }) : super(subscriptionName: 'build-bucket-presubmit-sub'); + + final LuciBuildServiceV2 luciBuildService; + final GithubChecksServiceV2 githubChecksService; + final SchedulerV2 scheduler; + + @override + Future post() async { + if (message.data == null) { + log.info('no data in message'); + return Body.empty; + } + + final bbv2.PubSubCallBack pubSubCallBack = bbv2.PubSubCallBack(); + pubSubCallBack.mergeFromProto3Json(jsonDecode(message.data!) as Map); + + final bbv2.BuildsV2PubSub buildsV2PubSub = pubSubCallBack.buildPubsub; + + if (!buildsV2PubSub.hasBuild()) { + log.info('no build information in message'); + return Body.empty; + } + + final bbv2.Build build = buildsV2PubSub.build; + + final String builderName = build.builder.builder; + + final List tags = build.tags; + + log.fine('Available tags: ${tags.toString()}'); + + // Skip status update if we can not get the sha tag. + if (tags.where((element) => element.key == 'buildset').isEmpty) { + log.warning('Buildset tag not included, skipping Status Updates'); + return Body.empty; + } + + log.fine('Setting status (${build.status.toString()}) for $builderName'); + + if (!pubSubCallBack.hasUserData()) { + log.info('No user data was found in this request'); + return Body.empty; + } + + Map userDataMap = {}; + try { + userDataMap = json.decode(String.fromCharCodes(pubSubCallBack.userData)); + log.info('User data was not base64 encoded.'); + } on FormatException { + userDataMap = UserData.decodeUserDataBytes(pubSubCallBack.userData); + log.info('Decoding base64 encoded user data.'); + } + + if (userDataMap.containsKey('repo_owner') && userDataMap.containsKey('repo_name')) { + final RepositorySlug slug = + RepositorySlug(userDataMap['repo_owner'] as String, userDataMap['repo_name'] as String); + + bool rescheduled = false; + if (githubChecksService.taskFailed(build.status)) { + final int currentAttempt = githubChecksService.currentAttempt(tags); + final int maxAttempt = await _getMaxAttemptV2( + userDataMap, + slug, + builderName, + ); + if (currentAttempt < maxAttempt) { + rescheduled = true; + log.fine('Rerunning failed task: $builderName'); + await luciBuildService.rescheduleBuild( + builderName: builderName, + build: build, + rescheduleAttempt: currentAttempt + 1, + userDataMap: userDataMap, + ); + } + } + await githubChecksService.updateCheckStatus( + build: build, + userDataMap: userDataMap, + luciBuildService: luciBuildService, + slug: slug, + rescheduled: rescheduled, + ); + } else { + log.info('This repo does not support checks API'); + } + return Body.empty; + } + + Future _getMaxAttemptV2( + Map userData, + RepositorySlug slug, + String builderName, + ) async { + final Commit commit = Commit( + branch: userData['commit_branch'] as String, + repository: slug.fullName, + sha: userData['commit_sha'] as String, + ); + late CiYaml ciYaml; + if (commit.branch == Config.defaultBranch(commit.slug)) { + ciYaml = await scheduler.getCiYaml(commit, validate: true); + } else { + ciYaml = await scheduler.getCiYaml(commit); + } + + // Do not block on the target not found. + if (!ciYaml.presubmitTargets.any((element) => element.value.name == builderName)) { + // do not reschedule + log.warning('Did not find builder with name: $builderName in ciYaml for ${commit.sha}'); + final List availableBuilderList = ciYaml.presubmitTargets.map((Target e) => e.value.name).toList(); + log.warning('ciYaml presubmit targets found: $availableBuilderList'); + return 1; + } + + final Target target = ciYaml.presubmitTargets.where((element) => element.value.name == builderName).single; + final Map properties = target.getProperties(); + if (!properties.containsKey('presubmit_max_attempts')) { + return 1; + } + return properties['presubmit_max_attempts'] as int; + } +} diff --git a/app_dart/lib/src/request_handlers/reset_prod_task.dart b/app_dart/lib/src/request_handlers/reset_prod_task.dart index 402a37236..f6bae84b1 100644 --- a/app_dart/lib/src/request_handlers/reset_prod_task.dart +++ b/app_dart/lib/src/request_handlers/reset_prod_task.dart @@ -4,6 +4,7 @@ import 'dart:async'; +import 'package:cocoon_service/cocoon_service.dart'; import 'package:gcloud/db.dart'; import 'package:github/github.dart'; import 'package:meta/meta.dart'; @@ -16,13 +17,8 @@ import '../model/ci_yaml/ci_yaml.dart'; import '../model/ci_yaml/target.dart'; import '../model/google/token_info.dart'; import '../request_handling/api_request_handler.dart'; -import '../request_handling/body.dart'; import '../request_handling/exceptions.dart'; -import '../service/config.dart'; import '../service/datastore.dart'; -import '../service/firestore.dart'; -import '../service/luci_build_service.dart'; -import '../service/scheduler.dart'; import '../service/logging.dart'; /// Reruns a postsubmit LUCI build. @@ -88,7 +84,7 @@ class ResetProdTask extends ApiRequestHandler { gitBranch: branch!, sha: sha!, ); - final tasks = await datastore.db.query(ancestorKey: commitKey).run().toList(); + final List tasks = await datastore.db.query(ancestorKey: commitKey).run().toList(); final List> futures = >[]; for (final Task task in tasks) { if (!Task.taskFailStatusSet.contains(task.status)) continue; @@ -161,12 +157,16 @@ class ResetProdTask extends ApiRequestHandler { final int currentAttempt = task.attempts!; taskDocumentName = '$kDatabase/documents/${firestore.kTaskCollectionId}/${sha}_${taskName}_$currentAttempt'; } - taskDocument = - await firestore.Task.fromFirestore(firestoreService: firestoreService, documentName: taskDocumentName); + taskDocument = await firestore.Task.fromFirestore( + firestoreService: firestoreService, + documentName: taskDocumentName, + ); + final Map> tags = >{ 'triggered_by': [email], 'trigger_type': ['manual_retry'], }; + final bool isRerunning = await luciBuildService.checkRerunBuilder( commit: commit, task: task, diff --git a/app_dart/lib/src/request_handlers/reset_prod_task_v2.dart b/app_dart/lib/src/request_handlers/reset_prod_task_v2.dart new file mode 100644 index 000000000..ce25a1ae4 --- /dev/null +++ b/app_dart/lib/src/request_handlers/reset_prod_task_v2.dart @@ -0,0 +1,226 @@ +// Copyright 2020 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:buildbucket/buildbucket_pb.dart' as bbv2; +import 'package:cocoon_service/src/service/luci_build_service_v2.dart'; +import 'package:cocoon_service/src/service/scheduler_v2.dart'; +import 'package:gcloud/db.dart'; +import 'package:github/github.dart'; +import 'package:meta/meta.dart'; + +import '../model/appengine/commit.dart'; +import '../model/appengine/key_helper.dart'; +import '../model/appengine/task.dart'; +import '../model/firestore/task.dart' as firestore; +import '../model/ci_yaml/ci_yaml.dart'; +import '../model/ci_yaml/target.dart'; +import '../model/google/token_info.dart'; +import '../request_handling/api_request_handler.dart'; +import '../request_handling/body.dart'; +import '../request_handling/exceptions.dart'; +import '../service/config.dart'; +import '../service/datastore.dart'; +import '../service/firestore.dart'; + +/// Reruns a postsubmit LUCI build. +/// +/// Expects either [taskKeyParam] or a set of params that give enough detail to lookup a task in datastore. +@immutable +class ResetProdTaskV2 extends ApiRequestHandler { + const ResetProdTaskV2({ + required super.config, + required super.authenticationProvider, + required this.luciBuildService, + required this.scheduler, + @visibleForTesting DatastoreServiceProvider? datastoreProvider, + }) : datastoreProvider = datastoreProvider ?? DatastoreService.defaultProvider; + + final DatastoreServiceProvider datastoreProvider; + final LuciBuildServiceV2 luciBuildService; + final SchedulerV2 scheduler; + + static const String branchParam = 'Branch'; + static const String taskKeyParam = 'Key'; + static const String ownerParam = 'Owner'; + static const String repoParam = 'Repo'; + static const String commitShaParam = 'Commit'; + static const String taskDocumentNameParam = 'taskDocumentName'; + + /// Name of the task to be retried. + /// + /// If "all" is given, all failed tasks will be retried. This enables + /// oncalls to quickly recover a commit without the tedium of the UI. + static const String taskParam = 'Task'; + + @override + Future post() async { + final DatastoreService datastore = datastoreProvider(config.db); + final FirestoreService firestoreService = await config.createFirestoreService(); + final String? encodedKey = requestData![taskKeyParam] as String?; + String? branch = requestData![branchParam] as String?; + final String owner = requestData![ownerParam] as String? ?? 'flutter'; + final String? repo = requestData![repoParam] as String?; + final String? sha = requestData![commitShaParam] as String?; + final TokenInfo token = await tokenInfo(request!); + final String? taskName = requestData![taskParam] as String?; + // When Frontend is switched to Firstore, the task document name will be passed over. + final String? taskDocumentName = requestData![taskDocumentNameParam] as String?; + + RepositorySlug? slug; + if (encodedKey != null && encodedKey.isNotEmpty) { + // Check params required for dashboard. + checkRequiredParameters([taskKeyParam]); + } else { + // Checks params required when this API is called with curl. + checkRequiredParameters([commitShaParam, taskParam, repoParam]); + slug = RepositorySlug(owner, repo!); + branch ??= Config.defaultBranch(slug); + } + + if (taskName == 'all') { + final Key commitKey = Commit.createKey( + db: datastore.db, + slug: slug!, + gitBranch: branch!, + sha: sha!, + ); + final List tasks = await datastore.db.query(ancestorKey: commitKey).run().toList(); + final List> futures = >[]; + for (final Task task in tasks) { + if (!Task.taskFailStatusSet.contains(task.status)) continue; + futures.add( + rerun( + datastore: datastore, + firestoreService: firestoreService, + branch: branch, + sha: sha, + taskName: task.name, + slug: slug, + email: token.email!, + ), + ); + } + await Future.wait(futures); + } else { + await rerun( + datastore: datastore, + firestoreService: firestoreService, + encodedKey: encodedKey, + branch: branch, + sha: sha, + taskName: taskName, + taskDocumentName: taskDocumentName, + slug: slug, + email: token.email!, + ignoreChecks: true, + ); + } + + return Body.empty; + } + + Future rerun({ + required DatastoreService datastore, + required FirestoreService firestoreService, + String? encodedKey, + String? branch, + String? sha, + String? taskName, + RepositorySlug? slug, + String? taskDocumentName, + required String email, + bool ignoreChecks = false, + }) async { + // Prepares Datastore task. + final Task task = await _getTaskFromNamedParams( + datastore: datastore, + encodedKey: encodedKey, + branch: branch, + name: taskName, + sha: sha, + slug: slug, + ); + final Commit commit = await _getCommitFromTask(datastore, task); + sha ??= commit.id!.split('/').last; + taskName ??= task.name; + + final CiYaml ciYaml = await scheduler.getCiYaml(commit); + final Target target = ciYaml.postsubmitTargets.singleWhere((Target target) => target.value.name == task.name); + + // Prepares Firestore task. + firestore.Task? taskDocument; + if (taskDocumentName == null) { + final int currentAttempt = task.attempts!; + taskDocumentName = '$kDatabase/documents/${firestore.kTaskCollectionId}/${sha}_${taskName}_$currentAttempt'; + } + taskDocument = await firestore.Task.fromFirestore( + firestoreService: firestoreService, + documentName: taskDocumentName, + ); + + final List tags = [ + bbv2.StringPair( + key: 'triggered_by', + value: email, + ), + bbv2.StringPair( + key: 'trigger_type', + value: 'manual_retry', + ), + ]; + + final bool isRerunning = await luciBuildService.checkRerunBuilder( + commit: commit, + task: task, + target: target, + datastore: datastore, + tags: tags, + ignoreChecks: ignoreChecks, + firestoreService: firestoreService, + taskDocument: taskDocument, + ); + + // For human retries from the dashboard, notify if a task failed to rerun. + if (ignoreChecks && isRerunning == false) { + throw InternalServerError('Failed to rerun $taskName'); + } + } + + /// Retrieve [Task] from [DatastoreService] from either an encoded key or commit + task name info. + /// + /// If [encodedKey] is passed, [KeyHelper] will decode it directly and return the associated entity. + /// + /// Otherwise, [name], [branch], [sha], and [slug] must be passed to find the [Task]. + Future _getTaskFromNamedParams({ + required DatastoreService datastore, + String? encodedKey, + String? branch, + String? name, + String? sha, + RepositorySlug? slug, + }) async { + if (encodedKey != null && encodedKey.isNotEmpty) { + final Key key = config.keyHelper.decode(encodedKey) as Key; + return datastore.lookupByValue(key); + } + final Key commitKey = Commit.createKey( + db: datastore.db, + slug: slug!, + gitBranch: branch!, + sha: sha!, + ); + return Task.fromDatastore( + datastore: datastore, + commitKey: commitKey, + name: name!, + ); + } + + /// Returns the [Commit] associated with [Task]. + Future _getCommitFromTask(DatastoreService datastore, Task task) async { + return (await datastore.lookupByKey(>[task.parentKey!])).single!; + } +} diff --git a/app_dart/lib/src/request_handlers/reset_try_task_v2.dart b/app_dart/lib/src/request_handlers/reset_try_task_v2.dart new file mode 100644 index 000000000..3278b18f7 --- /dev/null +++ b/app_dart/lib/src/request_handlers/reset_try_task_v2.dart @@ -0,0 +1,62 @@ +// Copyright 2020 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:cocoon_service/src/service/scheduler_v2.dart'; +import 'package:github/github.dart'; +import 'package:meta/meta.dart'; + +import '../../cocoon_service.dart'; +import '../request_handling/api_request_handler.dart'; +import '../request_handling/exceptions.dart'; + +/// Runs all the applicable tasks for a given PR and commit hash. This will be +/// used to unblock rollers when creating a new commit is not possible. +@immutable +class ResetTryTaskV2 extends ApiRequestHandler { + const ResetTryTaskV2({ + required super.config, + required super.authenticationProvider, + required this.scheduler, + }); + + final SchedulerV2 scheduler; + + static const String kOwnerParam = 'owner'; + static const String kRepoParam = 'repo'; + static const String kPullRequestNumberParam = 'pr'; + static const String kBuilderParam = 'builders'; + + @override + Future get() async { + checkRequiredQueryParameters([kRepoParam, kPullRequestNumberParam]); + final String owner = request!.uri.queryParameters[kOwnerParam] ?? 'flutter'; + final String repo = request!.uri.queryParameters[kRepoParam]!; + final String pr = request!.uri.queryParameters[kPullRequestNumberParam]!; + final String builders = request!.uri.queryParameters[kBuilderParam] ?? ''; + final List builderList = getBuilderList(builders); + + final int? prNumber = int.tryParse(pr); + if (prNumber == null) { + throw const BadRequestException('$kPullRequestNumberParam must be a number'); + } + final RepositorySlug slug = RepositorySlug(owner, repo); + final GitHub github = await config.createGitHubClient(slug: slug); + final PullRequest pullRequest = await github.pullRequests.get(slug, prNumber); + await scheduler.triggerPresubmitTargets(pullRequest: pullRequest, builderTriggerList: builderList); + return Body.empty; + } + + /// Parses [builders] to a String list. + /// + /// The [builders] parameter is expecting comma joined string, e.g. 'builder1, builder2'. + /// Returns an empty list if no [builders] is specified. + List getBuilderList(String builders) { + if (builders.isEmpty) { + return []; + } + return builders.split(',').map((String builder) => builder.trim()).toList(); + } +} diff --git a/app_dart/lib/src/request_handlers/scheduler/batch_backfiller_v2.dart b/app_dart/lib/src/request_handlers/scheduler/batch_backfiller_v2.dart new file mode 100644 index 000000000..8a9710ce5 --- /dev/null +++ b/app_dart/lib/src/request_handlers/scheduler/batch_backfiller_v2.dart @@ -0,0 +1,261 @@ +// Copyright 2019 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:cocoon_service/cocoon_service.dart'; +import 'package:cocoon_service/src/model/appengine/task.dart'; +import 'package:cocoon_service/src/model/firestore/task.dart' as firestore; +import 'package:cocoon_service/src/service/datastore.dart'; +import 'package:cocoon_service/src/service/scheduler/policy.dart'; +import 'package:cocoon_service/src/service/scheduler_v2.dart'; +import 'package:gcloud/db.dart'; +import 'package:github/github.dart'; +import 'package:googleapis/firestore/v1.dart'; +import 'package:meta/meta.dart'; +import 'package:retry/retry.dart'; + +import '../../model/ci_yaml/ci_yaml.dart'; +import '../../model/ci_yaml/target.dart'; +import '../../request_handling/exceptions.dart'; +import '../../service/logging.dart'; + +/// Cron request handler for scheduling targets when capacity becomes available. +/// +/// Targets that have a [BatchPolicy] need to have backfilling enabled to ensure that ToT is always being tested. +@immutable +class BatchBackfillerV2 extends RequestHandler { + /// Creates a subscription for sending BuildBucket requests. + const BatchBackfillerV2({ + required super.config, + required this.scheduler, + @visibleForTesting this.datastoreProvider = DatastoreService.defaultProvider, + }); + + final DatastoreServiceProvider datastoreProvider; + final SchedulerV2 scheduler; + + @override + Future get() async { + final List> futures = >[]; + + for (RepositorySlug slug in config.supportedRepos) { + futures.add(backfillRepository(slug)); + } + + // Process all repos asynchronously + await Future.wait(futures); + + return Body.empty; + } + + Future backfillRepository(RepositorySlug slug) async { + final DatastoreService datastore = datastoreProvider(config.db); + final List tasks = + await (datastore.queryRecentTasks(slug: slug, commitLimit: config.backfillerCommitLimit)).toList(); + + // Construct Task columns to scan for backfilling + final Map> taskMap = >{}; + for (FullTask fullTask in tasks) { + if (taskMap.containsKey(fullTask.task.name)) { + taskMap[fullTask.task.name]!.add(fullTask); + } else { + taskMap[fullTask.task.name!] = [fullTask]; + } + } + + // Check if should be scheduled (there is no yellow runs). Run the most recent gray. + List> backfill = >[]; + for (List taskColumn in taskMap.values) { + final FullTask task = taskColumn.first; + final CiYaml ciYaml = await scheduler.getCiYaml(task.commit); + final List ciYamlTargets = ciYaml.backfillTargets; + // Skips scheduling if the task is not in TOT commit anymore. + final bool taskInToT = ciYamlTargets.map((Target target) => target.value.name).toList().contains(task.task.name); + if (!taskInToT) { + continue; + } + final Target target = ciYamlTargets.singleWhere((target) => target.value.name == task.task.name); + if (target.schedulerPolicy is! BatchPolicy) { + continue; + } + final FullTask? backfillTask = _backfillTask(target, taskColumn); + final int? priority = backfillPriority(taskColumn.map((e) => e.task).toList()); + if (priority != null && backfillTask != null) { + backfill.add(Tuple(target, backfillTask, priority)); + } + } + + // Get the number of targets to be backfilled in each cycle. + backfill = getFilteredBackfill(backfill); + + log.fine('Backfilling ${backfill.length} builds'); + log.fine(backfill.map((Tuple tuple) => tuple.first.value.name)); + + // Update tasks status as in progress to avoid duplicate scheduling. + final List backfillTasks = backfill.map((Tuple tuple) => tuple.second.task).toList(); + try { + await datastore.withTransaction((Transaction transaction) async { + transaction.queueMutations(inserts: backfillTasks); + await transaction.commit(); + log.fine( + 'Updated ${backfillTasks.length} tasks: ${backfillTasks.map((e) => e.name).toList()} when backfilling.', + ); + }); + // TODO(keyonghan): remove try catch logic after validated to work. + try { + await updateTaskDocuments(backfillTasks); + } catch (error) { + log.warning('Failed to update batch backfilled task documents in Firestore: $error'); + } + + // Schedule all builds asynchronously. + // Schedule after db updates to avoid duplicate scheduling when db update fails. + await _scheduleWithRetries(backfill); + } catch (error) { + log.severe('Failed to update tasks when backfilling: $error'); + } + } + + /// Updates task documents in Firestore. + Future updateTaskDocuments(List tasks) async { + if (tasks.isEmpty) { + return; + } + final List taskDocuments = tasks.map((e) => firestore.taskToDocument(e)).toList(); + final List writes = documentsToWrites(taskDocuments, exists: true); + final FirestoreService firestoreService = await config.createFirestoreService(); + await firestoreService.writeViaTransaction(writes); + } + + /// Filters [config.backfillerTargetLimit] targets to backfill. + /// + /// High priority targets will be guranteed to get back filled first. If more targets + /// than [config.backfillerTargetLimit], pick the limited number of targets after a + /// shuffle. This is to make sure all targets are picked with the same chance. + List> getFilteredBackfill(List> backfill) { + if (backfill.length <= config.backfillerTargetLimit) { + return backfill; + } + final List> filteredBackfill = >[]; + final List> highPriorityBackfill = + backfill.where((element) => element.third == LuciBuildService.kRerunPriority).toList(); + final List> normalPriorityBackfill = + backfill.where((element) => element.third != LuciBuildService.kRerunPriority).toList(); + if (highPriorityBackfill.length >= config.backfillerTargetLimit) { + highPriorityBackfill.shuffle(); + filteredBackfill.addAll(highPriorityBackfill.sublist(0, config.backfillerTargetLimit)); + } else { + filteredBackfill.addAll(highPriorityBackfill); + normalPriorityBackfill.shuffle(); + filteredBackfill + .addAll(normalPriorityBackfill.sublist(0, config.backfillerTargetLimit - highPriorityBackfill.length)); + } + return filteredBackfill; + } + + /// Schedules tasks with retry when hitting pub/sub server errors. + Future _scheduleWithRetries(List> backfill) async { + const RetryOptions retryOptions = Config.schedulerRetry; + try { + await retryOptions.retry( + () async { + final List>> tupleLists = + await Future.wait>>(backfillRequestList(backfill)); + if (tupleLists.any((List> tupleList) => tupleList.isNotEmpty)) { + final int nonEmptyListLenght = tupleLists.where((element) => element.isNotEmpty).toList().length; + log.info('Backfill fails and retry backfilling $nonEmptyListLenght targets.'); + backfill = _updateBackfill(backfill, tupleLists); + throw InternalServerError('Failed to backfill ${backfill.length} targets.'); + } + }, + retryIf: (Exception e) => e is InternalServerError, + ); + } catch (error) { + log.severe('Failed to backfill ${backfill.length} targets due to error: $error'); + } + } + + /// Updates the [backfill] list with those that fail to get scheduled. + /// + /// [tupleLists] maintains the same tuple order as those in [backfill]. + /// Each element from [backfill] is encapsulated as a list in [tupleLists] to prepare for + /// [scheduler.luciBuildService.schedulePostsubmitBuilds]. + List> _updateBackfill( + List> backfill, + List>> tupleLists, + ) { + final List> updatedBackfill = >[]; + for (int i = 0; i < tupleLists.length; i++) { + if (tupleLists[i].isNotEmpty) { + updatedBackfill.add(backfill[i]); + } + } + return updatedBackfill; + } + + /// Creates a list of backfill requests. + List>>> backfillRequestList(List> backfill) { + final List>>> futures = >>>[]; + for (Tuple tuple in backfill) { + // TODO(chillers): The backfill priority is always going to be low. If this is a ToT task, we should run it at the default priority. + final Tuple toBeScheduled = Tuple( + tuple.first, + tuple.second.task, + tuple.third, + ); + futures.add( + scheduler.luciBuildService.schedulePostsubmitBuilds( + commit: tuple.second.commit, + toBeScheduled: [toBeScheduled], + ), + ); + } + + return futures; + } + + /// Returns priority for back filled targets. + /// + /// Skips scheduling newly created targets whose available entries are + /// less than `BatchPolicy.kBatchSize`. + /// + /// Uses a higher priority if there is an earlier failed build. Otherwise, + /// uses default `LuciBuildService.kBackfillPriority` + int? backfillPriority(List tasks) { + if (tasks.length < BatchPolicy.kBatchSize) { + return null; + } + if (shouldRerunPriority(tasks, BatchPolicy.kBatchSize)) { + return LuciBuildService.kRerunPriority; + } + return LuciBuildService.kBackfillPriority; + } + + /// Returns the most recent [FullTask] to backfill. + /// + /// A [FullTask] is only returned iff: + /// 1. There are no running builds (yellow) + /// 2. There are tasks that haven't been run (gray) + /// + /// This is naive, and doesn't rely on knowing the actual Flutter infra capacity. + /// + /// Otherwise, returns null indicating nothing should be backfilled. + FullTask? _backfillTask(Target target, List tasks) { + final List relevantTasks = tasks.where((FullTask task) => task.task.name == target.value.name).toList(); + if (relevantTasks.any((FullTask task) => task.task.status == Task.statusInProgress)) { + // Don't schedule more builds where there is already a running task + return null; + } + + final List backfillTask = + relevantTasks.where((FullTask task) => task.task.status == Task.statusNew).toList(); + if (backfillTask.isEmpty) { + return null; + } + + // First item in the list is guranteed to be most recent. + // Mark task as in progress to ensure it isn't scheduled over + backfillTask.first.task.status = Task.statusInProgress; + return backfillTask.first; + } +} diff --git a/app_dart/lib/src/request_handlers/scheduler/scheduler_request_subscription.dart b/app_dart/lib/src/request_handlers/scheduler/scheduler_request_subscription.dart new file mode 100644 index 000000000..44f505b28 --- /dev/null +++ b/app_dart/lib/src/request_handlers/scheduler/scheduler_request_subscription.dart @@ -0,0 +1,124 @@ +// Copyright 2019 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; + +import 'package:cocoon_service/src/request_handling/subscription_handler_v2.dart'; +import 'package:meta/meta.dart'; +import 'package:retry/retry.dart'; + +import '../../../cocoon_service.dart'; +import '../../service/build_bucket_v2_client.dart'; +import 'package:buildbucket/buildbucket_pb.dart' as bbv2; +import '../../request_handling/exceptions.dart'; +import '../../service/logging.dart'; + +/// Subscription for making requests to BuildBucket. +/// +/// The PubSub subscription is set up here: +/// https://console.cloud.google.com/cloudpubsub/subscription/detail/cocoon-scheduler-requests?project=flutter-dashboard +/// +/// This endpoint allows Cocoon to defer BuildBucket requests off the main request loop. This is critical when new +/// commits are pushed, and they can schedule 100+ builds at once. +/// +/// This endpoint takes in a POST request with the JSON of a [bbv2.BatchRequest]. In practice, the +/// [bbv2.BatchRequest] should contain a single request. +@immutable +class SchedulerRequestSubscriptionV2 extends SubscriptionHandlerV2 { + /// Creates a subscription for sending BuildBucket requests. + const SchedulerRequestSubscriptionV2({ + required super.cache, + required super.config, + required this.buildBucketClient, + super.authProvider, + this.retryOptions = Config.schedulerRetry, + }) : super(subscriptionName: 'cocoon-scheduler-requests-sub'); + + final BuildBucketV2Client buildBucketClient; + + final RetryOptions retryOptions; + + @override + Future post() async { + if (message.data == null) { + log.info('no data in message'); + throw const BadRequestException('no data in message'); + } + + // final String data = message.data!; + log.fine('attempting to read message ${message.data}'); + + final bbv2.BatchRequest batchRequest = bbv2.BatchRequest.create(); + + // Merge from json only works with the integer field names. + batchRequest.mergeFromProto3Json(jsonDecode(message.data!) as Map); + + log.info('Read the following data: ${batchRequest.toProto3Json().toString()}'); + + /// Retry scheduling builds upto 3 times. + /// + /// Log error message when still failing after retry. Avoid endless rescheduling + /// by acking the pub/sub message without throwing an exception. + String? unscheduledBuilds; + try { + await retryOptions.retry( + () async { + final List requestsToRetry = await _sendBatchRequest(batchRequest); + + // Make a copy of the requests that are passed in as if simply access the list + // we make changes for all instances. + final List requestListCopy = []; + requestListCopy.addAll(requestsToRetry); + batchRequest.requests.clear(); + batchRequest.requests.addAll(requestListCopy); + + unscheduledBuilds = requestsToRetry.map((e) => e.scheduleBuild.builder).toString(); + if (requestsToRetry.isNotEmpty) { + throw InternalServerError('Failed to schedule builds: $unscheduledBuilds.'); + } + }, + retryIf: (Exception e) => e is InternalServerError, + ); + } catch (e) { + log.warning('Failed to schedule builds on exception: $unscheduledBuilds.'); + return Body.forString('Failed to schedule builds: $unscheduledBuilds.'); + } + + return Body.empty; + } + + /// Returns [List] of requests that need to be retried. + Future> _sendBatchRequest(bbv2.BatchRequest request) async { + log.info('Sending batch request for ${request.toProto3Json().toString()}'); + + bbv2.BatchResponse response; + try { + response = await buildBucketClient.batch(request); + } catch (e) { + log.severe('Exception making batch Requests.'); + rethrow; + } + + log.info('Made ${request.requests.length} and received ${response.responses.length}'); + log.info('Responses: ${response.responses}'); + + // By default, retry everything. Then remove requests with a verified response. + // THese are the requests in the batch request object. Just requests. + final List retry = request.requests; + + for (bbv2.BatchResponse_Response batchResponseResponse in response.responses) { + if (batchResponseResponse.hasScheduleBuild()) { + retry.removeWhere((element) => batchResponseResponse.scheduleBuild.builder == element.scheduleBuild.builder); + } else { + log.warning('Response does not have schedule build: $batchResponseResponse'); + } + + if (batchResponseResponse.hasError() && batchResponseResponse.error.code != 0) { + log.info('Non-zero grpc code: $batchResponseResponse'); + } + } + + return retry; + } +} diff --git a/app_dart/lib/src/request_handlers/vacuum_github_commits_v2.dart b/app_dart/lib/src/request_handlers/vacuum_github_commits_v2.dart new file mode 100644 index 000000000..b283e6e1e --- /dev/null +++ b/app_dart/lib/src/request_handlers/vacuum_github_commits_v2.dart @@ -0,0 +1,121 @@ +// Copyright 2019 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:cocoon_service/src/service/scheduler_v2.dart'; +import 'package:gcloud/db.dart'; +import 'package:github/github.dart' as gh; +import 'package:meta/meta.dart'; +import 'package:truncate/truncate.dart'; + +import '../model/appengine/commit.dart'; +import '../request_handling/api_request_handler.dart'; +import '../request_handling/body.dart'; +import '../service/config.dart'; +import '../service/datastore.dart'; +import '../service/github_service.dart'; +import '../service/logging.dart'; + +/// Query GitHub for commits from the past day and ensure they exist in datastore. +@immutable +class VacuumGithubCommitsV2 extends ApiRequestHandler { + const VacuumGithubCommitsV2({ + required super.config, + required super.authenticationProvider, + required this.scheduler, + @visibleForTesting this.datastoreProvider = DatastoreService.defaultProvider, + }); + + final DatastoreServiceProvider datastoreProvider; + + final SchedulerV2 scheduler; + + static const String branchParam = 'branch'; + + @override + Future get() async { + final DatastoreService datastore = datastoreProvider(config.db); + + for (gh.RepositorySlug slug in config.supportedRepos) { + final String branch = request!.uri.queryParameters[branchParam] ?? Config.defaultBranch(slug); + await _vacuumRepository(slug, datastore: datastore, branch: branch); + } + + return Body.empty; + } + + Future _vacuumRepository( + gh.RepositorySlug slug, { + DatastoreService? datastore, + required String branch, + }) async { + final GithubService githubService = await config.createGithubService(slug); + final List commits = await _vacuumBranch( + slug, + branch, + datastore: datastore, + githubService: githubService, + ); + await scheduler.addCommits(commits); + } + + Future> _vacuumBranch( + gh.RepositorySlug slug, + String branch, { + DatastoreService? datastore, + required GithubService githubService, + }) async { + List commits = []; + // Sliding window of times to add commits from. + final DateTime queryAfter = DateTime.now().subtract(const Duration(days: 1)); + final DateTime queryBefore = DateTime.now().subtract(const Duration(minutes: 3)); + try { + log.fine('Listing commit for slug: $slug branch: $branch and msSinceEpoch: ${queryAfter.millisecondsSinceEpoch}'); + commits = await githubService.listBranchedCommits(slug, branch, queryAfter.millisecondsSinceEpoch); + log.fine('Retrieved ${commits.length} commits from GitHub'); + // Do not try to add recent commits as they may already be processed + // by cocoon, which can cause race conditions. + commits = commits + .where( + (gh.RepositoryCommit commit) => + commit.commit!.committer!.date!.millisecondsSinceEpoch < queryBefore.millisecondsSinceEpoch, + ) + .toList(); + } on gh.GitHubError catch (error) { + log.severe('$error'); + } + + return _toDatastoreCommit(slug, commits, datastore, branch); + } + + /// Convert [gh.RepositoryCommit] to Cocoon's [Commit] format. + Future> _toDatastoreCommit( + gh.RepositorySlug slug, + List commits, + DatastoreService? datastore, + String branch, + ) async { + final List recentCommits = []; + for (gh.RepositoryCommit commit in commits) { + final String id = '${slug.fullName}/$branch/${commit.sha}'; + final Key key = datastore!.db.emptyKey.append(Commit, id: id); + recentCommits.add( + Commit( + key: key, + timestamp: commit.commit!.committer!.date!.millisecondsSinceEpoch, + repository: slug.fullName, + sha: commit.sha!, + author: commit.author!.login!, + authorAvatarUrl: commit.author!.avatarUrl!, + // The field has a size of 1500 we need to ensure the commit message + // is at most 1500 chars long. + message: truncate(commit.commit!.message!, 1490, omission: '...'), + branch: branch, + ), + ); + } + return recentCommits; + } +} diff --git a/app_dart/lib/src/service/build_bucket_v2_client.dart b/app_dart/lib/src/service/build_bucket_v2_client.dart index 70527eef6..71fe1751f 100644 --- a/app_dart/lib/src/service/build_bucket_v2_client.dart +++ b/app_dart/lib/src/service/build_bucket_v2_client.dart @@ -70,7 +70,6 @@ class BuildBucketV2Client { log.info('Making bbv2 request with path: $url and body: $request'); - //TODO most likely have issues here: final http.Response response = await httpClient.post( url, body: request, diff --git a/app_dart/test/request_handlers/dart_internal_subscription_test.dart b/app_dart/test/request_handlers/dart_internal_subscription_test.dart index ebbcdfb10..886369522 100644 --- a/app_dart/test/request_handlers/dart_internal_subscription_test.dart +++ b/app_dart/test/request_handlers/dart_internal_subscription_test.dart @@ -2,13 +2,16 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:convert'; + +import 'package:buildbucket/buildbucket_pb.dart' as bbv2; import 'package:cocoon_service/cocoon_service.dart'; import 'package:cocoon_service/src/model/appengine/commit.dart'; import 'package:cocoon_service/src/model/appengine/task.dart'; import 'package:cocoon_service/src/model/firestore/task.dart' as firestore; -import 'package:cocoon_service/src/model/luci/buildbucket.dart'; -import 'package:cocoon_service/src/model/luci/push_message.dart' as push; +import 'package:cocoon_service/src/model/luci/pubsub_message_v2.dart'; import 'package:cocoon_service/src/service/datastore.dart'; +import 'package:fixnum/fixnum.dart'; import 'package:gcloud/db.dart'; import 'package:googleapis/firestore/v1.dart' hide Status; import 'package:mockito/mockito.dart'; @@ -17,53 +20,86 @@ import 'package:test/test.dart'; import '../src/datastore/fake_config.dart'; import '../src/request_handling/fake_authentication.dart'; import '../src/request_handling/fake_http.dart'; -import '../src/request_handling/subscription_tester.dart'; +import '../src/request_handling/subscription_v2_tester.dart'; import '../src/utilities/entity_generators.dart'; import '../src/utilities/mocks.dart'; void main() { + // Omit the timestamps for expect purposes. + const String buildJson = ''' +{ + "id": "8766855135863637953", + "builder": { + "project": "dart-internal", + "bucket": "flutter", + "builder": "Linux packaging_release_builder" + }, + "number": 123456, + "status": "SUCCESS", + "input": { + "gitilesCommit": { + "project": "flutter/flutter", + "id": "HASH12345", + "ref": "refs/heads/test-branch" + } + } +} +'''; + + const String buildMessageJson = ''' +{ + "build": { + "id": "8766855135863637953", + "builder": { + "project": "dart-internal", + "bucket": "flutter", + "builder": "Linux packaging_release_builder" + }, + "number": 123456, + "status": "SUCCESS", + "input": { + "gitilesCommit": { + "project": "flutter/flutter", + "id": "HASH12345", + "ref": "refs/heads/test-branch" + } + } + } +} +'''; + late DartInternalSubscription handler; late FakeConfig config; late FakeHttpRequest request; - late MockBuildBucketClient buildBucketClient; - late SubscriptionTester tester; + late MockBuildBucketV2Client buildBucketV2Client; + late SubscriptionV2Tester tester; late MockFirestoreService mockFirestoreService; late Commit commit; - final DateTime startTime = DateTime(2023, 1, 1, 0, 0, 0); - final DateTime endTime = DateTime(2023, 1, 1, 0, 14, 23); + + // ignore: unused_local_variable const String project = 'dart-internal'; const String bucket = 'flutter'; const String builder = 'Linux packaging_release_builder'; - const int buildId = 123456; + const int buildNumber = 123456; + // ignore: unused_local_variable + final Int64 buildId = Int64(8766855135863637953); const String fakeHash = 'HASH12345'; const String fakeBranch = 'test-branch'; - const String fakePubsubMessage = ''' - { - "build": { - "id": "$buildId", - "builder": { - "project": "$project", - "bucket": "$bucket", - "builder": "$builder" - } - } - } - '''; setUp(() async { mockFirestoreService = MockFirestoreService(); config = FakeConfig(firestoreService: mockFirestoreService); - buildBucketClient = MockBuildBucketClient(); + buildBucketV2Client = MockBuildBucketV2Client(); handler = DartInternalSubscription( cache: CacheService(inMemory: true), config: config, authProvider: FakeAuthenticationProvider(), - buildBucketClient: buildBucketClient, + buildBucketV2Client: buildBucketV2Client, datastoreProvider: (DatastoreDB db) => DatastoreService(config.db, 5), ); request = FakeHttpRequest(); - tester = SubscriptionTester( + tester = SubscriptionV2Tester( request: request, ); @@ -76,27 +112,20 @@ void main() { timestamp: 0, ); - final Build fakeBuild = Build( - builderId: const BuilderId(project: project, bucket: bucket, builder: builder), - number: buildId, - id: 'fake-build-id', - status: Status.success, - startTime: startTime, - endTime: endTime, - input: const Input( - gitilesCommit: GitilesCommit( - project: 'flutter/flutter', - hash: fakeHash, - ref: 'refs/heads/$fakeBranch', - ), - ), - ); + // final bbv2.PubSubCallBack pubSubCallBackTest = bbv2.PubSubCallBack(); + // pubSubCallBackTest.mergeFromProto3Json(jsonDecode(message)); + final bbv2.Build build = bbv2.Build().createEmptyInstance(); + build.mergeFromProto3Json(jsonDecode(buildJson) as Map); + + const PushMessageV2 pushMessageV2 = PushMessageV2(data: buildJson, messageId: '798274983'); + tester.message = pushMessageV2; + when( - buildBucketClient.getBuild( + buildBucketV2Client.getBuild( any, buildBucketUri: 'https://cr-buildbucket.appspot.com/prpc/buildbucket.v2.Builds', ), - ).thenAnswer((_) => Future.value(fakeBuild)); + ).thenAnswer((_) => Future.value(build)); final List datastoreCommit = [commit]; await config.db.commit(inserts: datastoreCommit); @@ -111,12 +140,12 @@ void main() { ).thenAnswer((Invocation invocation) { return Future.value(BatchWriteResponse()); }); - tester.message = const push.PushMessage(data: fakePubsubMessage); + tester.message = const PushMessageV2(data: buildMessageJson); await tester.post(handler); verify( - buildBucketClient.getBuild(any), + buildBucketV2Client.getBuild(any), ).called(1); // This is used for testing to pull the data out of the "datastore" so that @@ -124,7 +153,7 @@ void main() { late Task taskInDb; late Commit commitInDb; config.db.values.forEach((k, v) { - if (v is Task && v.buildNumberList == buildId.toString()) { + if (v is Task && v.buildNumberList == buildNumber.toString()) { taskInDb = v; } if (v is Commit) { @@ -146,16 +175,13 @@ void main() { // Ensure the task in the db is exactly what we expect final Task expectedTask = Task( attempts: 1, - buildNumber: buildId, - buildNumberList: buildId.toString(), + buildNumber: buildNumber, + buildNumberList: buildNumber.toString(), builderName: builder, commitKey: commitInDb.key, - createTimestamp: startTime.millisecondsSinceEpoch, - endTimestamp: endTime.millisecondsSinceEpoch, luciBucket: bucket, name: builder, stageName: 'dart-internal', - startTimestamp: startTime.millisecondsSinceEpoch, status: 'Succeeded', key: commit.key.append(Task), timeoutInMinutes: 0, @@ -194,12 +220,9 @@ void main() { buildNumberList: existingTaskId.toString(), builderName: builder, commitKey: commit.key, - createTimestamp: startTime.millisecondsSinceEpoch, - endTimestamp: endTime.millisecondsSinceEpoch, luciBucket: bucket, name: builder, stageName: 'dart-internal', - startTimestamp: startTime.millisecondsSinceEpoch, status: 'Succeeded', key: commit.key.append(Task), timeoutInMinutes: 0, @@ -210,17 +233,18 @@ void main() { final List datastoreCommit = [fakeTask]; await config.db.commit(inserts: datastoreCommit); - tester.message = const push.PushMessage(data: fakePubsubMessage); + const PushMessageV2 pushMessageV2 = PushMessageV2(data: buildMessageJson, messageId: '798274983'); + tester.message = pushMessageV2; await tester.post(handler); verify( - buildBucketClient.getBuild(any), + buildBucketV2Client.getBuild(any), ).called(1); // This is used for testing to pull the data out of the "datastore" so that // we can verify what was saved. - final String expectedBuilderList = '${existingTaskId.toString()},${buildId.toString()}'; + final String expectedBuilderList = '${existingTaskId.toString()},${buildNumber.toString()}'; late Task taskInDb; late Commit commitInDb; config.db.values.forEach((k, v) { @@ -246,16 +270,13 @@ void main() { // Ensure the task in the db is exactly what we expect final Task expectedTask = Task( attempts: 2, - buildNumber: buildId, + buildNumber: buildNumber, buildNumberList: expectedBuilderList, builderName: builder, commitKey: commitInDb.key, - createTimestamp: startTime.millisecondsSinceEpoch, - endTimestamp: endTime.millisecondsSinceEpoch, luciBucket: bucket, name: builder, stageName: 'dart-internal', - startTimestamp: startTime.millisecondsSinceEpoch, status: 'Succeeded', key: commit.key.append(Task), timeoutInMinutes: 0, @@ -278,67 +299,93 @@ void main() { expect(insertedTaskDocument.status, expectedTask.status); }); - test('ignores null message', () async { - tester.message = const push.PushMessage(data: null); - expect(await tester.post(handler), equals(Body.empty)); - }); - test('ignores message with empty build data', () async { - tester.message = const push.PushMessage(data: '{}'); + tester.message = const PushMessageV2(); expect(await tester.post(handler), equals(Body.empty)); }); + // // TODO create a construction method for this to simplify testing. test('ignores message not from flutter bucket', () async { - tester.message = const push.PushMessage( - data: ''' - { - "build": { - "id": "$buildId", - "builder": { - "project": "$project", - "bucket": "dart", - "builder": "$builder" - } + const String dartMessage = ''' +{ + "build": { + "id": "8766855135863637953", + "builder": { + "project": "dart-internal", + "bucket": "dart", + "builder": "Linux packaging_release_builder" + }, + "number": 123456, + "status": "SUCCESS", + "input": { + "gitilesCommit": { + "project":"flutter/flutter", + "id":"HASH12345", + "ref":"refs/heads/test-branch" } } - ''', - ); + } +} +'''; + + const PushMessageV2 pushMessageV2 = PushMessageV2(data: dartMessage, messageId: '798274983'); + tester.message = pushMessageV2; expect(await tester.post(handler), equals(Body.empty)); }); test('ignores message not from dart-internal project', () async { - tester.message = const push.PushMessage( - data: ''' - { - "build": { - "id": "$buildId", - "builder": { - "project": "different-project", - "bucket": "$bucket", - "builder": "$builder" - } + const String unsupportedProjectMessage = ''' +{ + "build": { + "id": "8766855135863637953", + "builder": { + "project": "unsupported-project", + "bucket": "dart", + "builder": "Linux packaging_release_builder" + }, + "number": 123456, + "status": "SUCCESS", + "input": { + "gitilesCommit": { + "project": "flutter/flutter", + "id": "HASH12345", + "ref": "refs/heads/test-branch" } } - ''', - ); + } +} +'''; + + const PushMessageV2 pushMessageV2 = PushMessageV2(data: unsupportedProjectMessage, messageId: '798274983'); + tester.message = pushMessageV2; expect(await tester.post(handler), equals(Body.empty)); }); test('ignores message not from an accepted builder', () async { - tester.message = const push.PushMessage( - data: ''' - { - "build": { - "id": "$buildId", - "builder": { - "project": "different-project", - "bucket": "$bucket", - "builder": "different-builder" - } + const String unknownBuilderMessage = ''' +{ + "build": { + "id": "8766855135863637953", + "builder": { + "project": "dart-internal", + "bucket": "dart", + "builder": "different builder" + }, + "number": 123456, + "status": "SUCCESS", + "input": { + "gitilesCommit": { + "project": "flutter/flutter", + "id": "HASH12345", + "ref": "refs/heads/test-branch" } } - ''', - ); + } +} +'''; + + const PushMessageV2 pushMessageV2 = PushMessageV2(data: unknownBuilderMessage, messageId: '798274983'); + tester.message = pushMessageV2; expect(await tester.post(handler), equals(Body.empty)); }); } diff --git a/app_dart/test/request_handlers/github/webhook_subscription_test.dart b/app_dart/test/request_handlers/github/webhook_subscription_test.dart index 941a85505..48506c9df 100644 --- a/app_dart/test/request_handlers/github/webhook_subscription_test.dart +++ b/app_dart/test/request_handlers/github/webhook_subscription_test.dart @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'package:buildbucket/buildbucket_pb.dart' as bbv2; + import 'package:cocoon_service/src/model/appengine/commit.dart'; import 'package:cocoon_service/src/model/luci/buildbucket.dart'; import 'package:cocoon_service/src/model/luci/push_message.dart' as pm; @@ -19,10 +21,12 @@ import '../../src/datastore/fake_config.dart'; import '../../src/datastore/fake_datastore.dart'; import '../../src/request_handling/fake_http.dart'; import '../../src/request_handling/subscription_tester.dart'; +import '../../src/service/fake_build_bucket_v2_client.dart'; import '../../src/service/fake_buildbucket.dart'; import '../../src/service/fake_github_service.dart'; import '../../src/service/fake_gerrit_service.dart'; import '../../src/service/fake_scheduler.dart'; +import '../../src/service/fake_scheduler_v2.dart'; import '../../src/utilities/entity_generators.dart'; import '../../src/utilities/mocks.dart'; import '../../src/utilities/webhook_generators.dart'; @@ -30,17 +34,18 @@ import '../../src/utilities/webhook_generators.dart'; void main() { late GithubWebhookSubscription webhook; late FakeBuildBucketClient fakeBuildBucketClient; + late FakeBuildBucketV2Client fakeBuildBucketV2Client; late FakeConfig config; late FakeDatastoreDB db; late FakeGithubService githubService; late FakeHttpRequest request; late FakeScheduler scheduler; + late FakeSchedulerV2 schedulerV2; late FakeGerritService gerritService; late MockCommitService commitService; late MockGitHub gitHubClient; late MockFirestoreService mockFirestoreService; late MockGithubChecksUtil mockGithubChecksUtil; - late MockGithubChecksService mockGithubChecksService; late MockIssuesService issuesService; late MockPullRequestsService pullRequestsService; late SubscriptionTester tester; @@ -89,15 +94,17 @@ void main() { when(pullRequestsService.edit(any, any, title: anyNamed('title'), state: anyNamed('state'), base: anyNamed('base'))) .thenAnswer((_) async => PullRequest()); fakeBuildBucketClient = FakeBuildBucketClient(); + fakeBuildBucketV2Client = FakeBuildBucketV2Client(); mockGithubChecksUtil = MockGithubChecksUtil(); scheduler = FakeScheduler( config: config, buildbucket: fakeBuildBucketClient, githubChecksUtil: mockGithubChecksUtil, ); + schedulerV2 = + FakeSchedulerV2(config: config, buildbucket: fakeBuildBucketV2Client, githubChecksUtil: mockGithubChecksUtil); tester = SubscriptionTester(request: request); - mockGithubChecksService = MockGithubChecksService(); when(gitHubClient.issues).thenReturn(issuesService); when(gitHubClient.pullRequests).thenReturn(pullRequestsService); when(mockGithubChecksUtil.createCheckRun(any, any, any, any, output: anyNamed('output'))).thenAnswer((_) async { @@ -114,8 +121,8 @@ void main() { cache: CacheService(inMemory: true), datastoreProvider: (_) => DatastoreService(config.db, 5), gerritService: gerritService, - githubChecksService: mockGithubChecksService, scheduler: scheduler, + schedulerV2: schedulerV2, commitService: commitService, ); }); @@ -250,9 +257,9 @@ void main() { ); await tester.post(webhook); - - expect(scheduler.cancelPreSubmitTargetsCallCnt, 1); - expect(scheduler.addPullRequestCallCnt, 0); + // TODO this is v2 to route event temporarily from v1 to v2. + expect(schedulerV2.cancelPreSubmitTargetsCallCnt, 1); + expect(schedulerV2.addPullRequestCallCnt, 0); }); test('Acts on closed, cancels presubmit targets, add pr for postsubmit target create', () async { @@ -269,8 +276,8 @@ void main() { await tester.post(webhook); - expect(scheduler.cancelPreSubmitTargetsCallCnt, 1); - expect(scheduler.addPullRequestCallCnt, 1); + expect(schedulerV2.cancelPreSubmitTargetsCallCnt, 1); + expect(schedulerV2.addPullRequestCallCnt, 1); }); test('Acts on opened against master when default is main', () async { @@ -315,8 +322,8 @@ void main() { ), ).called(1); - expect(scheduler.triggerPresubmitTargetsCallCount, 1); - scheduler.resetTriggerPresubmitTargetsCallCount(); + expect(schedulerV2.triggerPresubmitTargetsCallCount, 1); + schedulerV2.resetTriggerPresubmitTargetsCallCount(); }); test('Acts on edited against master when default is main', () async { @@ -362,8 +369,8 @@ void main() { ), ).called(1); - expect(scheduler.triggerPresubmitTargetsCallCount, 1); - scheduler.resetTriggerPresubmitTargetsCallCount(); + expect(schedulerV2.triggerPresubmitTargetsCallCount, 1); + schedulerV2.resetTriggerPresubmitTargetsCallCount(); }); // We already schedule checks when a draft is opened, don't need to re-test @@ -389,46 +396,46 @@ void main() { expect(batchRequestCalled, isFalse); }); - // TODO reeneable after merge. - // test('Triggers builds when opening a draft PR', () async { - // const int issueNumber = 123; - - // tester.message = generateGithubWebhookMessage( - // action: 'opened', - // number: issueNumber, - // isDraft: true, - // ); - // bool batchRequestCalled = false; - - // Future getBatchResponse() async { - // batchRequestCalled = true; - // return BatchResponse( - // responses: [ - // Response( - // searchBuilds: SearchBuildsResponse( - // builds: [ - // generateBuild(999, name: 'Linux', status: Status.ended), - // ], - // ), - // ), - // Response( - // searchBuilds: SearchBuildsResponse( - // builds: [ - // generateBuild(998, name: 'Linux', status: Status.ended), - // ], - // ), - // ), - // ], - // ); - // } - - // fakeBuildBucketClient.batchResponse = getBatchResponse; - - // await tester.post(webhook); - - // expect(batchRequestCalled, isTrue); - // expect(scheduler.cancelPreSubmitTargetsCallCnt, 1); - // }); + test('Triggers builds when opening a draft PR', () async { + const int issueNumber = 123; + + tester.message = generateGithubWebhookMessage( + action: 'opened', + number: issueNumber, + isDraft: true, + ); + + bool batchRequestCalled = false; + + Future getBatchResponse() async { + batchRequestCalled = true; + return bbv2.BatchResponse( + responses: [ + bbv2.BatchResponse_Response( + searchBuilds: bbv2.SearchBuildsResponse( + builds: [ + bbv2.Build(number: 999, builder: bbv2.BuilderID(builder: 'Linux'), status: bbv2.Status.SUCCESS), + ], + ), + ), + bbv2.BatchResponse_Response( + searchBuilds: bbv2.SearchBuildsResponse( + builds: [ + bbv2.Build(number: 998, builder: bbv2.BuilderID(builder: 'Linux'), status: bbv2.Status.SUCCESS), + ], + ), + ), + ], + ); + } + + fakeBuildBucketV2Client.batchResponse = getBatchResponse; + + await tester.post(webhook); + + expect(batchRequestCalled, isTrue); + expect(schedulerV2.cancelPreSubmitTargetsCallCnt, 1); + }); test('Does nothing against cherry pick PR', () async { const int issueNumber = 123; @@ -2186,45 +2193,45 @@ void foo() { ); }); - // TODO reeneable after merge. - // test('When synchronized, cancels existing builds and schedules new ones', () async { - // const int issueNumber = 12345; - // bool batchRequestCalled = false; - // Future getBatchResponse() async { - // batchRequestCalled = true; - // return BatchResponse( - // responses: [ - // Response( - // searchBuilds: SearchBuildsResponse( - // builds: [ - // generateBuild(999, name: 'Linux', status: Status.ended), - // ], - // ), - // ), - // Response( - // searchBuilds: SearchBuildsResponse( - // builds: [ - // generateBuild(998, name: 'Linux', status: Status.ended), - // ], - // ), - // ), - // ], - // ); - // } - - // fakeBuildBucketClient.batchResponse = getBatchResponse; - - // tester.message = generateGithubWebhookMessage( - // action: 'synchronize', - // number: issueNumber, - // ); - - // final MockRepositoriesService mockRepositoriesService = MockRepositoriesService(); - // when(gitHubClient.repositories).thenReturn(mockRepositoriesService); - - // await tester.post(webhook); - // expect(batchRequestCalled, isTrue); - // }); + test('When synchronized, cancels existing builds and schedules new ones', () async { + const int issueNumber = 12345; + bool batchRequestCalled = false; + + Future getBatchResponse() async { + batchRequestCalled = true; + return bbv2.BatchResponse( + responses: [ + bbv2.BatchResponse_Response( + searchBuilds: bbv2.SearchBuildsResponse( + builds: [ + bbv2.Build(number: 999, builder: bbv2.BuilderID(builder: 'Linux'), status: bbv2.Status.SUCCESS), + ], + ), + ), + bbv2.BatchResponse_Response( + searchBuilds: bbv2.SearchBuildsResponse( + builds: [ + bbv2.Build(number: 998, builder: bbv2.BuilderID(builder: 'Linux'), status: bbv2.Status.SUCCESS), + ], + ), + ), + ], + ); + } + + fakeBuildBucketV2Client.batchResponse = getBatchResponse; + + tester.message = generateGithubWebhookMessage( + action: 'synchronize', + number: issueNumber, + ); + + final MockRepositoriesService mockRepositoriesService = MockRepositoriesService(); + when(gitHubClient.repositories).thenReturn(mockRepositoriesService); + + await tester.post(webhook); + expect(batchRequestCalled, isTrue); + }); group('BuildBucket', () { const int issueNumber = 123; diff --git a/app_dart/test/request_handlers/postsubmit_luci_subscription_v2_test.dart b/app_dart/test/request_handlers/postsubmit_luci_subscription_v2_test.dart new file mode 100644 index 000000000..e4e3cdab7 --- /dev/null +++ b/app_dart/test/request_handlers/postsubmit_luci_subscription_v2_test.dart @@ -0,0 +1,582 @@ +// Copyright 2019 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:buildbucket/buildbucket_pb.dart' as bbv2; +import 'package:cocoon_service/cocoon_service.dart'; +import 'package:cocoon_service/src/model/appengine/commit.dart'; +import 'package:cocoon_service/src/model/appengine/task.dart'; +import 'package:cocoon_service/src/model/firestore/commit.dart' as firestore_commit; +import 'package:cocoon_service/src/model/firestore/task.dart' as firestore; +import 'package:cocoon_service/src/request_handlers/postsubmit_luci_subscription_v2.dart'; +import 'package:cocoon_service/src/request_handling/exceptions.dart'; +import 'package:cocoon_service/src/service/datastore.dart'; +import 'package:googleapis/firestore/v1.dart'; +import 'package:mockito/mockito.dart'; +import 'package:test/test.dart'; + +import '../src/datastore/fake_config.dart'; +import '../src/request_handling/fake_authentication.dart'; +import '../src/request_handling/fake_http.dart'; +import '../src/request_handling/subscription_v2_tester.dart'; +import '../src/service/fake_luci_build_service_v2.dart'; +import '../src/service/fake_scheduler_v2.dart'; +import '../src/utilities/build_bucket_v2_messages.dart'; +import '../src/utilities/entity_generators.dart'; +import '../src/utilities/mocks.dart'; +import 'package:fixnum/fixnum.dart'; + +void main() { + late PostsubmitLuciSubscriptionV2 handler; + late FakeConfig config; + late FakeHttpRequest request; + late MockFirestoreService mockFirestoreService; + late SubscriptionV2Tester tester; + late MockGithubChecksServiceV2 mockGithubChecksService; + late MockGithubChecksUtil mockGithubChecksUtil; + late FakeSchedulerV2 scheduler; + firestore.Task? firestoreTask; + firestore_commit.Commit? firestoreCommit; + late int attempt; + + setUp(() async { + firestoreTask = null; + attempt = 0; + mockGithubChecksUtil = MockGithubChecksUtil(); + mockFirestoreService = MockFirestoreService(); + config = FakeConfig( + maxLuciTaskRetriesValue: 3, + firestoreService: mockFirestoreService, + ); + mockGithubChecksService = MockGithubChecksServiceV2(); + when(mockGithubChecksService.githubChecksUtil).thenReturn(mockGithubChecksUtil); + when(mockGithubChecksUtil.createCheckRun(any, any, any, any, output: anyNamed('output'))) + .thenAnswer((_) async => generateCheckRun(1, name: 'Linux A')); + when( + mockGithubChecksService.updateCheckStatus( + build: anyNamed('build'), + userDataMap: anyNamed('userDataMap'), + luciBuildService: anyNamed('luciBuildService'), + slug: anyNamed('slug'), + ), + ).thenAnswer((_) async => true); + when( + mockFirestoreService.getDocument( + captureAny, + ), + ).thenAnswer((Invocation invocation) { + attempt++; + if (attempt == 1) { + return Future.value( + firestoreTask, + ); + } else { + return Future.value( + firestoreCommit, + ); + } + }); + when( + mockFirestoreService.queryRecentCommits( + limit: captureAnyNamed('limit'), + slug: captureAnyNamed('slug'), + branch: captureAnyNamed('branch'), + ), + ).thenAnswer((Invocation invocation) { + return Future>.value( + [firestoreCommit!], + ); + }); + // when( + // mockFirestoreService.getDocument( + // 'projects/flutter-dashboard/databases/cocoon/documents/tasks/87f88734747805589f2131753620d61b22922822_Linux A_1', + // ), + // ).thenAnswer((Invocation invocation) { + // return Future.value( + // firestoreCommit, + // ); + // }); + when( + mockFirestoreService.batchWriteDocuments( + captureAny, + captureAny, + ), + ).thenAnswer((Invocation invocation) { + return Future.value(BatchWriteResponse()); + }); + final FakeLuciBuildServiceV2 luciBuildService = FakeLuciBuildServiceV2( + config: config, + githubChecksUtil: mockGithubChecksUtil, + ); + scheduler = FakeSchedulerV2( + ciYaml: exampleConfig, + config: config, + luciBuildService: luciBuildService, + ); + handler = PostsubmitLuciSubscriptionV2( + cache: CacheService(inMemory: true), + config: config, + authProvider: FakeAuthenticationProvider(), + githubChecksService: mockGithubChecksService, + datastoreProvider: (_) => DatastoreService(config.db, 5), + scheduler: scheduler, + ); + request = FakeHttpRequest(); + + tester = SubscriptionV2Tester( + request: request, + ); + }); + + test('throws exception when task document name is not in message', () async { + const Map userDataMap = { + 'commit_key': 'flutter/main/abc123', + }; + + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.SUCCESS, + builder: '', + userData: userDataMap, + ); + + expect(() => tester.post(handler), throwsA(isA())); + }); + + test('updates task based on message', () async { + firestoreTask = generateFirestoreTask(1, attempts: 2, name: 'Linux A'); + when(mockGithubChecksService.currentAttempt(any)).thenAnswer((_) => 2); + final Commit commit = generateCommit(1, sha: '87f88734747805589f2131753620d61b22922822'); + final Task task = generateTask( + 4507531199512576, + name: 'Linux A', + parent: commit, + ); + final String taskDocumentName = '${commit.sha}_${task.name}_${task.attempts}'; + + final Map userDataMap = { + 'task_key': '${task.key.id}', + 'commit_key': '${task.key.parent?.id}', + 'firestore_commit_document_name': commit.sha, + 'firestore_task_document_name': taskDocumentName, + }; + + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.SUCCESS, + userData: userDataMap, + ); + + config.db.values[commit.key] = commit; + config.db.values[task.key] = task; + + expect(task.status, Task.statusNew); + expect(task.endTimestamp, 0); + + // Firestore checks before API call. + expect(firestoreTask!.status, Task.statusNew); + expect(firestoreTask!.buildNumber, null); + + await tester.post(handler); + + expect(task.status, Task.statusSucceeded); + expect(task.endTimestamp, 1697672824674); + + // Firestore checks after API call. + final List captured = verify(mockFirestoreService.batchWriteDocuments(captureAny, captureAny)).captured; + expect(captured.length, 2); + final BatchWriteRequest batchWriteRequest = captured[0] as BatchWriteRequest; + expect(batchWriteRequest.writes!.length, 1); + final Document updatedDocument = batchWriteRequest.writes![0].update!; + expect(updatedDocument.name, firestoreTask!.name); + expect(firestoreTask!.status, Task.statusSucceeded); + expect(firestoreTask!.buildNumber, 259942); + }); + + test('skips task processing when build is with scheduled status', () async { + firestoreTask = generateFirestoreTask(1, name: 'Linux A', status: firestore.Task.statusInProgress); + final Commit commit = generateCommit(1, sha: '87f88734747805589f2131753620d61b22922822'); + final Task task = generateTask( + 4507531199512576, + name: 'Linux A', + parent: commit, + status: Task.statusInProgress, + ); + config.db.values[task.key] = task; + config.db.values[commit.key] = commit; + final String taskDocumentName = '${commit.sha}_${task.name}_${task.attempts}'; + + final Map userDataMap = { + 'task_key': '${task.key.id}', + 'commit_key': '${task.key.parent?.id}', + 'firestore_commit_document_name': commit.sha, + 'firestore_task_document_name': taskDocumentName, + }; + + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.SCHEDULED, + builder: 'Linux A', + userData: userDataMap, + ); + + expect(firestoreTask!.status, firestore.Task.statusInProgress); + expect(firestoreTask!.attempts, 1); + expect(await tester.post(handler), Body.empty); + expect(firestoreTask!.status, firestore.Task.statusInProgress); + }); + + test('skips task processing when task has already finished', () async { + firestoreTask = generateFirestoreTask(1, name: 'Linux A', status: firestore.Task.statusSucceeded); + final Commit commit = generateCommit(1, sha: '87f88734747805589f2131753620d61b22922822'); + final Task task = generateTask( + 4507531199512576, + name: 'Linux A', + parent: commit, + status: Task.statusSucceeded, + ); + config.db.values[task.key] = task; + config.db.values[commit.key] = commit; + final String taskDocumentName = '${commit.sha}_${task.name}_${task.attempts}'; + + final Map userDataMap = { + 'task_key': '${task.key.id}', + 'commit_key': '${task.key.parent?.id}', + 'firestore_commit_document_name': commit.sha, + 'firestore_task_document_name': taskDocumentName, + }; + + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.STARTED, + builder: 'Linux A', + userData: userDataMap, + ); + + expect(task.status, Task.statusSucceeded); + expect(task.attempts, 1); + expect(await tester.post(handler), Body.empty); + expect(task.status, Task.statusSucceeded); + }); + + test('skips task processing when target has been deleted', () async { + firestoreTask = generateFirestoreTask(1, name: 'Linux B', status: firestore.Task.statusSucceeded); + final Commit commit = generateCommit(1, sha: '87f88734747805589f2131753620d61b22922822'); + final Task task = generateTask( + 4507531199512576, + name: 'Linux B', + parent: commit, + status: Task.statusSucceeded, + ); + config.db.values[task.key] = task; + config.db.values[commit.key] = commit; + final String taskDocumentName = '${commit.sha}_${task.name}_${task.attempts}'; + + final Map userDataMap = { + 'task_key': '${task.key.id}', + 'commit_key': '${task.key.parent?.id}', + 'firestore_commit_document_name': commit.sha, + 'firestore_task_document_name': taskDocumentName, + }; + + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.STARTED, + builder: 'Linux B', + userData: userDataMap, + ); + + expect(task.status, Task.statusSucceeded); + expect(task.attempts, 1); + expect(await tester.post(handler), Body.empty); + }); + + test('does not fail on empty user data', () async { + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.SUCCESS, + builder: 'Linux A', + ); + expect(await tester.post(handler), Body.empty); + }); + + test('on failed builds auto-rerun the build', () async { + firestoreTask = generateFirestoreTask( + 1, + name: 'Linux A', + status: firestore.Task.statusFailed, + commitSha: '87f88734747805589f2131753620d61b22922822', + ); + firestoreCommit = generateFirestoreCommit(1, sha: '87f88734747805589f2131753620d61b22922822'); + final Commit commit = generateCommit(1, sha: '87f88734747805589f2131753620d61b22922822'); + final Task task = generateTask( + 4507531199512576, + name: 'Linux A', + parent: commit, + status: Task.statusFailed, + ); + config.db.values[task.key] = task; + config.db.values[commit.key] = commit; + final String taskDocumentName = '${commit.sha}_${task.name}_${task.attempts}'; + + final Map userDataMap = { + 'task_key': '${task.key.id}', + 'commit_key': '${task.key.parent?.id}', + 'firestore_commit_document_name': commit.sha, + 'firestore_task_document_name': taskDocumentName, + }; + + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.FAILURE, + builder: 'Linux A', + userData: userDataMap, + ); + + expect(firestoreTask!.status, firestore.Task.statusFailed); + expect(firestoreTask!.attempts, 1); + expect(await tester.post(handler), Body.empty); + final List captured = verify(mockFirestoreService.batchWriteDocuments(captureAny, captureAny)).captured; + expect(captured.length, 2); + final BatchWriteRequest batchWriteRequest = captured[0] as BatchWriteRequest; + expect(batchWriteRequest.writes!.length, 1); + final Document insertedTaskDocument = batchWriteRequest.writes![0].update!; + final firestore.Task resultTask = firestore.Task.fromDocument(taskDocument: insertedTaskDocument); + expect(resultTask.status, firestore.Task.statusInProgress); + expect(resultTask.attempts, 2); + }); + + test('on canceled builds auto-rerun the build if they timed out', () async { + firestoreTask = generateFirestoreTask( + 1, + name: 'Linux A', + status: firestore.Task.statusInfraFailure, + commitSha: '87f88734747805589f2131753620d61b22922822', + ); + firestoreCommit = generateFirestoreCommit(1, sha: '87f88734747805589f2131753620d61b22922822'); + final Commit commit = generateCommit(1, sha: '87f88734747805589f2131753620d61b22922822'); + final Task task = generateTask( + 4507531199512576, + name: 'Linux A', + parent: commit, + status: Task.statusInfraFailure, + ); + config.db.values[task.key] = task; + config.db.values[commit.key] = commit; + final String taskDocumentName = '${commit.sha}_${task.name}_${task.attempts}'; + + final Map userDataMap = { + 'task_key': '${task.key.id}', + 'commit_key': '${task.key.parent?.id}', + 'firestore_commit_document_name': commit.sha, + 'firestore_task_document_name': taskDocumentName, + }; + + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.CANCELED, + builder: 'Linux A', + userData: userDataMap, + ); + + expect(firestoreTask!.status, firestore.Task.statusInfraFailure); + expect(firestoreTask!.attempts, 1); + expect(await tester.post(handler), Body.empty); + final List captured = verify(mockFirestoreService.batchWriteDocuments(captureAny, captureAny)).captured; + expect(captured.length, 2); + final BatchWriteRequest batchWriteRequest = captured[0] as BatchWriteRequest; + expect(batchWriteRequest.writes!.length, 1); + final Document insertedTaskDocument = batchWriteRequest.writes![0].update!; + final firestore.Task resultTask = firestore.Task.fromDocument(taskDocument: insertedTaskDocument); + expect(resultTask.status, firestore.Task.statusInProgress); + expect(resultTask.attempts, 2); + }); + + test('on builds resulting in an infra failure auto-rerun the build if they timed out', () async { + firestoreTask = generateFirestoreTask( + 1, + name: 'Linux A', + status: firestore.Task.statusInfraFailure, + commitSha: '87f88734747805589f2131753620d61b22922822', + ); + firestoreCommit = generateFirestoreCommit(1, sha: '87f88734747805589f2131753620d61b22922822'); + final Commit commit = generateCommit(1, sha: '87f88734747805589f2131753620d61b22922822'); + final Task task = generateTask( + 4507531199512576, + name: 'Linux A', + parent: commit, + status: Task.statusInfraFailure, + ); + config.db.values[task.key] = task; + config.db.values[commit.key] = commit; + final String taskDocumentName = '${commit.sha}_${task.name}_${task.attempts}'; + + final Map userDataMap = { + 'task_key': '${task.key.id}', + 'commit_key': '${task.key.parent?.id}', + 'firestore_commit_document_name': commit.sha, + 'firestore_task_document_name': taskDocumentName, + }; + + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.INFRA_FAILURE, + builder: 'Linux A', + userData: userDataMap, + ); + + expect(task.status, Task.statusInfraFailure); + expect(task.attempts, 1); + expect(await tester.post(handler), Body.empty); + final List captured = verify(mockFirestoreService.batchWriteDocuments(captureAny, captureAny)).captured; + expect(captured.length, 2); + final BatchWriteRequest batchWriteRequest = captured[0] as BatchWriteRequest; + expect(batchWriteRequest.writes!.length, 1); + final Document insertedTaskDocument = batchWriteRequest.writes![0].update!; + final firestore.Task resultTask = firestore.Task.fromDocument(taskDocument: insertedTaskDocument); + expect(resultTask.status, firestore.Task.statusInProgress); + expect(resultTask.attempts, 2); + }); + + test('non-bringup target updates check run', () async { + firestoreTask = generateFirestoreTask(1, name: 'Linux nonbringup'); + scheduler.ciYaml = nonBringupPackagesConfig; + when( + mockGithubChecksService.updateCheckStatus( + build: anyNamed('build'), + userDataMap: anyNamed('userDataMap'), + luciBuildService: anyNamed('luciBuildService'), + slug: anyNamed('slug'), + ), + ).thenAnswer((_) async => true); + when(mockGithubChecksService.currentAttempt(any)).thenAnswer((_) => 2); + final Commit commit = generateCommit(1, sha: '87f88734747805589f2131753620d61b22922822', repo: 'packages'); + final Task task = generateTask( + 4507531199512576, + name: 'Linux nonbringup', + parent: commit, + ); + config.db.values[commit.key] = commit; + config.db.values[task.key] = task; + final String taskDocumentName = '${commit.sha}_${task.name}_${task.attempts}'; + + final Map userDataMap = { + 'task_key': '${task.key.id}', + 'commit_key': '${task.key.parent?.id}', + 'firestore_commit_document_name': commit.sha, + 'firestore_task_document_name': taskDocumentName, + }; + + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.SUCCESS, + builder: 'Linux A', + userData: userDataMap, + ); + + await tester.post(handler); + verify( + mockGithubChecksService.updateCheckStatus( + build: anyNamed('build'), + userDataMap: anyNamed('userDataMap'), + luciBuildService: anyNamed('luciBuildService'), + slug: anyNamed('slug'), + ), + ).called(1); + }); + + test('bringup target does not update check run', () async { + firestoreTask = generateFirestoreTask(1, name: 'Linux bringup'); + scheduler.ciYaml = bringupPackagesConfig; + when( + mockGithubChecksService.updateCheckStatus( + build: anyNamed('build'), + userDataMap: anyNamed('userDataMap'), + luciBuildService: anyNamed('luciBuildService'), + slug: anyNamed('slug'), + ), + ).thenAnswer((_) async => true); + when(mockGithubChecksService.currentAttempt(any)).thenAnswer((_) => 2); + final Commit commit = generateCommit(1, sha: '87f88734747805589f2131753620d61b22922822'); + final Task task = generateTask( + 4507531199512576, + name: 'Linux bringup', + parent: commit, + ); + config.db.values[commit.key] = commit; + config.db.values[task.key] = task; + final String taskDocumentName = '${commit.sha}_${task.name}_${task.attempts}'; + + final Map userDataMap = { + 'task_key': '${task.key.id}', + 'commit_key': '${task.key.parent?.id}', + 'firestore_commit_document_name': commit.sha, + 'firestore_task_document_name': taskDocumentName, + }; + + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.SUCCESS, + builder: 'Linux bringup', + userData: userDataMap, + ); + + await tester.post(handler); + verifyNever( + mockGithubChecksService.updateCheckStatus( + build: anyNamed('build'), + userDataMap: anyNamed('userDataMap'), + luciBuildService: anyNamed('luciBuildService'), + slug: anyNamed('slug'), + ), + ); + }); + + test('unsupported repo target does not update check run', () async { + scheduler.ciYaml = unsupportedPostsubmitCheckrunConfig; + when( + mockGithubChecksService.updateCheckStatus( + build: anyNamed('build'), + userDataMap: anyNamed('userDataMap'), + luciBuildService: anyNamed('luciBuildService'), + slug: anyNamed('slug'), + ), + ).thenAnswer((_) async => true); + when(mockGithubChecksService.currentAttempt(any)).thenAnswer((_) => 2); + firestoreTask = generateFirestoreTask(1, attempts: 2, name: 'Linux flutter'); + + final Commit commit = generateCommit(1, sha: '87f88734747805589f2131753620d61b22922822'); + final Task task = generateTask( + 4507531199512576, + name: 'Linux flutter', + parent: commit, + ); + config.db.values[commit.key] = commit; + config.db.values[task.key] = task; + final String taskDocumentName = '${commit.sha}_${task.name}_${task.attempts}'; + + final Map userDataMap = { + 'task_key': '${task.key.id}', + 'commit_key': '${task.key.parent?.id}', + 'firestore_commit_document_name': commit.sha, + 'firestore_task_document_name': taskDocumentName, + }; + + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.SUCCESS, + builder: 'Linux bringup', + userData: userDataMap, + ); + + await tester.post(handler); + verifyNever( + mockGithubChecksService.updateCheckStatus( + build: anyNamed('build'), + userDataMap: anyNamed('userDataMap'), + luciBuildService: anyNamed('luciBuildService'), + slug: anyNamed('slug'), + ), + ); + }); +} diff --git a/app_dart/test/request_handlers/presubmit_luci_subscription_test.dart b/app_dart/test/request_handlers/presubmit_luci_subscription_test.dart index 87e7cb735..013d88b6d 100644 --- a/app_dart/test/request_handlers/presubmit_luci_subscription_test.dart +++ b/app_dart/test/request_handlers/presubmit_luci_subscription_test.dart @@ -12,7 +12,6 @@ import '../src/datastore/fake_config.dart'; import '../src/request_handling/fake_authentication.dart'; import '../src/request_handling/fake_http.dart'; import '../src/request_handling/subscription_tester.dart'; -import '../src/service/fake_buildbucket.dart'; import '../src/service/fake_luci_build_service.dart'; import '../src/service/fake_scheduler.dart'; import '../src/utilities/mocks.dart'; @@ -22,7 +21,6 @@ const String ref = 'deadbeef'; void main() { late PresubmitLuciSubscription handler; - late FakeBuildBucketClient buildbucket; late FakeConfig config; late MockGitHub mockGitHubClient; late FakeHttpRequest request; @@ -34,7 +32,6 @@ void main() { setUp(() async { config = FakeConfig(); - buildbucket = FakeBuildBucketClient(); mockLuciBuildService = MockLuciBuildService(); mockGithubChecksService = MockGithubChecksService(); @@ -46,7 +43,6 @@ void main() { handler = PresubmitLuciSubscription( cache: CacheService(inMemory: true), config: config, - buildBucketClient: buildbucket, luciBuildService: FakeLuciBuildService(config: config), githubChecksService: mockGithubChecksService, authProvider: FakeAuthenticationProvider(), diff --git a/app_dart/test/request_handlers/presubmit_luci_subscription_v2_test.dart b/app_dart/test/request_handlers/presubmit_luci_subscription_v2_test.dart new file mode 100644 index 000000000..4c2144637 --- /dev/null +++ b/app_dart/test/request_handlers/presubmit_luci_subscription_v2_test.dart @@ -0,0 +1,303 @@ +// Copyright 2019 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:buildbucket/buildbucket_pb.dart' as bbv2; +import 'package:cocoon_service/cocoon_service.dart'; +import 'package:cocoon_service/src/request_handlers/presubmit_luci_subscription_v2.dart'; +import 'package:fixnum/fixnum.dart'; +import 'package:mockito/mockito.dart'; +import 'package:test/test.dart'; + +import '../src/datastore/fake_config.dart'; +import '../src/request_handling/fake_authentication.dart'; +import '../src/request_handling/fake_http.dart'; +import '../src/request_handling/subscription_v2_tester.dart'; +import '../src/service/fake_luci_build_service_v2.dart'; +import '../src/service/fake_scheduler_v2.dart'; +import '../src/utilities/build_bucket_v2_messages.dart'; +import '../src/utilities/mocks.dart'; + +const String ref = 'deadbeef'; + +void main() { + late PresubmitLuciSubscriptionV2 handler; + late FakeConfig config; + late MockGitHub mockGitHubClient; + late FakeHttpRequest request; + late SubscriptionV2Tester tester; + late MockRepositoriesService mockRepositoriesService; + late MockGithubChecksServiceV2 mockGithubChecksService; + late MockLuciBuildServiceV2 mockLuciBuildService; + late FakeSchedulerV2 scheduler; + + setUp(() async { + config = FakeConfig(); + mockLuciBuildService = MockLuciBuildServiceV2(); + + mockGithubChecksService = MockGithubChecksServiceV2(); + scheduler = FakeSchedulerV2( + ciYaml: examplePresubmitRescheduleConfig, + config: config, + luciBuildService: mockLuciBuildService, + ); + + handler = PresubmitLuciSubscriptionV2( + cache: CacheService(inMemory: true), + config: config, + luciBuildService: FakeLuciBuildServiceV2(config: config), + githubChecksService: mockGithubChecksService, + authProvider: FakeAuthenticationProvider(), + scheduler: scheduler, + ); + request = FakeHttpRequest(); + + tester = SubscriptionV2Tester( + request: request, + ); + + mockGitHubClient = MockGitHub(); + mockRepositoriesService = MockRepositoriesService(); + when(mockGitHubClient.repositories).thenReturn(mockRepositoriesService); + config.githubClient = mockGitHubClient; + }); + + test('Requests without repo_owner and repo_name do not update checks', () async { + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.SUCCESS, + builder: 'Linux Host Engine', + addBuildSet: false, + ); + + await tester.post(handler); + + verifyNever( + mockGithubChecksService.updateCheckStatus( + build: anyNamed('build'), + userDataMap: anyNamed('userDataMap'), + luciBuildService: anyNamed('luciBuildService'), + slug: anyNamed('slug'), + ), + ); + }); + + test('Requests with repo_owner and repo_name update checks', () async { + when( + mockGithubChecksService.updateCheckStatus( + build: anyNamed('build'), + userDataMap: anyNamed('userDataMap'), + luciBuildService: anyNamed('luciBuildService'), + slug: anyNamed('slug'), + ), + ).thenAnswer((_) async => true); + + when(mockGithubChecksService.taskFailed(any)).thenAnswer((_) => false); + + const Map userDataMap = { + 'repo_owner': 'flutter', + 'repo_name': 'cocoon', + }; + + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.SUCCESS, + builder: 'Linux Host Engine', + userData: userDataMap, + ); + + await tester.post(handler); + verify( + mockGithubChecksService.updateCheckStatus( + build: anyNamed('build'), + userDataMap: anyNamed('userDataMap'), + luciBuildService: anyNamed('luciBuildService'), + slug: anyNamed('slug'), + ), + ).called(1); + }); + + test('Requests when task failed but no need to reschedule', () async { + when( + mockGithubChecksService.updateCheckStatus( + build: anyNamed('build'), + userDataMap: anyNamed('userDataMap'), + luciBuildService: anyNamed('luciBuildService'), + slug: anyNamed('slug'), + ), + ).thenAnswer((_) async => true); + when(mockGithubChecksService.taskFailed(any)).thenAnswer((_) => true); + when(mockGithubChecksService.currentAttempt(any)).thenAnswer((_) => 1); + + const Map userDataMap = { + 'repo_owner': 'flutter', + 'commit_branch': 'main', + 'commit_sha': 'abc', + 'repo_name': 'flutter', + }; + + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.SUCCESS, + builder: 'Linux A', + userData: userDataMap, + ); + + final bbv2.BuildsV2PubSub buildsV2PubSub = createBuild( + Int64(1), + status: bbv2.Status.SUCCESS, + builder: 'Linux A', + ); + + when( + mockLuciBuildService.rescheduleBuild( + build: buildsV2PubSub.build, + builderName: 'Linux Coverage', + rescheduleAttempt: 0, + userDataMap: userDataMap, + ), + ).thenAnswer( + (_) async => bbv2.Build( + id: Int64(8905920700440101120), + builder: bbv2.BuilderID(bucket: 'luci.flutter.prod', project: 'flutter', builder: 'Linux Coverage'), + ), + ); + + await tester.post(handler); + verifyNever( + mockLuciBuildService.rescheduleBuild( + build: buildsV2PubSub.build, + builderName: 'Linux Coverage', + rescheduleAttempt: 0, + userDataMap: userDataMap, + ), + ); + verify( + mockGithubChecksService.updateCheckStatus( + build: anyNamed('build'), + userDataMap: anyNamed('userDataMap'), + luciBuildService: anyNamed('luciBuildService'), + slug: anyNamed('slug'), + ), + ).called(1); + }); + + test('Requests when task failed but need to reschedule', () async { + when( + mockGithubChecksService.updateCheckStatus( + build: anyNamed('build'), + userDataMap: anyNamed('userDataMap'), + luciBuildService: anyNamed('luciBuildService'), + slug: anyNamed('slug'), + rescheduled: true, + ), + ).thenAnswer((_) async => true); + when(mockGithubChecksService.taskFailed(any)).thenAnswer((_) => true); + when(mockGithubChecksService.currentAttempt(any)).thenAnswer((_) => 0); + + const Map userDataMap = { + 'repo_owner': 'flutter', + 'commit_branch': 'main', + 'commit_sha': 'abc', + 'repo_name': 'flutter', + }; + + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.SUCCESS, + builder: 'Linux B', + userData: userDataMap, + ); + + final bbv2.BuildsV2PubSub buildsV2PubSub = createBuild( + Int64(1), + status: bbv2.Status.SUCCESS, + builder: 'Linux A', + ); + + when( + mockLuciBuildService.rescheduleBuild( + build: buildsV2PubSub.build, + builderName: 'Linux Coverage', + rescheduleAttempt: 1, + userDataMap: userDataMap, + ), + ).thenAnswer( + (_) async => bbv2.Build( + id: Int64(8905920700440101120), + builder: bbv2.BuilderID(bucket: 'luci.flutter.prod', project: 'flutter', builder: 'Linux B'), + ), + ); + await tester.post(handler); + verifyNever( + mockLuciBuildService.rescheduleBuild( + build: buildsV2PubSub.build, + builderName: 'Linux Coverage', + rescheduleAttempt: 1, + userDataMap: userDataMap, + ), + ); + verify( + mockGithubChecksService.updateCheckStatus( + build: anyNamed('build'), + userDataMap: anyNamed('userDataMap'), + luciBuildService: anyNamed('luciBuildService'), + slug: anyNamed('slug'), + rescheduled: true, + ), + ).called(1); + }); + + test('Build not rescheduled if not found in ciYaml list.', () async { + when( + mockGithubChecksService.updateCheckStatus( + build: anyNamed('build'), + userDataMap: anyNamed('userDataMap'), + luciBuildService: anyNamed('luciBuildService'), + slug: anyNamed('slug'), + rescheduled: false, + ), + ).thenAnswer((_) async => true); + when(mockGithubChecksService.taskFailed(any)).thenAnswer((_) => true); + when(mockGithubChecksService.currentAttempt(any)).thenAnswer((_) => 1); + + const Map userDataMap = { + 'repo_owner': 'flutter', + 'commit_branch': 'main', + 'commit_sha': 'abc', + 'repo_name': 'flutter', + }; + + tester.message = createPushMessageV2( + Int64(1), + status: bbv2.Status.SUCCESS, + builder: 'Linux C', + userData: userDataMap, + ); + + final bbv2.BuildsV2PubSub buildsV2PubSub = createBuild( + Int64(1), + status: bbv2.Status.SUCCESS, + builder: 'Linux C', + ); + + await tester.post(handler); + verifyNever( + mockLuciBuildService.rescheduleBuild( + build: buildsV2PubSub.build, + builderName: 'Linux C', + userDataMap: userDataMap, + rescheduleAttempt: 1, + ), + ); + verify( + mockGithubChecksService.updateCheckStatus( + build: anyNamed('build'), + userDataMap: anyNamed('userDataMap'), + luciBuildService: anyNamed('luciBuildService'), + slug: anyNamed('slug'), + rescheduled: false, + ), + ).called(1); + }); +} diff --git a/app_dart/test/request_handlers/reset_prod_task_v2_test.dart b/app_dart/test/request_handlers/reset_prod_task_v2_test.dart new file mode 100644 index 000000000..9d34494e9 --- /dev/null +++ b/app_dart/test/request_handlers/reset_prod_task_v2_test.dart @@ -0,0 +1,230 @@ +// Copyright 2020 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:cocoon_service/cocoon_service.dart'; +import 'package:cocoon_service/src/model/appengine/commit.dart'; +import 'package:cocoon_service/src/model/appengine/task.dart'; +import 'package:cocoon_service/src/model/firestore/task.dart' as firestore; +import 'package:cocoon_service/src/request_handlers/reset_prod_task_v2.dart'; +import 'package:cocoon_service/src/request_handling/exceptions.dart'; +import 'package:googleapis/firestore/v1.dart'; +import 'package:mockito/mockito.dart'; +import 'package:test/test.dart'; + +import '../src/datastore/fake_config.dart'; +import '../src/datastore/fake_datastore.dart'; +import '../src/request_handling/api_request_handler_tester.dart'; +import '../src/request_handling/fake_authentication.dart'; +import '../src/service/fake_scheduler_v2.dart'; +import '../src/utilities/entity_generators.dart'; +import '../src/utilities/mocks.dart'; + +void main() { + group('ResetProdTask', () { + FakeClientContext clientContext; + late ResetProdTaskV2 handler; + late FakeConfig config; + FakeKeyHelper keyHelper; + late MockLuciBuildServiceV2 mockLuciBuildService; + late MockFirestoreService mockFirestoreService; + late ApiRequestHandlerTester tester; + late Commit commit; + late Task task; + final firestore.Task firestoreTask = generateFirestoreTask(1, attempts: 1); + + setUp(() { + final FakeDatastoreDB datastoreDB = FakeDatastoreDB(); + clientContext = FakeClientContext(); + mockFirestoreService = MockFirestoreService(); + clientContext.isDevelopmentEnvironment = false; + keyHelper = FakeKeyHelper(applicationContext: clientContext.applicationContext); + config = FakeConfig( + dbValue: datastoreDB, + keyHelperValue: keyHelper, + supportedBranchesValue: [ + Config.defaultBranch(Config.flutterSlug), + ], + firestoreService: mockFirestoreService, + ); + final FakeAuthenticatedContext authContext = FakeAuthenticatedContext(clientContext: clientContext); + tester = ApiRequestHandlerTester(context: authContext); + mockLuciBuildService = MockLuciBuildServiceV2(); + handler = ResetProdTaskV2( + config: config, + authenticationProvider: FakeAuthenticationProvider(clientContext: clientContext), + luciBuildService: mockLuciBuildService, + scheduler: FakeSchedulerV2( + config: config, + ciYaml: exampleConfig, + ), + ); + commit = generateCommit(1); + task = generateTask( + 1, + name: 'Linux A', + parent: commit, + status: Task.statusFailed, + ); + tester.requestData = { + 'Key': config.keyHelper.encode(task.key), + }; + + when( + mockFirestoreService.getDocument( + captureAny, + ), + ).thenAnswer((Invocation invocation) { + return Future.value( + firestoreTask, + ); + }); + + when( + mockLuciBuildService.checkRerunBuilder( + commit: anyNamed('commit'), + datastore: anyNamed('datastore'), + task: anyNamed('task'), + target: anyNamed('target'), + tags: anyNamed('tags'), + ignoreChecks: anyNamed('ignoreChecks'), + firestoreService: mockFirestoreService, + taskDocument: anyNamed('taskDocument'), + ), + ).thenAnswer((_) async => true); + }); + test('Schedule new task', () async { + config.db.values[task.key] = task; + config.db.values[commit.key] = commit; + expect(await tester.post(handler), Body.empty); + + final List captured = verify(mockFirestoreService.getDocument(captureAny)).captured; + expect(captured.length, 1); + final String documentName = captured[0] as String; + expect( + documentName, + '$kDatabase/documents/${firestore.kTaskCollectionId}/${commit.sha}_${task.name}_${task.attempts}', + ); + }); + + test('schedule new task when task document is aviable', () async { + config.db.values[task.key] = task; + config.db.values[commit.key] = commit; + tester.requestData = { + 'taskDocumentName': + '$kDatabase/documents/${firestore.kTaskCollectionId}/${commit.sha}_${task.name}_${task.attempts}}', + 'Commit': commit.sha, + 'Task': task.name, + 'Repo': commit.slug.name, + }; + expect(await tester.post(handler), Body.empty); + }); + + test('Re-schedule passing all the parameters', () async { + config.db.values[task.key] = task; + config.db.values[commit.key] = commit; + tester.requestData = { + 'Commit': commit.sha, + 'Task': task.name, + 'Repo': commit.slug.name, + }; + expect(await tester.post(handler), Body.empty); + verify( + mockLuciBuildService.checkRerunBuilder( + commit: anyNamed('commit'), + datastore: anyNamed('datastore'), + task: anyNamed('task'), + target: anyNamed('target'), + tags: anyNamed('tags'), + ignoreChecks: true, + firestoreService: mockFirestoreService, + taskDocument: anyNamed('taskDocument'), + ), + ).called(1); + }); + + test('Rerun all failed tasks when task name is all', () async { + final Task taskA = generateTask(2, name: 'Linux A', parent: commit, status: Task.statusFailed); + final Task taskB = generateTask(3, name: 'Mac A', parent: commit, status: Task.statusFailed); + config.db.values[taskA.key] = taskA; + config.db.values[taskB.key] = taskB; + config.db.values[commit.key] = commit; + tester.requestData = { + 'Commit': commit.sha, + 'Task': 'all', + 'Repo': commit.slug.name, + }; + expect(await tester.post(handler), Body.empty); + verify( + mockLuciBuildService.checkRerunBuilder( + commit: anyNamed('commit'), + datastore: anyNamed('datastore'), + task: anyNamed('task'), + target: anyNamed('target'), + tags: anyNamed('tags'), + ignoreChecks: false, + firestoreService: mockFirestoreService, + taskDocument: anyNamed('taskDocument'), + ), + ).called(2); + }); + + test('Rerun all runs nothing when everything is passed', () async { + final Task task = generateTask(2, name: 'Windows A', parent: commit, status: Task.statusSucceeded); + config.db.values[task.key] = task; + config.db.values[commit.key] = commit; + tester.requestData = { + 'Commit': commit.sha, + 'Task': 'all', + 'Repo': commit.slug.name, + }; + expect(await tester.post(handler), Body.empty); + verifyNever( + mockLuciBuildService.checkRerunBuilder( + commit: anyNamed('commit'), + datastore: anyNamed('datastore'), + task: anyNamed('task'), + target: anyNamed('target'), + tags: anyNamed('tags'), + firestoreService: anyNamed('firestoreService'), + taskDocument: anyNamed('taskDocument'), + ignoreChecks: false, + ), + ); + }); + + test('Re-schedule without any parameters raises exception', () async { + tester.requestData = {}; + expect(() => tester.post(handler), throwsA(isA())); + }); + + test('Re-schedule existing task even though taskName is missing in the task', () async { + config.db.values[task.key] = task; + config.db.values[commit.key] = commit; + expect(await tester.post(handler), Body.empty); + }); + + test('Fails if task is not rerun', () async { + when( + mockLuciBuildService.checkRerunBuilder( + commit: anyNamed('commit'), + datastore: anyNamed('datastore'), + task: anyNamed('task'), + target: anyNamed('target'), + tags: anyNamed('tags'), + ignoreChecks: true, + firestoreService: mockFirestoreService, + taskDocument: anyNamed('taskDocument'), + ), + ).thenAnswer((_) async => false); + config.db.values[task.key] = task; + config.db.values[commit.key] = commit; + expect(() => tester.post(handler), throwsA(isA())); + }); + + test('Fails if commit does not exist', () async { + config.db.values[task.key] = task; + expect(() => tester.post(handler), throwsA(isA())); + }); + }); +} diff --git a/app_dart/test/request_handlers/reset_try_task_v2_test.dart b/app_dart/test/request_handlers/reset_try_task_v2_test.dart new file mode 100644 index 000000000..f383f9de6 --- /dev/null +++ b/app_dart/test/request_handlers/reset_try_task_v2_test.dart @@ -0,0 +1,99 @@ +// Copyright 2020 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:cocoon_service/src/request_handlers/reset_try_task_v2.dart'; +import 'package:cocoon_service/src/request_handling/body.dart'; +import 'package:cocoon_service/src/request_handling/exceptions.dart'; +import 'package:github/github.dart'; +import 'package:mockito/mockito.dart'; +import 'package:test/test.dart'; + +import '../src/datastore/fake_config.dart'; +import '../src/request_handling/api_request_handler_tester.dart'; +import '../src/request_handling/fake_authentication.dart'; +import '../src/request_handling/fake_http.dart'; +import '../src/service/fake_github_service.dart'; +import '../src/service/fake_scheduler_v2.dart'; +import '../src/utilities/entity_generators.dart'; +import '../src/utilities/mocks.dart'; + +void main() { + group('ResetTryTask', () { + late ApiRequestHandlerTester tester; + FakeClientContext clientContext; + late ResetTryTaskV2 handler; + late FakeConfig config; + FakeSchedulerV2 fakeScheduler; + FakeAuthenticatedContext authContext; + MockGitHub mockGithub; + MockPullRequestsService mockPullRequestsService; + late MockGithubChecksUtil mockGithubChecksUtil; + + setUp(() { + clientContext = FakeClientContext(); + clientContext.isDevelopmentEnvironment = false; + authContext = FakeAuthenticatedContext(clientContext: clientContext); + mockGithub = MockGitHub(); + mockPullRequestsService = MockPullRequestsService(); + config = FakeConfig(githubClient: mockGithub, githubService: FakeGithubService()); + mockGithubChecksUtil = MockGithubChecksUtil(); + tester = ApiRequestHandlerTester(context: authContext); + fakeScheduler = FakeSchedulerV2( + config: config, + githubChecksUtil: mockGithubChecksUtil, + ); + handler = ResetTryTaskV2( + config: config, + authenticationProvider: FakeAuthenticationProvider(clientContext: clientContext), + scheduler: fakeScheduler, + ); + when(mockGithub.pullRequests).thenReturn(mockPullRequestsService); + when(mockPullRequestsService.get(any, 123)).thenAnswer((_) async => generatePullRequest(id: 123)); + }); + + test('Empty repo', () async { + tester.request = FakeHttpRequest( + queryParametersValue: { + 'pr': '123', + }, + ); + expect(() => tester.get(handler), throwsA(isA())); + }); + + test('Empty pr', () async { + tester.request = FakeHttpRequest( + queryParametersValue: { + 'repo': 'flutter', + }, + ); + expect(() => tester.get(handler), throwsA(isA())); + }); + + test('Trigger builds if all parameters are correct', () async { + when(mockGithubChecksUtil.createCheckRun(any, any, any, any, output: anyNamed('output'))).thenAnswer((_) async { + return CheckRun.fromJson(const { + 'id': 1, + 'started_at': '2020-05-10T02:49:31Z', + 'check_suite': {'id': 2}, + }); + }); + tester.request = FakeHttpRequest( + queryParametersValue: { + ResetTryTaskV2.kRepoParam: 'flutter', + ResetTryTaskV2.kPullRequestNumberParam: '123', + }, + ); + expect(await tester.get(handler), Body.empty); + }); + + test('Parses empty builder correctly', () { + final List builders = handler.getBuilderList(''); + expect(builders.isEmpty, true); + }); + + test('Parses non-empty builder correctly', () { + expect(handler.getBuilderList('a, b, c'), ['a', 'b', 'c']); + }); + }); +} diff --git a/app_dart/test/request_handlers/scheduler/batch_backfiller_v2_test.dart b/app_dart/test/request_handlers/scheduler/batch_backfiller_v2_test.dart new file mode 100644 index 000000000..f7d75f985 --- /dev/null +++ b/app_dart/test/request_handlers/scheduler/batch_backfiller_v2_test.dart @@ -0,0 +1,327 @@ +// Copyright 2019 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:buildbucket/buildbucket_pb.dart' as bbv2; +import 'package:cocoon_service/cocoon_service.dart'; +import 'package:cocoon_service/src/model/appengine/commit.dart'; +import 'package:cocoon_service/src/model/appengine/task.dart'; +import 'package:cocoon_service/src/model/firestore/task.dart' as firestore; +import 'package:cocoon_service/src/model/ci_yaml/target.dart'; +import 'package:cocoon_service/src/request_handlers/scheduler/batch_backfiller_v2.dart'; +import 'package:googleapis/firestore/v1.dart'; +import 'package:mockito/mockito.dart'; +import 'package:test/test.dart'; + +import '../../src/datastore/fake_config.dart'; +import '../../src/datastore/fake_datastore.dart'; +import '../../src/request_handling/fake_pubsub.dart'; +import '../../src/request_handling/request_handler_tester.dart'; +import '../../src/service/fake_luci_build_service_v2.dart'; +import '../../src/service/fake_scheduler_v2.dart'; +import '../../src/utilities/entity_generators.dart'; +import '../../src/utilities/mocks.dart'; + +final List commits = [ + generateCommit(3), + generateCommit(2), + generateCommit(1), +]; + +void main() { + late BatchBackfillerV2 handler; + late RequestHandlerTester tester; + late FakeDatastoreDB db; + late FakePubSub pubsub; + late FakeSchedulerV2 scheduler; + late MockGithubChecksUtil mockGithubChecksUtil; + late Config config; + late MockFirestoreService mockFirestoreService; + + group('BatchBackfiller', () { + setUp(() async { + mockFirestoreService = MockFirestoreService(); + + db = FakeDatastoreDB()..addOnQuery((Iterable results) => commits); + + config = FakeConfig( + dbValue: db, + backfillerTargetLimitValue: 2, + firestoreService: mockFirestoreService, + ); + + pubsub = FakePubSub(); + + mockGithubChecksUtil = MockGithubChecksUtil(); + + when( + mockGithubChecksUtil.createCheckRun( + any, + any, + any, + any, + output: anyNamed('output'), + ), + ).thenAnswer((_) async => generateCheckRun(1)); + + when( + mockFirestoreService.writeViaTransaction( + captureAny, + ), + ).thenAnswer((Invocation invocation) { + return Future.value(CommitResponse()); + }); + + scheduler = FakeSchedulerV2( + config: config, + ciYaml: batchPolicyConfig, + githubChecksUtil: mockGithubChecksUtil, + luciBuildService: FakeLuciBuildServiceV2( + config: config, + pubsub: pubsub, + githubChecksUtil: mockGithubChecksUtil, + ), + ); + + handler = BatchBackfillerV2( + config: config, + scheduler: scheduler, + ); + + tester = RequestHandlerTester(); + }); + + test('does not backfill on completed task column', () async { + final List allGreen = [ + generateTask(1, name: 'Linux_android A', status: Task.statusSucceeded), + generateTask(2, name: 'Linux_android A', status: Task.statusSucceeded), + generateTask(3, name: 'Linux_android A', status: Task.statusSucceeded), + ]; + db.addOnQuery((Iterable results) => allGreen); + await tester.get(handler); + expect(pubsub.messages, isEmpty); + }); + + test('does not backfill when there is a running task', () async { + final List middleTaskInProgress = [ + generateTask(1, name: 'Linux_android A', status: Task.statusNew), + generateTask(2, name: 'Linux_android A', status: Task.statusInProgress), + generateTask(3, name: 'Linux_android A', status: Task.statusNew), + ]; + db.addOnQuery((Iterable results) => middleTaskInProgress); + await tester.get(handler); + expect(pubsub.messages, isEmpty); + }); + + test('does not backfill when task does not exist in TOT', () async { + scheduler = FakeSchedulerV2( + config: config, + ciYaml: notInToTConfig, + githubChecksUtil: mockGithubChecksUtil, + luciBuildService: FakeLuciBuildServiceV2( + config: config, + pubsub: pubsub, + githubChecksUtil: mockGithubChecksUtil, + ), + ); + handler = BatchBackfillerV2( + config: config, + scheduler: scheduler, + ); + final List allGray = [ + generateTask(1, name: 'Linux_android B', status: Task.statusNew), + ]; + db.addOnQuery((Iterable results) => allGray); + await tester.get(handler); + expect(pubsub.messages.length, 0); + }); + + test('backfills latest task', () async { + final List allGray = [ + generateTask(1, name: 'Linux_android A', status: Task.statusNew), + generateTask(2, name: 'Linux_android A', status: Task.statusNew), + generateTask(3, name: 'Linux_android A', status: Task.statusNew), + ]; + db.addOnQuery((Iterable results) => allGray); + await tester.get(handler); + expect(pubsub.messages.length, 1); + + final bbv2.BatchRequest batchRequest = bbv2.BatchRequest.create(); + batchRequest.mergeFromProto3Json(pubsub.messages.first); + + final bbv2.ScheduleBuildRequest scheduleBuildRequest = batchRequest.requests.first.scheduleBuild; + + expect(scheduleBuildRequest.priority, LuciBuildService.kBackfillPriority); + }); + + test('does not backfill targets when number of available tasks is less than BatchPolicy.kBatchSize', () async { + final List scheduleA = [ + generateTask(1, name: 'Linux_android A', status: Task.statusNew), + ]; + db.addOnQuery((Iterable results) => scheduleA); + await tester.get(handler); + expect(pubsub.messages.length, 0); + }); + + test('backfills earlier failed task with higher priority', () async { + final List allGray = [ + generateTask(1, name: 'Linux_android A', status: Task.statusNew), + generateTask(2, name: 'Linux_android A', status: Task.statusNew), + generateTask(3, name: 'Linux_android A', status: Task.statusFailed), + ]; + db.addOnQuery((Iterable results) => allGray); + await tester.get(handler); + expect(pubsub.messages.length, 1); + + final bbv2.BatchRequest batchRequest = bbv2.BatchRequest.create(); + batchRequest.mergeFromProto3Json(pubsub.messages.first); + + final bbv2.ScheduleBuildRequest scheduleBuildRequest = batchRequest.requests.first.scheduleBuild; + + expect(scheduleBuildRequest.priority, LuciBuildService.kRerunPriority); + }); + + test('backfills task successfully with retry', () async { + pubsub.exceptionFlag = true; + pubsub.exceptionRepetition = 1; + final List allGray = [ + generateTask(1, name: 'Linux_android A', status: Task.statusNew), + generateTask(2, name: 'Linux_android A', status: Task.statusNew), + generateTask(3, name: 'Linux_android A', status: Task.statusFailed), + ]; + db.addOnQuery((Iterable results) => allGray); + await tester.get(handler); + expect(pubsub.messages.length, 1); + + final bbv2.BatchRequest batchRequest = bbv2.BatchRequest.create(); + batchRequest.mergeFromProto3Json(pubsub.messages.first); + + final bbv2.ScheduleBuildRequest scheduleBuildRequest = batchRequest.requests.first.scheduleBuild; + + expect(scheduleBuildRequest.priority, LuciBuildService.kRerunPriority); + }); + + test('fails to backfill tasks when retry limit is hit', () async { + pubsub.exceptionFlag = true; + pubsub.exceptionRepetition = 3; + final List allGray = [ + generateTask(1, name: 'Linux_android A', status: Task.statusNew), + generateTask(2, name: 'Linux_android A', status: Task.statusNew), + generateTask(3, name: 'Linux_android A', status: Task.statusFailed), + ]; + db.addOnQuery((Iterable results) => allGray); + await tester.get(handler); + expect(pubsub.messages.length, 0); + }); + + test('backfills older task', () async { + final List oldestGray = [ + generateTask(1, name: 'Linux_android A', status: Task.statusSucceeded), + generateTask(2, name: 'Linux_android A', status: Task.statusSucceeded), + generateTask(3, name: 'Linux_android A', status: Task.statusNew), + ]; + db.addOnQuery((Iterable results) => oldestGray); + await tester.get(handler); + expect(pubsub.messages.length, 1); + }); + + test('updates task as in-progress after backfilling', () async { + final List oldestGray = [ + generateTask(1, name: 'Linux_android A', status: Task.statusSucceeded), + generateTask(2, name: 'Linux_android A', status: Task.statusSucceeded), + generateTask(3, name: 'Linux_android A', status: Task.statusNew), + ]; + db.addOnQuery((Iterable results) => oldestGray); + final Task task = oldestGray[2]; + expect(db.values.length, 0); + expect(task.status, Task.statusNew); + await tester.get(handler); + expect(db.values.length, 1); + expect(task.status, Task.statusInProgress); + + final List captured = verify(mockFirestoreService.writeViaTransaction(captureAny)).captured; + expect(captured.length, 1); + final List commitResponse = captured[0] as List; + expect(commitResponse.length, 1); + final firestore.Task taskDocuemnt = firestore.Task.fromDocument(taskDocument: commitResponse[0].update!); + expect(taskDocuemnt.status, firestore.Task.statusInProgress); + }); + + test('skip scheduling builds if datastore commit fails', () async { + db.commitException = true; + final List oldestGray = [ + generateTask(1, name: 'Linux_android A', status: Task.statusSucceeded), + generateTask(2, name: 'Linux_android A', status: Task.statusSucceeded), + generateTask(3, name: 'Linux_android A', status: Task.statusNew), + ]; + db.addOnQuery((Iterable results) => oldestGray); + expect(db.values.length, 0); + await tester.get(handler); + expect(db.values.length, 0); + expect(pubsub.messages.length, 0); + }); + + test('backfills only column A when B does not need backfill', () async { + final List scheduleA = [ + // Linux_android A + generateTask(1, name: 'Linux_android A', status: Task.statusSucceeded), + generateTask(2, name: 'Linux_android A', status: Task.statusSucceeded), + generateTask(3, name: 'Linux_android A', status: Task.statusNew), + // Linux_android B + generateTask(1, name: 'Linux_android B', status: Task.statusSucceeded), + generateTask(2, name: 'Linux_android B', status: Task.statusSucceeded), + generateTask(3, name: 'Linux_android B', status: Task.statusSucceeded), + ]; + db.addOnQuery((Iterable results) => scheduleA); + await tester.get(handler); + expect(pubsub.messages.length, 1); + }); + + test('backfills both column A and B', () async { + final List scheduleA = [ + // Linux_android A + generateTask(1, name: 'Linux_android A', status: Task.statusSucceeded), + generateTask(2, name: 'Linux_android A', status: Task.statusSucceeded), + generateTask(3, name: 'Linux_android A', status: Task.statusNew), + // Linux_android B + generateTask(1, name: 'Linux_android B', status: Task.statusSucceeded), + generateTask(2, name: 'Linux_android B', status: Task.statusSucceeded), + generateTask(3, name: 'Linux_android B', status: Task.statusNew), + ]; + db.addOnQuery((Iterable results) => scheduleA); + await tester.get(handler); + expect(pubsub.messages.length, 2); + }); + + test('backfills limited targets when number of available targets exceeds backfillerTargetLimit ', () async { + final List scheduleA = [ + // Linux_android A + generateTask(1, name: 'Linux_android A', status: Task.statusNew), + generateTask(2, name: 'Linux_android A', status: Task.statusNew), + // Linux_android B + generateTask(1, name: 'Linux_android B', status: Task.statusNew), + generateTask(2, name: 'Linux_android B', status: Task.statusNew), + // Linux_android C + generateTask(1, name: 'Linux_android C', status: Task.statusNew), + generateTask(2, name: 'Linux_android C', status: Task.statusNew), + ]; + db.addOnQuery((Iterable results) => scheduleA); + await tester.get(handler); + expect(pubsub.messages.length, 2); + }); + + group('getFilteredBackfill', () { + test('backfills high priorty targets first', () async { + final List> backfill = >[ + Tuple(generateTarget(1), FullTask(generateTask(1), generateCommit(1)), LuciBuildService.kRerunPriority), + Tuple(generateTarget(2), FullTask(generateTask(2), generateCommit(2)), LuciBuildService.kBackfillPriority), + Tuple(generateTarget(3), FullTask(generateTask(3), generateCommit(3)), LuciBuildService.kRerunPriority), + ]; + final List> filteredBackfill = handler.getFilteredBackfill(backfill); + expect(filteredBackfill.length, 2); + expect(filteredBackfill[0].third, LuciBuildService.kRerunPriority); + expect(filteredBackfill[1].third, LuciBuildService.kRerunPriority); + }); + }); + }); +} diff --git a/app_dart/test/request_handlers/scheduler/scheduler_request_subscription_test.dart b/app_dart/test/request_handlers/scheduler/scheduler_request_subscription_test.dart new file mode 100644 index 000000000..eeadaf68c --- /dev/null +++ b/app_dart/test/request_handlers/scheduler/scheduler_request_subscription_test.dart @@ -0,0 +1,163 @@ +// Copyright 2019 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:cocoon_service/cocoon_service.dart'; +import 'package:cocoon_service/src/model/luci/pubsub_message_v2.dart'; +import 'package:cocoon_service/src/request_handlers/scheduler/scheduler_request_subscription.dart'; +import 'package:cocoon_service/src/request_handling/exceptions.dart'; +import 'package:fixnum/fixnum.dart'; +import 'package:mockito/mockito.dart'; +import 'package:retry/retry.dart'; +import 'package:test/test.dart'; +import 'package:buildbucket/buildbucket_pb.dart' as bbv2; + +import '../../src/datastore/fake_config.dart'; +import '../../src/request_handling/fake_authentication.dart'; +import '../../src/request_handling/fake_http.dart'; +import '../../src/request_handling/subscription_v2_tester.dart'; +import '../../src/utilities/mocks.dart'; + +void main() { + late SchedulerRequestSubscriptionV2 handler; + late SubscriptionV2Tester tester; + + late MockBuildBucketV2Client buildBucketV2Client; + + setUp(() async { + buildBucketV2Client = MockBuildBucketV2Client(); + handler = SchedulerRequestSubscriptionV2( + cache: CacheService(inMemory: true), + config: FakeConfig(), + authProvider: FakeAuthenticationProvider(), + buildBucketClient: buildBucketV2Client, + retryOptions: const RetryOptions( + maxAttempts: 3, + maxDelay: Duration.zero, + ), + ); + + tester = SubscriptionV2Tester( + request: FakeHttpRequest(), + ); + }); + + test('throws exception when BatchRequest cannot be decoded', () async { + tester.message = const PushMessageV2(); + expect(() => tester.post(handler), throwsA(isA())); + }); + + test('schedules request to buildbucket using v2', () async { + final bbv2.BuilderID responseBuilderID = bbv2.BuilderID(); + responseBuilderID.builder = 'Linux A'; + + final bbv2.Build responseBuild = bbv2.Build(); + responseBuild.id = Int64(12345); + responseBuild.builder = responseBuilderID; + + // has a list of BatchResponse_Response + final bbv2.BatchResponse batchResponse = bbv2.BatchResponse(); + final bbv2.BatchResponse_Response batchResponseResponse = bbv2.BatchResponse_Response(); + batchResponseResponse.scheduleBuild = responseBuild; + batchResponse.responses.add(batchResponseResponse); + + when(buildBucketV2Client.batch(any)).thenAnswer((_) async => batchResponse); + + // We cannot construct the object manually with the protos as we cannot write out + // the json with all the required double quotes and testing fails. + const String messageData = ''' +{ + "requests": [ + { + "scheduleBuild": { + "builder": { + "builder": "Linux A" + } + } + } + ] +} +'''; + + const PushMessageV2 pushMessageV2 = PushMessageV2(data: messageData, messageId: '798274983'); + tester.message = pushMessageV2; + final Body body = await tester.post(handler); + expect(body, Body.empty); + }); + + test('retries schedule build if no response comes back', () async { + final bbv2.BuilderID responseBuilderID = bbv2.BuilderID(); + responseBuilderID.builder = 'Linux A'; + + final bbv2.Build responseBuild = bbv2.Build(); + responseBuild.id = Int64(12345); + responseBuild.builder = responseBuilderID; + + // has a list of BatchResponse_Response + final bbv2.BatchResponse batchResponse = bbv2.BatchResponse(); + + final bbv2.BatchResponse_Response batchResponseResponse = bbv2.BatchResponse_Response(); + batchResponseResponse.scheduleBuild = responseBuild; + + batchResponse.responses.add(batchResponseResponse); + + int attempt = 0; + + when(buildBucketV2Client.batch(any)).thenAnswer((_) async { + attempt += 1; + if (attempt == 2) { + return batchResponse; + } + + return bbv2.BatchResponse().createEmptyInstance(); + }); + + const String messageData = ''' +{ + "requests": [ + { + "scheduleBuild": { + "builder": { + "builder": "Linux A" + } + } + } + ] +} +'''; + + const PushMessageV2 pushMessageV2 = PushMessageV2(data: messageData, messageId: '798274983'); + tester.message = pushMessageV2; + final Body body = await tester.post(handler); + + expect(body, Body.empty); + expect(verify(buildBucketV2Client.batch(any)).callCount, 2); + }); + + test('acking message and logging error when no response comes back after retry limit', () async { + when(buildBucketV2Client.batch(any)).thenAnswer((_) async { + return bbv2.BatchResponse().createEmptyInstance(); + }); + + const String messageData = ''' +{ + "requests": [ + { + "scheduleBuild": { + "builder": { + "builder": "Linux A" + } + } + } + ] +} +'''; + + const PushMessageV2 pushMessageV2 = PushMessageV2(data: messageData, messageId: '798274983'); + tester.message = pushMessageV2; + final Body body = await tester.post(handler); + + expect(body, isNotNull); + expect(verify(buildBucketV2Client.batch(any)).callCount, 3); + }); +} diff --git a/app_dart/test/request_handlers/vacuum_github_commits_v2_test.dart b/app_dart/test/request_handlers/vacuum_github_commits_v2_test.dart new file mode 100644 index 000000000..8041d6145 --- /dev/null +++ b/app_dart/test/request_handlers/vacuum_github_commits_v2_test.dart @@ -0,0 +1,185 @@ +// Copyright 2019 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:cocoon_service/src/model/appengine/commit.dart'; +import 'package:cocoon_service/src/request_handlers/vacuum_github_commits_v2.dart'; +import 'package:cocoon_service/src/request_handling/body.dart'; +import 'package:cocoon_service/src/service/config.dart'; +import 'package:cocoon_service/src/service/datastore.dart'; +import 'package:gcloud/db.dart' as gcloud_db; +import 'package:gcloud/db.dart'; +import 'package:github/github.dart'; +import 'package:googleapis/bigquery/v2.dart'; +import 'package:mockito/mockito.dart'; +import 'package:test/test.dart'; + +import '../src/datastore/fake_config.dart'; +import '../src/datastore/fake_datastore.dart'; +import '../src/request_handling/api_request_handler_tester.dart'; +import '../src/request_handling/fake_authentication.dart'; +import '../src/service/fake_github_service.dart'; +import '../src/service/fake_scheduler_v2.dart'; +import '../src/utilities/mocks.dart'; + +void main() { + group('VacuumGithubCommits', () { + late FakeConfig config; + FakeAuthenticationProvider auth; + late FakeDatastoreDB db; + FakeSchedulerV2 scheduler; + late ApiRequestHandlerTester tester; + late MockFirestoreService mockFirestoreService; + late VacuumGithubCommitsV2 handler; + + late List githubCommits; + late int yieldedCommitCount; + + List commitList() { + final List commits = []; + for (String sha in githubCommits) { + final User author = User() + ..login = 'Username' + ..avatarUrl = 'http://example.org/avatar.jpg'; + final GitCommitUser committer = + GitCommitUser('Username', 'Username@abc.com', DateTime.fromMillisecondsSinceEpoch(int.parse(sha))); + final GitCommit gitCommit = GitCommit() + ..message = 'commit message' + ..committer = committer; + commits.add( + RepositoryCommit() + ..sha = sha + ..author = author + ..commit = gitCommit, + ); + } + return commits; + } + + Commit shaToCommit(String sha, String branch, RepositorySlug slug) { + return Commit( + key: db.emptyKey.append(Commit, id: '${slug.fullName}/$branch/$sha'), + repository: slug.fullName, + sha: sha, + branch: branch, + timestamp: int.parse(sha), + ); + } + + setUp(() { + final MockRepositoriesService repositories = MockRepositoriesService(); + final FakeGithubService githubService = FakeGithubService(); + final MockTabledataResource tabledataResourceApi = MockTabledataResource(); + mockFirestoreService = MockFirestoreService(); + when(tabledataResourceApi.insertAll(any, any, any, any)).thenAnswer((_) async { + return TableDataInsertAllResponse(); + }); + + yieldedCommitCount = 0; + db = FakeDatastoreDB(); + config = FakeConfig( + tabledataResource: tabledataResourceApi, + githubService: githubService, + firestoreService: mockFirestoreService, + dbValue: db, + supportedBranchesValue: [ + 'master', + 'main', + ], + supportedReposValue: { + Config.cocoonSlug, + Config.engineSlug, + Config.flutterSlug, + Config.packagesSlug, + }, + ); + + auth = FakeAuthenticationProvider(); + scheduler = FakeSchedulerV2( + config: config, + ciYaml: exampleConfig, + ); + tester = ApiRequestHandlerTester(); + handler = VacuumGithubCommitsV2( + config: config, + authenticationProvider: auth, + datastoreProvider: (DatastoreDB db) => DatastoreService(config.db, 5), + scheduler: scheduler, + ); + + githubService.listCommitsBranch = (String branch, int hours) { + return commitList(); + }; + + when(githubService.github.repositories).thenReturn(repositories); + }); + + test('succeeds when GitHub returns no commits', () async { + githubCommits = []; + config.supportedBranchesValue = ['master']; + final Body body = await tester.get(handler); + expect(yieldedCommitCount, 0); + expect(db.values, isEmpty); + expect(await body.serialize().toList(), isEmpty); + }); + + test('does not fail on empty commit list', () async { + githubCommits = []; + expect(db.values.values.whereType().length, 0); + await tester.get(handler); + expect(db.values.values.whereType().length, 0); + }); + + test('does not add recent commits', () async { + githubCommits = ['${DateTime.now().millisecondsSinceEpoch}']; + + expect(db.values.values.whereType().length, 0); + await tester.get(handler); + expect(db.values.values.whereType().length, 0); + }); + + test('inserts all relevant fields of the commit', () async { + githubCommits = ['1']; + expect(db.values.values.whereType().length, 0); + await tester.get(handler); + expect(db.values.values.whereType().length, config.supportedRepos.length); + final List commits = db.values.values.whereType().toList(); + final Commit commit = commits.first; + expect(commit.repository, 'flutter/cocoon'); + expect(commit.branch, 'main'); + expect(commit.sha, '1'); + expect(commit.timestamp, 1); + expect(commit.author, 'Username'); + expect(commit.authorAvatarUrl, 'http://example.org/avatar.jpg'); + expect(commit.message, 'commit message'); + expect(commits[1].repository, Config.engineSlug.fullName); + expect(commits[2].repository, Config.flutterSlug.fullName); + }); + + test('skips commits for which transaction commit fails', () async { + githubCommits = ['2', '3', '4']; + + /// This test is simulating an existing branch, which must already + /// have at least one commit in the datastore. + final Commit commit = shaToCommit('1', 'main', Config.engineSlug); + db.values[commit.key] = commit; + + db.onCommit = (List> inserts, List> deletes) { + if (inserts.whereType().where((Commit commit) => commit.sha == '3').isNotEmpty) { + throw StateError('Commit failed'); + } + }; + final Body body = await tester.get(handler); + + /// The +1 is coming from the engine repository and manually added commit on the top of this test. + expect(db.values.values.whereType().length, 8 + 1); // 2 commits for 4 repos + expect(db.values.values.whereType().map(toSha), containsAll(['1', '2', '4'])); + expect(db.values.values.whereType().map(toTimestamp), containsAll([1, 2, 4])); + expect(await body.serialize().toList(), isEmpty); + }); + }); +} + +String toSha(Commit commit) => commit.sha!; + +int toTimestamp(Commit commit) => commit.timestamp!; diff --git a/app_dart/test/server_test.dart b/app_dart/test/server_test.dart index addb63d98..80c974602 100644 --- a/app_dart/test/server_test.dart +++ b/app_dart/test/server_test.dart @@ -5,14 +5,18 @@ import 'package:cocoon_service/cocoon_service.dart'; import 'package:cocoon_service/server.dart'; import 'package:cocoon_service/src/service/commit_service.dart'; +import 'package:cocoon_service/src/service/github_checks_service_v2.dart'; import 'package:test/test.dart'; import 'src/datastore/fake_config.dart'; import 'src/request_handling/fake_authentication.dart'; +import 'src/service/fake_build_bucket_v2_client.dart'; import 'src/service/fake_buildbucket.dart'; import 'src/service/fake_gerrit_service.dart'; import 'src/service/fake_luci_build_service.dart'; +import 'src/service/fake_luci_build_service_v2.dart'; import 'src/service/fake_scheduler.dart'; +import 'src/service/fake_scheduler_v2.dart'; void main() { test('verify server can be created', () { @@ -25,11 +29,15 @@ void main() { swarmingAuthProvider: FakeAuthenticationProvider(), branchService: BranchService(config: FakeConfig(), gerritService: FakeGerritService()), buildBucketClient: FakeBuildBucketClient(), + buildBucketV2Client: FakeBuildBucketV2Client(), luciBuildService: FakeLuciBuildService(config: FakeConfig()), + luciBuildServiceV2: FakeLuciBuildServiceV2(config: FakeConfig()), githubChecksService: GithubChecksService(FakeConfig()), + githubChecksServiceV2: GithubChecksServiceV2(FakeConfig()), commitService: CommitService(config: FakeConfig()), gerritService: FakeGerritService(), scheduler: FakeScheduler(config: FakeConfig()), + schedulerV2: FakeSchedulerV2(config: FakeConfig()), ); }); } diff --git a/cron.yaml b/cron.yaml index bf1525e53..bfc4d19bc 100644 --- a/cron.yaml +++ b/cron.yaml @@ -7,6 +7,10 @@ cron: url: /api/vacuum-github-commits schedule: every 6 hours +- description: retrieve missing commits v2 + url: /api/v2/vacuum-github-commits + schedule: every 6 hours + # TODO(keyonghan): will delete if `In Progress` hanging issue is resolved: # https://github.com/flutter/flutter/issues/120395#issuecomment-1444810718 - description: vacuum stale tasks @@ -17,6 +21,10 @@ cron: url: /api/scheduler/batch-backfiller schedule: every 5 minutes +- description: backfills builds via the build bucket v2 api + url: /api/v2/scheduler/batch-backfiller + schedule: every 5 minutes + - description: sends build status to GitHub to annotate flutter PRs and commits url: /api/push-build-status-to-github?repo=flutter/flutter schedule: every 1 minutes