Skip to content

Commit

Permalink
Merge branch 'flutter:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
@ricardoamador
ricardoamador authored Mar 20, 2024
2 parents abbb359 + 7a070f6 commit a9d0e5c
Show file tree
Hide file tree
Showing 10 changed files with 845 additions and 836 deletions.
15 changes: 15 additions & 0 deletions app_dart/lib/src/model/firestore/github_build_status.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,21 @@ class GithubBuildStatus extends Document {

int? get updates => int.parse(fields![kGithubBuildStatusUpdatesField]!.integerValue!);

String setStatus(String status) {
fields![kGithubBuildStatusStatusField] = Value(stringValue: status);
return status;
}

int setUpdates(int updates) {
fields![kGithubBuildStatusUpdatesField] = Value(integerValue: updates.toString());
return updates;
}

int setUpdateTimeMillis(int updateTimeMillis) {
fields![kGithubBuildStatusUpdateTimeMillisField] = Value(integerValue: updateTimeMillis.toString());
return updateTimeMillis;
}

@override
String toString() {
final StringBuffer buf = StringBuffer()
Expand Down
87 changes: 29 additions & 58 deletions app_dart/lib/src/request_handlers/postsubmit_luci_subscription.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,9 @@ class PostsubmitLuciSubscription extends SubscriptionHandler {

final String? rawTaskKey = buildPushMessage.userData['task_key'] as String?;
final String? rawCommitKey = buildPushMessage.userData['commit_key'] as String?;
if (rawCommitKey == null) {
throw const BadRequestException('userData does not contain commit_key');
final String? taskDocumentName = buildPushMessage.userData['firestore_task_document_name'] as String?;
if (taskDocumentName == null) {
throw const BadRequestException('userData does not contain firestore_task_document_name');
}
final Build? build = buildPushMessage.build;
if (build == null) {
Expand All @@ -67,57 +68,48 @@ class PostsubmitLuciSubscription extends SubscriptionHandler {
}
final Key<String> commitKey = Key<String>(Key<dynamic>.emptyKey(Partition(null)), Commit, rawCommitKey);
Task? task;
final String? taskName = build.buildParameters?['builder_name'] as String?;
if (rawTaskKey == null || rawTaskKey.isEmpty || rawTaskKey == 'null') {
log.fine('Pulling builder name from parameters_json...');
log.fine(build.buildParameters);
if (taskName == null || taskName.isEmpty) {
throw const BadRequestException('task_key is null and parameters_json does not contain the builder name');
}
final List<Task> tasks = await datastore.queryRecentTasksByName(name: taskName).toList();
task = tasks.singleWhere((Task task) => task.parentKey?.id == commitKey.id);
} else {
log.fine('Looking up key...');
final int taskId = int.parse(rawTaskKey);
final Key<int> taskKey = Key<int>(commitKey, Task, taskId);
task = await datastore.lookupByValue<Task>(taskKey);
}
log.fine('Found $task');

firestore.Task taskDocument = firestore.Task();
firestore.Task? firestoreTask;
log.fine('Looking up task document...');
final int taskId = int.parse(rawTaskKey!);
final Key<int> taskKey = Key<int>(commitKey, Task, taskId);
task = await datastore.lookupByValue<Task>(taskKey);
firestoreTask = await firestore.Task.fromFirestore(
firestoreService: firestoreService,
documentName: '$kDatabase/documents/${firestore.kTaskCollectionId}/$taskDocumentName',
);
log.fine('Found $firestoreTask');

if (_shouldUpdateTask(build, task)) {
final String oldTaskStatus = task.status;
if (_shouldUpdateTask(build, firestoreTask)) {
final String oldTaskStatus = firestoreTask.status;
firestoreTask.updateFromBuild(build);
task.updateFromBuild(build);
await datastore.insert(<Task>[task]);
try {
taskDocument = await updateFirestore(build, rawCommitKey, task.name!, firestoreService);
} catch (error) {
log.warning('Failed to update task in Firestore: $error');
}
log.fine('Updated datastore from $oldTaskStatus to ${task.status}');
final List<Write> writes = documentsToWrites([firestoreTask], exists: true);
await firestoreService.batchWriteDocuments(BatchWriteRequest(writes: writes), kDatabase);
log.fine('Updated datastore from $oldTaskStatus to ${firestoreTask.status}');
} else {
log.fine('skip processing for build with status scheduled or task with status finished.');
}

final Commit commit = await datastore.lookupByValue<Commit>(commitKey);
final CiYaml ciYaml = await scheduler.getCiYaml(commit);
final List<Target> postsubmitTargets = ciYaml.postsubmitTargets;
if (!postsubmitTargets.any((element) => element.value.name == task!.name)) {
log.warning('Target ${task.name} has been deleted from TOT. Skip updating.');
if (!postsubmitTargets.any((element) => element.value.name == firestoreTask!.taskName)) {
log.warning('Target ${firestoreTask.taskName} has been deleted from TOT. Skip updating.');
return Body.empty;
}
final Target target = postsubmitTargets.singleWhere((Target target) => target.value.name == task!.name);
if (task.status == Task.statusFailed ||
task.status == Task.statusInfraFailure ||
task.status == Task.statusCancelled) {
final Target target =
postsubmitTargets.singleWhere((Target target) => target.value.name == firestoreTask!.taskName);
if (firestoreTask.status == firestore.Task.statusFailed ||
firestoreTask.status == firestore.Task.statusInfraFailure ||
firestoreTask.status == firestore.Task.statusCancelled) {
log.fine('Trying to auto-retry...');
final bool retried = await scheduler.luciBuildService.checkRerunBuilder(
commit: commit,
target: target,
task: task,
datastore: datastore,
taskDocument: taskDocument,
taskDocument: firestoreTask,
firestoreService: firestoreService,
);
log.info('Retried: $retried');
Expand All @@ -144,28 +136,7 @@ class PostsubmitLuciSubscription extends SubscriptionHandler {
// b) `completed`: update info like status.
// 2) the task is already completed.
// The task may have been marked as completed from test framework via update-task-status API.
bool _shouldUpdateTask(Build build, Task task) {
return build.status != Status.scheduled && !Task.finishedStatusValues.contains(task.status);
}

/// Queries the task document and updates based on the latest build data.
Future<firestore.Task> updateFirestore(
Build build,
String commitKeyId,
String taskName,
FirestoreService firestoreService,
) async {
final int currentAttempt = githubChecksService.currentAttempt(build);
final String sha = commitKeyId.split('/').last;
final String documentName = '$kDatabase/documents/tasks/${sha}_${taskName}_$currentAttempt';
log.info('getting firestore document: $documentName');
final firestore.Task firestoreTask =
await firestore.Task.fromFirestore(firestoreService: firestoreService, documentName: documentName);
log.info('updating firestoreTask based on build');
firestoreTask.updateFromBuild(build);
log.info('finished updating firestoreTask based on builds');
final List<Write> writes = documentsToWrites([firestoreTask], exists: true);
await firestoreService.batchWriteDocuments(BatchWriteRequest(writes: writes), kDatabase);
return firestoreTask;
bool _shouldUpdateTask(Build build, firestore.Task task) {
return build.status != Status.scheduled && !firestore.Task.finishedStatusValues.contains(task.status);
}
}
42 changes: 24 additions & 18 deletions app_dart/lib/src/request_handlers/push_build_status_to_github.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,58 +46,64 @@ class PushBuildStatusToGithub extends ApiRequestHandler<Body> {

final BuildStatus status = (await buildStatusService.calculateCumulativeStatus(slug))!;
await _insertBigquery(slug, status.githubStatus, Config.defaultBranch(slug), config);
await _updatePRs(slug, status.githubStatus, datastore);
await _updatePRs(slug, status.githubStatus, datastore, firestoreService);
log.fine('All the PRs for $repository have been updated with $status');

return Body.empty;
}

Future<void> _updatePRs(RepositorySlug slug, String status, DatastoreService datastore) async {
Future<void> _updatePRs(
RepositorySlug slug,
String status,
DatastoreService datastore,
FirestoreService firestoreService,
) async {
final GitHub github = await config.createGitHubClient(slug: slug);
final List<GithubBuildStatusUpdate> updates = <GithubBuildStatusUpdate>[];
final List<GithubBuildStatus> githubBuildStatuses = <GithubBuildStatus>[];
await for (PullRequest pr in github.pullRequests.list(slug)) {
// Tree status is only put on PRs merging into ToT.
if (pr.base!.ref != Config.defaultBranch(slug)) {
log.fine('This PR is not staged to land on ${Config.defaultBranch(slug)}, skipping.');
continue;
}
final GithubBuildStatusUpdate update = await datastore.queryLastStatusUpdate(slug, pr);
if (update.status != status) {
log.fine('Updating status of ${slug.fullName}#${pr.number} from ${update.status} to $status');
final GithubBuildStatus githubBuildStatus =
await firestoreService.queryLastBuildStatus(slug, pr.number!, pr.head!.sha!);
if (githubBuildStatus.status != status) {
log.fine('Updating status of ${slug.fullName}#${pr.number} from ${githubBuildStatus.status} to $status');
final CreateStatus request = CreateStatus(status);
request.targetUrl = 'https://flutter-dashboard.appspot.com/#/build?repo=${slug.name}';
request.context = 'tree-status';
if (status != GithubBuildStatusUpdate.statusSuccess) {
if (status != GithubBuildStatus.statusSuccess) {
request.description = config.flutterBuildDescription;
}
try {
await github.repositories.createStatus(slug, pr.head!.sha!, request);
final int currentTimeMillisecondsSinceEpoch = DateTime.now().millisecondsSinceEpoch;
update.status = status;
update.updates = (update.updates ?? 0) + 1;
update.updateTimeMillis = DateTime.now().millisecondsSinceEpoch;
update.updateTimeMillis = currentTimeMillisecondsSinceEpoch;
updates.add(update);

githubBuildStatus.setStatus(status);
githubBuildStatus.setUpdates((githubBuildStatus.updates ?? 0) + 1);
githubBuildStatus.setUpdateTimeMillis(currentTimeMillisecondsSinceEpoch);
githubBuildStatuses.add(githubBuildStatus);
} catch (error) {
log.severe('Failed to post status update to ${slug.fullName}#${pr.number}: $error');
}
}
}
await datastore.insert(updates);
// TODO(keyonghan): remove try block after fully migrated to firestore
// https://github.com/flutter/flutter/issues/142951
try {
await updateGithubBuildStatusDocuments(updates);
} catch (error) {
log.warning('Failed to update github build status in Firestore: $error');
}
await updateGithubBuildStatusDocuments(githubBuildStatuses);
}

Future<void> updateGithubBuildStatusDocuments(List<GithubBuildStatusUpdate> updates) async {
if (updates.isEmpty) {
Future<void> updateGithubBuildStatusDocuments(List<GithubBuildStatus> githubBuildStatuses) async {
if (githubBuildStatuses.isEmpty) {
return;
}
final List<GithubBuildStatus> githubBuildStatusDocuments =
updates.map((e) => githubBuildStatusToDocument(e)).toList();
final List<Write> writes = documentsToWrites(githubBuildStatusDocuments);
final List<Write> writes = documentsToWrites(githubBuildStatuses);
final FirestoreService firestoreService = await config.createFirestoreService();
await firestoreService.batchWriteDocuments(BatchWriteRequest(writes: writes), kDatabase);
}
Expand Down
37 changes: 37 additions & 0 deletions app_dart/lib/src/service/firestore.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import 'package:googleapis/firestore/v1.dart';
import 'package:http/http.dart';

import '../model/firestore/commit.dart';
import '../model/firestore/github_build_status.dart';
import '../model/firestore/github_gold_status.dart';
import '../model/firestore/task.dart';
import 'access_client_provider.dart';
Expand Down Expand Up @@ -154,6 +155,42 @@ class FirestoreService {
}
}

/// Queries the last updated build status for the [slug], [prNumber], and [head].
///
/// If not existing, returns a fresh new Build status.
Future<GithubBuildStatus> queryLastBuildStatus(RepositorySlug slug, int prNumber, String head) async {
final Map<String, Object> filterMap = <String, Object>{
'$kGithubBuildStatusPrNumberField =': prNumber,
'$kGithubBuildStatusRepositoryField =': slug.fullName,
'$kGithubBuildStatusHeadField =': head,
};
final List<Document> documents = await query(kGithubBuildStatusCollectionId, filterMap);
final List<GithubBuildStatus> githubBuildStatuses =
documents.map((Document document) => GithubBuildStatus.fromDocument(githubBuildStatus: document)).toList();
if (githubBuildStatuses.isEmpty) {
return GithubBuildStatus.fromDocument(
githubBuildStatus: Document(
name: '$kDatabase/documents/$kGithubBuildStatusCollectionId/${head}_$prNumber',
fields: <String, Value>{
kGithubBuildStatusPrNumberField: Value(integerValue: prNumber.toString()),
kGithubBuildStatusHeadField: Value(stringValue: head),
kGithubBuildStatusStatusField: Value(stringValue: ''),
kGithubBuildStatusUpdatesField: Value(integerValue: '0'),
kGithubBuildStatusUpdateTimeMillisField:
Value(integerValue: DateTime.now().millisecondsSinceEpoch.toString()),
kGithubBuildStatusRepositoryField: Value(stringValue: slug.fullName),
},
),
);
} else {
if (githubBuildStatuses.length > 1) {
throw StateError('GithubBuildStatus should have no more than one entry on '
'repository ${slug.fullName}, pr $prNumber, and head $head.');
}
return githubBuildStatuses.single;
}
}

/// Returns Firestore [Value] based on corresponding object type.
Value getValueFromFilter(Object comparisonOject) {
if (comparisonOject is int) {
Expand Down
9 changes: 7 additions & 2 deletions app_dart/lib/src/service/luci_build_service.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import 'package:googleapis/pubsub/v1.dart';
import '../foundation/github_checks_util.dart';
import '../model/appengine/commit.dart';
import '../model/appengine/task.dart';
import '../model/firestore/task.dart' as f;
import '../model/firestore/task.dart' as firestore;
import '../model/ci_yaml/target.dart';
import '../model/github/checks.dart' as cocoon_checks;
import '../model/luci/buildbucket.dart';
Expand Down Expand Up @@ -606,6 +606,7 @@ class LuciBuildService {
final Map<String, dynamic> rawUserData = <String, dynamic>{
'commit_key': commitKey,
'task_key': taskKey,
'firestore_commit_document_name': commit.sha,
};

// Creates post submit checkrun only for unflaky targets from [config.postsubmitSupportedRepos].
Expand All @@ -618,6 +619,9 @@ class LuciBuildService {
tags['scheduler_job_id'] = <String>['flutter/${target.value.name}'];
// Default attempt is the initial attempt, which is 1.
tags['current_attempt'] = tags['current_attempt'] ?? <String>['1'];
final String currentAttempt = tags['current_attempt']!.single;
rawUserData['firestore_task_document_name'] = '${commit.sha}_${task.name}_$currentAttempt';

final Map<String, Object> processedProperties = target.getProperties();
processedProperties.addAll(properties ?? <String, Object>{});
processedProperties['git_branch'] = commit.branch!;
Expand Down Expand Up @@ -684,7 +688,7 @@ class LuciBuildService {
FirestoreService? firestoreService,
Map<String, List<String>>? tags,
bool ignoreChecks = false,
f.Task? taskDocument,
firestore.Task? taskDocument,
}) async {
if (ignoreChecks == false && await _shouldRerunBuilder(task, commit, datastore) == false) {
return false;
Expand All @@ -699,6 +703,7 @@ class LuciBuildService {
final int newAttempt = int.parse(taskDocument.name!.split('_').last) + 1;
tags['current_attempt'] = <String>[newAttempt.toString()];
taskDocument.resetAsRetry(attempt: newAttempt);
taskDocument.setStatus(firestore.Task.statusInProgress);
final List<Write> writes = documentsToWrites([taskDocument], exists: false);
await firestoreService!.batchWriteDocuments(BatchWriteRequest(writes: writes), kDatabase);
} catch (error) {
Expand Down
Loading

0 comments on commit a9d0e5c

Please sign in to comment.