Skip to content

Commit

Permalink
Fixing deployment problems which removed existing services, blocked f…
Browse files Browse the repository at this point in the history 
…orever, ran in parallel (#405)
  • Loading branch information
@MikeDombo
MikeDombo authored Sep 1, 2020
1 parent d7ee152 commit 98fa16e
Show file tree
Hide file tree
Showing 4 changed files with 31 additions and 30 deletions.
35 changes: 18 additions & 17 deletions ...s/java/com/aws/iot/evergreen/integrationtests/deployment/DeploymentConfigMergingTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,11 @@
import static com.aws.iot.evergreen.kernel.GenericExternalService.LIFECYCLE_RUN_NAMESPACE_TOPIC;
import static com.aws.iot.evergreen.kernel.Lifecycle.LIFECYCLE_STARTUP_NAMESPACE_TOPIC;
import static com.aws.iot.evergreen.testcommons.testutilities.ExceptionLogProtector.ignoreExceptionUltimateCauseWithMessage;
import static com.github.grantwest.eventually.EventuallyLambdaMatcher.eventuallyEval;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsInRelativeOrder;
import static org.hamcrest.Matchers.is;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
Expand Down Expand Up @@ -105,18 +107,18 @@ void GIVEN_kernel_running_with_some_config_WHEN_merge_simple_yaml_file_THEN_conf
kernel.launch();

// WHEN
AtomicBoolean mainRestarted = new AtomicBoolean(false);
CountDownLatch mainRestarted = new CountDownLatch(1);
kernel.getContext().addGlobalStateChangeListener((service, oldState, newState) -> {
if (service.getName().equals("main") && newState.equals(State.RUNNING) && oldState.equals(State.STARTING)) {
mainRestarted.set(true);
mainRestarted.countDown();
}
});
deploymentConfigMerger.mergeInNewConfig(testDeployment(),
(Map<Object, Object>) JSON.std.with(new YAMLFactory()).anyFrom(getClass().getResource("delta.yaml")))
.get(60, TimeUnit.SECONDS);

// THEN
assertTrue(mainRestarted.get());
assertTrue(mainRestarted.await(10, TimeUnit.SECONDS));
assertThat((String) kernel.findServiceTopic("main")
.find(SERVICE_LIFECYCLE_NAMESPACE_TOPIC, LIFECYCLE_RUN_NAMESPACE_TOPIC).getOnce(),
containsString("echo Now we\\'re in phase 3"));
Expand Down Expand Up @@ -147,10 +149,10 @@ void GIVEN_kernel_running_single_service_WHEN_merge_changes_service_THEN_service
assertTrue(mainRunning.await(5, TimeUnit.SECONDS));

// WHEN
AtomicBoolean mainRestarted = new AtomicBoolean(false);
CountDownLatch mainRestarted = new CountDownLatch(1);
kernel.getContext().addGlobalStateChangeListener((service, oldState, newState) -> {
if (service.getName().equals("main") && newState.equals(State.FINISHED) && oldState.equals(State.STARTING)) {
mainRestarted.set(true);
mainRestarted.countDown();
}
});

Expand All @@ -165,7 +167,7 @@ void GIVEN_kernel_running_single_service_WHEN_merge_changes_service_THEN_service
}}).get(60, TimeUnit.SECONDS);

// THEN
assertTrue(mainRestarted.get());
assertTrue(mainRestarted.await(10, TimeUnit.SECONDS));
assertEquals("redefined", kernel.findServiceTopic("main").find(SETENV_CONFIG_NAMESPACE, "HELLO").getOnce());
assertTrue(safeUpdateRegistered.get());

Expand Down Expand Up @@ -242,7 +244,7 @@ void GIVEN_kernel_running_single_service_WHEN_merge_change_adding_nested_depende
assertTrue(mainRunning.await(5, TimeUnit.SECONDS));

// WHEN
AtomicBoolean mainRestarted = new AtomicBoolean(false);
CountDownLatch mainRestarted = new CountDownLatch(1);
AtomicBoolean newService2Started = new AtomicBoolean(false);
AtomicBoolean newServiceStarted = new AtomicBoolean(false);

Expand All @@ -258,7 +260,7 @@ void GIVEN_kernel_running_single_service_WHEN_merge_change_adding_nested_depende
// Only count main as started if its dependency (new_service) has already been started
if (newServiceStarted.get() && service.getName().equals("main") && newState.equals(State.FINISHED)
&& oldState.equals(State.STARTING)) {
mainRestarted.set(true);
mainRestarted.countDown();
}
});

Expand Down Expand Up @@ -290,7 +292,7 @@ void GIVEN_kernel_running_single_service_WHEN_merge_change_adding_nested_depende
// THEN
assertTrue(newService2Started.get());
assertTrue(newServiceStarted.get());
assertTrue(mainRestarted.get());
assertTrue(mainRestarted.await(10, TimeUnit.SECONDS));
assertThat(kernel.orderedDependencies().stream().map(EvergreenService::getName).collect(Collectors.toList()),
containsInRelativeOrder("new_service2", "new_service", "main"));
}
Expand Down Expand Up @@ -335,7 +337,7 @@ void GIVEN_kernel_running_single_service_WHEN_merge_same_doc_happens_twice_THEN_
assertTrue(mainRunning.await(5, TimeUnit.SECONDS));

// do first merge
AtomicBoolean mainRestarted = new AtomicBoolean(false);
CountDownLatch mainRestarted = new CountDownLatch(1);
AtomicBoolean newService2Started = new AtomicBoolean(false);
AtomicBoolean newServiceStarted = new AtomicBoolean(false);
GlobalStateChangeListener listener = (service, oldState, newState) -> {
Expand All @@ -348,7 +350,7 @@ void GIVEN_kernel_running_single_service_WHEN_merge_same_doc_happens_twice_THEN_
// Only count main as started if its dependency (new_service) has already been started
if (newServiceStarted.get() && service.getName().equals("main") && newState.equals(State.FINISHED)
&& oldState.equals(State.STARTING)) {
mainRestarted.set(true);
mainRestarted.countDown();
}
};
kernel.getContext().addGlobalStateChangeListener(listener);
Expand All @@ -357,10 +359,9 @@ void GIVEN_kernel_running_single_service_WHEN_merge_same_doc_happens_twice_THEN_
deploymentConfigMerger.mergeInNewConfig(testDeployment(), newConfig).get(60, TimeUnit.SECONDS);

// Verify that first merge succeeded.
assertEquals(State.FINISHED, main.getState());
assertTrue(newService2Started.get());
assertTrue(newServiceStarted.get());
assertTrue(mainRestarted.get());
assertTrue(mainRestarted.await(10, TimeUnit.SECONDS));
assertThat(kernel.orderedDependencies().stream().map(EvergreenService::getName).collect(Collectors.toList()),
containsInRelativeOrder("new_service2", "new_service", "main"));

Expand Down Expand Up @@ -425,7 +426,7 @@ void GIVEN_kernel_running_services_WHEN_merge_removes_service_THEN_removed_servi
DeploymentResult deploymentResult = deploymentFuture.get(30, TimeUnit.SECONDS);
assertEquals(SUCCESSFUL, deploymentResult.getDeploymentStatus());
EvergreenService main = kernel.locate("main");
assertEquals(State.RUNNING, main.getState());
assertThat(main::getState, eventuallyEval(is(State.RUNNING)));
EvergreenService sleeperB = kernel.locate("sleeperB");
assertEquals(State.RUNNING, sleeperB.getState());
// ensure context finish all tasks
Expand Down Expand Up @@ -581,10 +582,10 @@ void GIVEN_kernel_running_single_service_WHEN_deployment_with_skip_safety_check_
assertTrue(mainRunning.await(5, TimeUnit.SECONDS));

// WHEN
AtomicBoolean mainRestarted = new AtomicBoolean(false);
CountDownLatch mainRestarted = new CountDownLatch(1);
kernel.getContext().addGlobalStateChangeListener((service, oldState, newState) -> {
if (service.getName().equals("main") && newState.equals(State.FINISHED) && oldState.equals(State.STARTING)) {
mainRestarted.set(true);
mainRestarted.countDown();
}
});
AtomicBoolean safeUpdateSkipped= new AtomicBoolean();
Expand All @@ -607,7 +608,7 @@ void GIVEN_kernel_running_single_service_WHEN_deployment_with_skip_safety_check_
}}).get(60, TimeUnit.SECONDS);

// THEN
assertTrue(mainRestarted.get());
assertTrue(mainRestarted.await(10, TimeUnit.SECONDS));
assertEquals("redefined", kernel.findServiceTopic("main").find(SETENV_CONFIG_NAMESPACE, "HELLO").getOnce());
assertTrue(safeUpdateSkipped.get());

Expand Down
4 changes: 3 additions & 1 deletion src/main/java/com/aws/iot/evergreen/deployment/DeploymentConfigMerger.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,8 @@ public static void waitForServicesToStart(Collection<EvergreenService> servicesT
allServicesRunning = false;
continue;
}
if (State.RUNNING.equals(state) || State.FINISHED.equals(state)) {
if (State.RUNNING.equals(state) || State.FINISHED.equals(state) || !service.shouldAutoStart()
&& service.reachedDesiredState()) {
continue;
}
allServicesRunning = false;
Expand Down Expand Up @@ -274,6 +275,7 @@ public Set<EvergreenService> servicesToTrack() throws ServiceLoadException {
EvergreenService eg = kernel.locate(serviceName);
servicesToTrack.add(eg);
}
servicesToTrack.remove(kernel.getMain());
return servicesToTrack;
}

Expand Down
20 changes: 9 additions & 11 deletions src/main/java/com/aws/iot/evergreen/deployment/DeploymentService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,18 +188,17 @@ protected void startup() throws InterruptedException {
// Assuming cancel will either cancel the current deployment or wait till it finishes
cancelCurrentDeployment();
}
if (currentDeploymentTaskMetadata != null && !deployment.getDeploymentType()
.equals(currentDeploymentTaskMetadata.getDeploymentType())) {
// deployment from another source, wait till the current deployment finishes
continue;
}
if (currentDeploymentTaskMetadata != null && deployment.getId()
.equals(currentDeploymentTaskMetadata.getDeploymentId()) && deployment.getDeploymentType()
.equals(currentDeploymentTaskMetadata.getDeploymentType())) {
//Duplicate message and already processing this deployment so nothing is needed
// Duplicate message and already processing this deployment so nothing is needed
deploymentsQueue.remove();
continue;
}
if (currentDeploymentTaskMetadata != null) {
// wait till the current deployment finishes
continue;
}
deploymentsQueue.remove();
if (!deployment.isCancelled()) {
createNewDeployment(deployment);
Expand Down Expand Up @@ -407,15 +406,14 @@ private DeploymentDocument parseAndValidateJobDocument(Deployment deployment) th
OBJECT_MAPPER.readValue(jobDocumentString, LocalOverrideRequest.class);
Map<String, String> rootComponents = new HashMap<>();
Set<String> rootComponentsInRequestedGroup = new HashSet<>();
config.lookupTopics(GROUP_TO_ROOT_COMPONENTS_TOPICS).lookupTopics(
config.lookupTopics(GROUP_TO_ROOT_COMPONENTS_TOPICS,
localOverrideRequest.getGroupName() == null ? DEFAULT_GROUP_NAME
: localOverrideRequest.getGroupName())
.deepForEachTopic(t -> rootComponentsInRequestedGroup.add(t.getName()));

//TODO: pulling the versions from kernel. Can pull it form the config itself.
.forEach(t -> rootComponentsInRequestedGroup.add(t.getName()));
//TODO: pulling the versions from kernel. Can pull it from the config itself.
// Confirm if pulling from config should not break any use case for local
if (!CollectionUtils.isNullOrEmpty(rootComponentsInRequestedGroup)) {
rootComponentsInRequestedGroup.stream().forEach(c -> {
rootComponentsInRequestedGroup.forEach(c -> {
Topics serviceTopic = kernel.findServiceTopic(c);
if (serviceTopic != null) {
String version = Coerce.toString(serviceTopic.find(VERSION_CONFIG_KEY));
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/aws/iot/evergreen/deployment/activator/DefaultActivator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,4 +148,4 @@ void rollback(DeploymentDocument deploymentDocument, CompletableFuture<Deploymen
});
});
}
}
}

0 comments on commit 98fa16e

Please sign in to comment.