use of com.hubspot.deploy.HealthcheckOptions in project Singularity by HubSpot.
the class SingularityHealthchecksTest method testStartupTimeout.
@Test
public void testStartupTimeout() {
initRequest();
final long hourAgo = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1);
final String deployId = "startup_timeout_test";
HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setMaxRetries(Optional.of(2)).setStartupTimeoutSeconds(Optional.of((int) TimeUnit.MINUTES.toSeconds(30))).build();
SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId).setHealthcheck(Optional.of(options));
db.setDeployHealthTimeoutSeconds(Optional.of(TimeUnit.DAYS.toMillis(1)));
SingularityDeploy deploy = initDeploy(db, hourAgo);
deployChecker.checkDeploys();
Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent());
SingularityTask task = launchTask(request, deploy, hourAgo, hourAgo + 1, 1, TaskState.TASK_RUNNING);
deployChecker.checkDeploys();
Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent());
taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.<Integer>absent(), Optional.of(1000L), hourAgo + 1, Optional.<String>absent(), Optional.of("ERROR"), task.getTaskId(), Optional.of(true)));
deployChecker.checkDeploys();
Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, deployId).get().getDeployState());
}
use of com.hubspot.deploy.HealthcheckOptions in project Singularity by HubSpot.
the class SingularityHealthchecksTest method testPortIndices.
@Test
public void testPortIndices() {
try {
setConfigurationForNoDelay();
initRequest();
HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setPortIndex(Optional.of(1)).setStartupDelaySeconds(Optional.of(0)).build();
firstDeploy = initAndFinishDeploy(request, new SingularityDeployBuilder(request.getId(), firstDeployId).setCommand(Optional.of("sleep 100")).setHealthcheck(Optional.of(options)), Optional.of(new Resources(1, 64, 3, 0)));
requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser);
scheduler.drainPendingQueue();
String[] portRange = { "80:82" };
sms.resourceOffers(Arrays.asList(createOffer(20, 20000, 50000, "slave1", "host1", Optional.<String>absent(), Collections.<String, String>emptyMap(), portRange)));
SingularityTaskId firstTaskId = taskManager.getActiveTaskIdsForRequest(requestId).get(0);
SingularityTask firstTask = taskManager.getTask(firstTaskId).get();
statusUpdate(firstTask, TaskState.TASK_RUNNING);
newTaskChecker.enqueueNewTaskCheck(firstTask, requestManager.getRequest(requestId), healthchecker);
Awaitility.await("healthcheck present").atMost(5, TimeUnit.SECONDS).until(() -> taskManager.getLastHealthcheck(firstTask.getTaskId()).isPresent());
Assert.assertTrue(taskManager.getLastHealthcheck(firstTask.getTaskId()).get().toString().contains("host1:81"));
} finally {
unsetConfigurationForNoDelay();
}
}
use of com.hubspot.deploy.HealthcheckOptions in project Singularity by HubSpot.
the class SingularityHealthchecksTest method testNewTaskCheckerRespectsDeployHealthcheckRetries.
@Test
public void testNewTaskCheckerRespectsDeployHealthcheckRetries() {
initRequest();
final String deployId = "new_task_healthcheck";
HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setMaxRetries(Optional.of(1)).build();
SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId).setHealthcheck(Optional.of(options));
SingularityDeploy deploy = initAndFinishDeploy(request, db, Optional.absent());
SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_RUNNING);
Assert.assertEquals(CheckTaskState.CHECK_IF_HEALTHCHECK_OVERDUE, newTaskChecker.getTaskState(task, requestManager.getRequest(requestId), healthchecker));
Assert.assertTrue(taskManager.getCleanupTaskIds().isEmpty());
taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), System.currentTimeMillis() + 1, Optional.<String>absent(), Optional.<String>absent(), task.getTaskId(), Optional.<Boolean>absent()));
Assert.assertEquals(CheckTaskState.CHECK_IF_HEALTHCHECK_OVERDUE, newTaskChecker.getTaskState(task, requestManager.getRequest(requestId), healthchecker));
taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), System.currentTimeMillis() + 1, Optional.<String>absent(), Optional.<String>absent(), task.getTaskId(), Optional.<Boolean>absent()));
Assert.assertEquals(CheckTaskState.UNHEALTHY_KILL_TASK, newTaskChecker.getTaskState(task, requestManager.getRequest(requestId), healthchecker));
}
use of com.hubspot.deploy.HealthcheckOptions in project Singularity by HubSpot.
the class SingularityHealthchecksTest method testHealthchecksTimeout.
@Test
public void testHealthchecksTimeout() {
initRequest();
final long hourAgo = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1);
final String deployId = "timeout_test";
HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setMaxRetries(Optional.of(2)).build();
SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId).setHealthcheck(Optional.of(options));
db.setDeployHealthTimeoutSeconds(Optional.of(TimeUnit.DAYS.toMillis(1)));
SingularityDeploy deploy = initDeploy(db, hourAgo);
deployChecker.checkDeploys();
Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent());
SingularityTask task = launchTask(request, deploy, hourAgo, hourAgo + 1, 1, TaskState.TASK_RUNNING);
deployChecker.checkDeploys();
Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent());
taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), hourAgo + 1, Optional.<String>absent(), Optional.<String>absent(), task.getTaskId(), Optional.<Boolean>absent()));
deployChecker.checkDeploys();
Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, deployId).get().getDeployState());
}
use of com.hubspot.deploy.HealthcheckOptions in project Singularity by HubSpot.
the class SingularityHealthchecksTest method testFailingStatusCodes.
@Test
public void testFailingStatusCodes() {
initRequest();
final String deployId = "retry_test";
List<Integer> failureCodes = ImmutableList.of(404);
HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setMaxRetries(Optional.of(3)).setFailureStatusCodes(Optional.of(failureCodes)).build();
SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId).setHealthcheck(Optional.of(options));
SingularityDeploy deploy = initDeploy(db, System.currentTimeMillis());
deployChecker.checkDeploys();
Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent());
SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_RUNNING);
deployChecker.checkDeploys();
Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent());
taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent(), task.getTaskId(), Optional.<Boolean>absent()));
deployChecker.checkDeploys();
Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent());
taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(404), Optional.of(1000L), System.currentTimeMillis() + 1, Optional.<String>absent(), Optional.<String>absent(), task.getTaskId(), Optional.<Boolean>absent()));
deployChecker.checkDeploys();
// Bad status code should cause instant failure even though retries remain
Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, deployId).get().getDeployState());
}
Aggregations