use of com.hubspot.singularity.SingularityDeployMarker in project Singularity by HubSpot.
the class DeployManager method saveDeploy.
public SingularityCreateResult saveDeploy(SingularityRequest request, SingularityDeployMarker deployMarker, SingularityDeploy deploy) {
final SingularityCreateResult deploySaveResult = create(getDeployDataPath(deploy.getRequestId(), deploy.getId()), deploy, deployTranscoder);
if (deploySaveResult == SingularityCreateResult.EXISTED) {
LOG.info("Deploy object for {} already existed (new marker: {})", deploy, deployMarker);
}
singularityEventListener.deployHistoryEvent(new SingularityDeployUpdate(deployMarker, Optional.of(deploy), DeployEventType.STARTING, Optional.<SingularityDeployResult>absent()));
create(getDeployMarkerPath(deploy.getRequestId(), deploy.getId()), deployMarker, deployMarkerTranscoder);
final Optional<SingularityRequestDeployState> currentState = getRequestDeployState(deploy.getRequestId());
Optional<SingularityDeployMarker> activeDeploy = Optional.absent();
Optional<SingularityDeployMarker> pendingDeploy = Optional.absent();
if (request.isDeployable()) {
if (currentState.isPresent()) {
activeDeploy = currentState.get().getActiveDeploy();
}
pendingDeploy = Optional.of(deployMarker);
} else {
activeDeploy = Optional.of(deployMarker);
}
final SingularityRequestDeployState newState = new SingularityRequestDeployState(deploy.getRequestId(), activeDeploy, pendingDeploy);
return saveNewRequestDeployState(newState);
}
use of com.hubspot.singularity.SingularityDeployMarker in project Singularity by HubSpot.
the class SingularitySchedulerTest method testCleanerFindsTasksWithSkippedHealthchecks.
@Test
public void testCleanerFindsTasksWithSkippedHealthchecks() {
initRequest();
// set up slaves so scale validate will pass
resourceOffers(2);
SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
long now = System.currentTimeMillis();
requestManager.saveHistory(new SingularityRequestHistory(now, Optional.<String>absent(), RequestHistoryType.UPDATED, request.toBuilder().setSkipHealthchecks(Optional.of(true)).setInstances(Optional.of(2)).build(), Optional.<String>absent()));
firstDeploy = initDeploy(new SingularityDeployBuilder(request.getId(), firstDeployId).setCommand(Optional.of("sleep 100")).setHealthcheckUri(Optional.of("http://uri")), System.currentTimeMillis());
SingularityTask taskOne = launchTask(request, firstDeploy, now + 1000, now + 2000, 1, TaskState.TASK_RUNNING);
finishDeploy(new SingularityDeployMarker(requestId, firstDeployId, now + 2000, Optional.<String>absent(), Optional.<String>absent()), firstDeploy);
SingularityRequest updatedRequest = request.toBuilder().setSkipHealthchecks(Optional.<Boolean>absent()).setInstances(Optional.of(2)).build();
requestManager.saveHistory(new SingularityRequestHistory(now + 3000, Optional.<String>absent(), RequestHistoryType.UPDATED, updatedRequest, Optional.<String>absent()));
SingularityTask newTaskTwoWithCheck = prepTask(updatedRequest, firstDeploy, now + 4000, 2);
taskManager.createTaskAndDeletePendingTask(newTaskTwoWithCheck);
statusUpdate(newTaskTwoWithCheck, TaskState.TASK_RUNNING, Optional.of(now + 5000));
taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(200), Optional.of(1000L), now + 6000, Optional.<String>absent(), Optional.<String>absent(), newTaskTwoWithCheck.getTaskId(), Optional.<Boolean>absent()));
SingularityTask unhealthyTaskThree = prepTask(updatedRequest, firstDeploy, now + 4000, 3);
taskManager.createTaskAndDeletePendingTask(unhealthyTaskThree);
statusUpdate(unhealthyTaskThree, TaskState.TASK_RUNNING, Optional.of(now + 5000));
List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIdsForRequest(requestId);
List<SingularityTaskId> healthyTaskIds = deployHealthHelper.getHealthyTasks(updatedRequest, Optional.of(firstDeploy), activeTaskIds, false);
Assert.assertTrue(!healthyTaskIds.contains(unhealthyTaskThree.getTaskId()));
// Healthchecked and skip-healthchecked tasks should both be here
Assert.assertEquals(2, healthyTaskIds.size());
Assert.assertEquals(DeployHealth.WAITING, deployHealthHelper.getDeployHealth(updatedRequest, Optional.of(firstDeploy), activeTaskIds, false));
taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(200), Optional.of(1000L), now + 6000, Optional.<String>absent(), Optional.<String>absent(), unhealthyTaskThree.getTaskId(), Optional.<Boolean>absent()));
Assert.assertEquals(DeployHealth.HEALTHY, deployHealthHelper.getDeployHealth(updatedRequest, Optional.of(firstDeploy), activeTaskIds, false));
}
use of com.hubspot.singularity.SingularityDeployMarker in project Singularity by HubSpot.
the class SingularityDeployChecker method checkDeploy.
private void checkDeploy(final SingularityPendingDeploy pendingDeploy, final List<SingularityDeployMarker> cancelDeploys, final Map<SingularityPendingDeploy, SingularityDeployKey> pendingDeployToKey, final Map<SingularityDeployKey, SingularityDeploy> deployKeyToDeploy, List<SingularityUpdatePendingDeployRequest> updateRequests) {
final SingularityDeployKey deployKey = pendingDeployToKey.get(pendingDeploy);
final Optional<SingularityDeploy> deploy = Optional.fromNullable(deployKeyToDeploy.get(deployKey));
Optional<SingularityRequestWithState> maybeRequestWithState = requestManager.getRequest(pendingDeploy.getDeployMarker().getRequestId());
if (!(maybeRequestWithState.isPresent() && maybeRequestWithState.get().getState() == RequestState.FINISHED) && !(configuration.isAllowDeployOfPausedRequests() && maybeRequestWithState.isPresent() && maybeRequestWithState.get().getState() == RequestState.PAUSED) && !SingularityRequestWithState.isActive(maybeRequestWithState)) {
LOG.warn("Deploy {} request was {}, removing deploy", pendingDeploy, SingularityRequestWithState.getRequestState(maybeRequestWithState));
if (shouldCancelLoadBalancer(pendingDeploy)) {
cancelLoadBalancer(pendingDeploy, SingularityDeployFailure.deployRemoved());
}
failPendingDeployDueToState(pendingDeploy, maybeRequestWithState, deploy);
return;
}
final SingularityDeployMarker pendingDeployMarker = pendingDeploy.getDeployMarker();
final Optional<SingularityDeployMarker> cancelRequest = findCancel(cancelDeploys, pendingDeployMarker);
final Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest = findUpdateRequest(updateRequests, pendingDeploy);
final SingularityRequestWithState requestWithState = maybeRequestWithState.get();
final SingularityRequest request = pendingDeploy.getUpdatedRequest().or(requestWithState.getRequest());
final List<SingularityTaskId> requestTasks = taskManager.getTaskIdsForRequest(request.getId());
final List<SingularityTaskId> activeTasks = taskManager.filterActiveTaskIds(requestTasks);
final List<SingularityTaskId> inactiveDeployMatchingTasks = new ArrayList<>(requestTasks.size());
for (SingularityTaskId taskId : requestTasks) {
if (taskId.getDeployId().equals(pendingDeployMarker.getDeployId()) && !activeTasks.contains(taskId)) {
inactiveDeployMatchingTasks.add(taskId);
}
}
final List<SingularityTaskId> deployMatchingTasks = new ArrayList<>(activeTasks.size());
final List<SingularityTaskId> allOtherMatchingTasks = new ArrayList<>(activeTasks.size());
for (SingularityTaskId taskId : activeTasks) {
if (taskId.getDeployId().equals(pendingDeployMarker.getDeployId())) {
deployMatchingTasks.add(taskId);
} else {
allOtherMatchingTasks.add(taskId);
}
}
SingularityDeployResult deployResult = getDeployResult(request, requestWithState.getState(), cancelRequest, pendingDeploy, updatePendingDeployRequest, deploy, deployMatchingTasks, allOtherMatchingTasks, inactiveDeployMatchingTasks);
LOG.info("Deploy {} had result {} after {}", pendingDeployMarker, deployResult, JavaUtils.durationFromMillis(System.currentTimeMillis() - pendingDeployMarker.getTimestamp()));
if (deployResult.getDeployState() == DeployState.SUCCEEDED) {
if (saveNewDeployState(pendingDeployMarker, Optional.of(pendingDeployMarker))) {
if (!(request.getRequestType() == RequestType.RUN_ONCE)) {
deleteObsoletePendingTasks(pendingDeploy);
}
finishDeploy(requestWithState, deploy, pendingDeploy, allOtherMatchingTasks, deployResult);
return;
} else {
LOG.warn("Failing deploy {} because it failed to save deploy state", pendingDeployMarker);
deployResult = new SingularityDeployResult(DeployState.FAILED_INTERNAL_STATE, Optional.of(String.format("Deploy had state %s but failed to persist it correctly", deployResult.getDeployState())), deployResult.getLbUpdate(), SingularityDeployFailure.failedToSave(), deployResult.getTimestamp());
}
} else if (!deployResult.getDeployState().isDeployFinished()) {
return;
}
// success case is handled, handle failure cases:
saveNewDeployState(pendingDeployMarker, Optional.<SingularityDeployMarker>absent());
finishDeploy(requestWithState, deploy, pendingDeploy, deployMatchingTasks, deployResult);
}
use of com.hubspot.singularity.SingularityDeployMarker in project Singularity by HubSpot.
the class DeployResource method deploy.
public SingularityRequestParent deploy(SingularityDeployRequest deployRequest, SingularityUser user) {
validator.checkActionEnabled(SingularityAction.DEPLOY);
SingularityDeploy deploy = deployRequest.getDeploy();
checkNotNullBadRequest(deploy, "DeployRequest must have a deploy object");
final Optional<String> deployUser = user.getEmail();
final String requestId = checkNotNullBadRequest(deploy.getRequestId(), "DeployRequest must have a non-null requestId");
SingularityRequestWithState requestWithState = fetchRequestWithState(requestId, user);
authorizationHelper.checkForAuthorization(requestWithState.getRequest(), user, SingularityAuthorizationScope.WRITE);
SingularityRequest request = requestWithState.getRequest();
final Optional<SingularityRequest> updatedValidatedRequest;
if (deployRequest.getUpdatedRequest().isPresent()) {
authorizationHelper.checkForAuthorizedChanges(deployRequest.getUpdatedRequest().get(), requestWithState.getRequest(), user);
updatedValidatedRequest = Optional.of(validator.checkSingularityRequest(deployRequest.getUpdatedRequest().get(), Optional.of(requestWithState.getRequest()), Optional.<SingularityDeploy>absent(), Optional.of(deploy)));
} else {
updatedValidatedRequest = Optional.absent();
}
if (updatedValidatedRequest.isPresent()) {
request = updatedValidatedRequest.get();
}
validator.checkScale(request, Optional.of(taskManager.getActiveTaskIdsForRequest(request.getId()).size()));
if (!deployRequest.isUnpauseOnSuccessfulDeploy() && !configuration.isAllowDeployOfPausedRequests()) {
checkConflict(requestWithState.getState() != RequestState.PAUSED, "Request %s is paused. Unable to deploy (it must be manually unpaused first)", requestWithState.getRequest().getId());
}
deploy = validator.checkDeploy(request, deploy, taskManager.getActiveTaskIdsForRequest(requestId), taskManager.getPendingTaskIdsForRequest(requestId));
final long now = System.currentTimeMillis();
SingularityDeployMarker deployMarker = new SingularityDeployMarker(requestId, deploy.getId(), now, deployUser, deployRequest.getMessage());
Optional<SingularityDeployProgress> deployProgress = Optional.absent();
if (request.isLongRunning()) {
deployProgress = Optional.of(new SingularityDeployProgress(Math.min(deploy.getDeployInstanceCountPerStep().or(request.getInstancesSafe()), request.getInstancesSafe()), 0, deploy.getDeployInstanceCountPerStep().or(request.getInstancesSafe()), deploy.getDeployStepWaitTimeMs().or(configuration.getDefaultDeployStepWaitTimeMs()), false, deploy.getAutoAdvanceDeploySteps().or(true), Collections.emptySet(), System.currentTimeMillis()));
}
SingularityPendingDeploy pendingDeployObj = new SingularityPendingDeploy(deployMarker, Optional.<SingularityLoadBalancerUpdate>absent(), DeployState.WAITING, deployProgress, updatedValidatedRequest);
boolean deployToUnpause = false;
if (requestWithState.getState() == RequestState.PAUSED && deployRequest.isUnpauseOnSuccessfulDeploy()) {
deployToUnpause = true;
requestManager.deployToUnpause(request, now, deployUser, deployRequest.getMessage());
}
boolean deployAlreadyInProgress = deployManager.createPendingDeploy(pendingDeployObj) == SingularityCreateResult.EXISTED;
if (deployAlreadyInProgress && deployToUnpause) {
requestManager.pause(request, now, deployUser, Optional.absent());
}
checkConflict(!deployAlreadyInProgress, "Pending deploy already in progress for %s - cancel it or wait for it to complete (%s)", requestId, deployManager.getPendingDeploy(requestId).orNull());
deployManager.saveDeploy(request, deployMarker, deploy);
if (request.isDeployable() && !(requestWithState.getState() == RequestState.PAUSED && configuration.isAllowDeployOfPausedRequests())) {
requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, deployMarker.getDeployId(), now, deployUser, PendingType.NEW_DEPLOY, deployRequest.getDeploy().getSkipHealthchecksOnDeploy(), deployRequest.getMessage()));
}
return fillEntireRequest(requestWithState, Optional.of(request));
}
use of com.hubspot.singularity.SingularityDeployMarker in project Singularity by HubSpot.
the class SingularitySchedulerTestBase method initDeploy.
protected SingularityDeploy initDeploy(SingularityDeployBuilder builder, long timestamp) {
SingularityDeployMarker marker = new SingularityDeployMarker(requestId, builder.getId(), timestamp, Optional.<String>absent(), Optional.<String>absent());
builder.setCommand(Optional.of("sleep 100"));
SingularityDeploy deploy = builder.build();
deployManager.saveDeploy(request, marker, deploy);
startDeploy(marker, timestamp);
return deploy;
}
Aggregations