use of com.hubspot.singularity.SingularityRequestDeployState in project Singularity by HubSpot.
the class DeployManager method saveDeploy.
public SingularityCreateResult saveDeploy(SingularityRequest request, SingularityDeployMarker deployMarker, SingularityDeploy deploy) {
final SingularityCreateResult deploySaveResult = create(getDeployDataPath(deploy.getRequestId(), deploy.getId()), deploy, deployTranscoder);
if (deploySaveResult == SingularityCreateResult.EXISTED) {
LOG.info("Deploy object for {} already existed (new marker: {})", deploy, deployMarker);
}
singularityEventListener.deployHistoryEvent(new SingularityDeployUpdate(deployMarker, Optional.of(deploy), DeployEventType.STARTING, Optional.<SingularityDeployResult>absent()));
create(getDeployMarkerPath(deploy.getRequestId(), deploy.getId()), deployMarker, deployMarkerTranscoder);
final Optional<SingularityRequestDeployState> currentState = getRequestDeployState(deploy.getRequestId());
Optional<SingularityDeployMarker> activeDeploy = Optional.absent();
Optional<SingularityDeployMarker> pendingDeploy = Optional.absent();
if (request.isDeployable()) {
if (currentState.isPresent()) {
activeDeploy = currentState.get().getActiveDeploy();
}
pendingDeploy = Optional.of(deployMarker);
} else {
activeDeploy = Optional.of(deployMarker);
}
final SingularityRequestDeployState newState = new SingularityRequestDeployState(deploy.getRequestId(), activeDeploy, pendingDeploy);
return saveNewRequestDeployState(newState);
}
use of com.hubspot.singularity.SingularityRequestDeployState in project Singularity by HubSpot.
the class SingularityDeployHistoryPersister method runActionOnPoll.
@Override
public void runActionOnPoll() {
LOG.info("Checking inactive deploys for deploy history persistance");
final long start = System.currentTimeMillis();
final List<SingularityDeployKey> allDeployIds = deployManager.getAllDeployIds();
final Map<String, SingularityRequestDeployState> byRequestId = deployManager.getAllRequestDeployStatesByRequestId();
final TreeMultimap<String, SingularityDeployHistory> deployHistoryByRequestId = TreeMultimap.create();
int numTotal = 0;
int numTransferred = 0;
for (SingularityDeployKey deployKey : allDeployIds) {
SingularityRequestDeployState deployState = byRequestId.get(deployKey.getRequestId());
if (!shouldTransferDeploy(deployState, deployKey)) {
continue;
}
Optional<SingularityDeployHistory> deployHistory = deployManager.getDeployHistory(deployKey.getRequestId(), deployKey.getDeployId(), true);
if (deployHistory.isPresent()) {
deployHistoryByRequestId.put(deployKey.getRequestId(), deployHistory.get());
} else {
LOG.info("Deploy history for key {} not found", deployKey);
}
}
for (Collection<SingularityDeployHistory> deployHistoryForRequest : deployHistoryByRequestId.asMap().values()) {
int i = 0;
for (SingularityDeployHistory deployHistory : deployHistoryForRequest) {
if (moveToHistoryOrCheckForPurge(deployHistory, i++)) {
numTransferred++;
}
numTotal++;
}
}
LOG.info("Transferred {} out of {} deploys in {}", numTransferred, numTotal, JavaUtils.duration(start));
}
use of com.hubspot.singularity.SingularityRequestDeployState in project Singularity by HubSpot.
the class SingularityDeployChecker method saveNewDeployState.
private boolean saveNewDeployState(SingularityDeployMarker pendingDeployMarker, Optional<SingularityDeployMarker> newActiveDeploy) {
Optional<SingularityRequestDeployState> deployState = deployManager.getRequestDeployState(pendingDeployMarker.getRequestId());
if (!deployState.isPresent()) {
LOG.error("Expected deploy state for deploy marker: {} but didn't find it", pendingDeployMarker);
return false;
}
deployManager.saveNewRequestDeployState(new SingularityRequestDeployState(deployState.get().getRequestId(), newActiveDeploy.or(deployState.get().getActiveDeploy()), Optional.absent()));
return true;
}
use of com.hubspot.singularity.SingularityRequestDeployState in project Singularity by HubSpot.
the class SingularityDeployChecker method finishDeploy.
private void finishDeploy(SingularityRequestWithState requestWithState, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, Iterable<SingularityTaskId> tasksToKill, SingularityDeployResult deployResult) {
SingularityRequest request = requestWithState.getRequest();
if (!request.isOneOff() && !(request.getRequestType() == RequestType.RUN_ONCE)) {
cleanupTasks(pendingDeploy, request, deployResult, tasksToKill);
}
if (deploy.isPresent() && deploy.get().getRunImmediately().isPresent()) {
String requestId = deploy.get().getRequestId();
String deployId = deploy.get().getId();
SingularityRunNowRequest runNowRequest = deploy.get().getRunImmediately().get();
List<SingularityTaskId> activeTasks = taskManager.getActiveTaskIdsForRequest(requestId);
List<SingularityPendingTaskId> pendingTasks = taskManager.getPendingTaskIdsForRequest(requestId);
SingularityPendingRequestBuilder builder = new SingularityPendingRequestBuilder().setRequestId(requestId).setDeployId(deployId).setTimestamp(deployResult.getTimestamp()).setUser(pendingDeploy.getDeployMarker().getUser()).setCmdLineArgsList(runNowRequest.getCommandLineArgs()).setRunId(runNowRequest.getRunId().or(Optional.of(UUID.randomUUID().toString()))).setSkipHealthchecks(runNowRequest.getSkipHealthchecks().or(request.getSkipHealthchecks())).setMessage(runNowRequest.getMessage().or(pendingDeploy.getDeployMarker().getMessage())).setResources(runNowRequest.getResources()).setRunAsUserOverride(runNowRequest.getRunAsUserOverride()).setEnvOverrides(runNowRequest.getEnvOverrides()).setExtraArtifacts(runNowRequest.getExtraArtifacts()).setRunAt(runNowRequest.getRunAt());
PendingType pendingType = null;
if (request.isScheduled()) {
if (activeTasks.isEmpty()) {
pendingType = PendingType.IMMEDIATE;
} else {
// Don't run scheduled task over a running task. Will be picked up on the next run.
pendingType = PendingType.NEW_DEPLOY;
}
} else if (!request.isLongRunning()) {
if (request.getInstances().isPresent() && (activeTasks.size() + pendingTasks.size() < request.getInstances().get())) {
pendingType = PendingType.ONEOFF;
} else {
// Don't run one-off / on-demand task when already at instance count cap
pendingType = PendingType.NEW_DEPLOY;
}
}
if (pendingType != null) {
builder.setPendingType(canceledOr(deployResult.getDeployState(), pendingType));
requestManager.addToPendingQueue(builder.build());
} else {
LOG.warn("Could not determine pending type for deploy {}.", deployId);
}
} else if (!request.isDeployable() && !request.isOneOff()) {
PendingType pendingType = canceledOr(deployResult.getDeployState(), PendingType.NEW_DEPLOY);
requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), pendingType, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.absent(), pendingDeploy.getDeployMarker().getMessage()));
}
if (deployResult.getDeployState() == DeployState.SUCCEEDED) {
if (request.isDeployable() && !request.isOneOff()) {
// remove the lock on bounces in case we deployed during a bounce
requestManager.markBounceComplete(request.getId());
}
if (requestWithState.getState() == RequestState.FINISHED) {
// A FINISHED request is moved to ACTIVE state so we can reevaluate the schedule
requestManager.activate(request, RequestHistoryType.UPDATED, System.currentTimeMillis(), deploy.isPresent() ? deploy.get().getUser() : Optional.absent(), Optional.absent());
}
}
deployManager.saveDeployResult(pendingDeploy.getDeployMarker(), deploy, deployResult);
if (request.isDeployable() && (deployResult.getDeployState() == DeployState.CANCELED || deployResult.getDeployState() == DeployState.FAILED || deployResult.getDeployState() == DeployState.OVERDUE)) {
Optional<SingularityRequestDeployState> maybeRequestDeployState = deployManager.getRequestDeployState(request.getId());
if (maybeRequestDeployState.isPresent() && maybeRequestDeployState.get().getActiveDeploy().isPresent() && !(requestWithState.getState() == RequestState.PAUSED || requestWithState.getState() == RequestState.DEPLOYING_TO_UNPAUSE)) {
requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), maybeRequestDeployState.get().getActiveDeploy().get().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), deployResult.getDeployState() == DeployState.CANCELED ? PendingType.DEPLOY_CANCELLED : PendingType.DEPLOY_FAILED, request.getSkipHealthchecks(), pendingDeploy.getDeployMarker().getMessage()));
}
}
if (request.isDeployable() && deployResult.getDeployState() == DeployState.SUCCEEDED && pendingDeploy.getDeployProgress().isPresent() && requestWithState.getState() != RequestState.PAUSED) {
if (pendingDeploy.getDeployProgress().get().getTargetActiveInstances() != request.getInstancesSafe()) {
requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), PendingType.UPDATED_REQUEST, request.getSkipHealthchecks(), pendingDeploy.getDeployMarker().getMessage()));
}
}
if (requestWithState.getState() == RequestState.DEPLOYING_TO_UNPAUSE) {
if (deployResult.getDeployState() == DeployState.SUCCEEDED) {
requestManager.activate(request, RequestHistoryType.DEPLOYED_TO_UNPAUSE, deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), Optional.<String>absent());
requestManager.deleteExpiringObject(SingularityExpiringPause.class, request.getId());
} else {
requestManager.pause(request, deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), Optional.<String>absent());
}
}
if (pendingDeploy.getUpdatedRequest().isPresent() && deployResult.getDeployState() == DeployState.SUCCEEDED) {
requestManager.update(pendingDeploy.getUpdatedRequest().get(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), Optional.<String>absent());
requestManager.deleteExpiringObject(SingularityExpiringScale.class, request.getId());
}
removePendingDeploy(pendingDeploy);
}
use of com.hubspot.singularity.SingularityRequestDeployState in project Singularity by HubSpot.
the class SingularityScheduler method handleCompletedTaskWithStatistics.
private Optional<PendingType> handleCompletedTaskWithStatistics(Optional<SingularityTask> task, SingularityTaskId taskId, long timestamp, ExtendedTaskState state, SingularityDeployStatistics deployStatistics, SingularityCreateResult taskHistoryUpdateCreateResult, Protos.TaskStatus status) {
final Optional<SingularityRequestWithState> maybeRequestWithState = requestManager.getRequest(taskId.getRequestId());
final Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(taskId.getRequestId());
if (!isRequestActive(maybeRequestWithState)) {
LOG.warn("Not scheduling a new task, {} is {}", taskId.getRequestId(), SingularityRequestWithState.getRequestState(maybeRequestWithState));
return Optional.absent();
}
RequestState requestState = maybeRequestWithState.get().getState();
final SingularityRequest request = maybePendingDeploy.isPresent() ? maybePendingDeploy.get().getUpdatedRequest().or(maybeRequestWithState.get().getRequest()) : maybeRequestWithState.get().getRequest();
final Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(request.getId());
if (!isDeployInUse(requestDeployState, taskId.getDeployId(), true)) {
LOG.debug("Task {} completed, but it didn't match active deploy state {} - ignoring", taskId.getId(), requestDeployState);
return Optional.absent();
}
if (taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED && requestState != RequestState.SYSTEM_COOLDOWN) {
mailer.queueTaskCompletedMail(task, taskId, request, state);
} else if (requestState == RequestState.SYSTEM_COOLDOWN) {
LOG.debug("Not sending a task completed email because task {} is in SYSTEM_COOLDOWN", taskId);
} else {
LOG.debug("Not sending a task completed email for task {} because Singularity already processed this update", taskId);
}
if (!status.hasReason() || !status.getReason().equals(Reason.REASON_INVALID_OFFERS)) {
if (!state.isSuccess() && taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED && cooldown.shouldEnterCooldown(request, taskId, requestState, deployStatistics, timestamp)) {
LOG.info("Request {} is entering cooldown due to task {}", request.getId(), taskId);
requestState = RequestState.SYSTEM_COOLDOWN;
requestManager.cooldown(request, System.currentTimeMillis());
mailer.sendRequestInCooldownMail(request);
}
} else {
LOG.debug("Not triggering cooldown due to TASK_LOST from invalid offers for request {}", request.getId());
}
PendingType pendingType = PendingType.TASK_DONE;
Optional<List<String>> cmdLineArgsList = Optional.absent();
if (!state.isSuccess() && shouldRetryImmediately(request, deployStatistics, task)) {
LOG.debug("Retrying {} because {}", request.getId(), state);
pendingType = PendingType.RETRY;
if (task.isPresent()) {
cmdLineArgsList = task.get().getTaskRequest().getPendingTask().getCmdLineArgsList();
}
} else if (!request.isAlwaysRunning()) {
return Optional.absent();
}
if (state.isSuccess() && requestState == RequestState.SYSTEM_COOLDOWN) {
// TODO send not cooldown anymore email
LOG.info("Request {} succeeded a task, removing from cooldown", request.getId());
requestManager.exitCooldown(request, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent());
}
SingularityPendingRequest pendingRequest = new SingularityPendingRequest(request.getId(), requestDeployState.get().getActiveDeploy().get().getDeployId(), System.currentTimeMillis(), Optional.absent(), pendingType, cmdLineArgsList, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent());
requestManager.addToPendingQueue(pendingRequest);
return Optional.of(pendingType);
}
Aggregations