Examples with SingularityTaskId com.hubspot.singularity.SingularityTaskId used on opensource projects

Example 36 with SingularityTaskId

use of com.hubspot.singularity.SingularityTaskId in project Singularity by HubSpot.

the class SingularityMesosOfferSchedulerTest method itAccountsForMaxHistoricalTaskUsage.

@Test
public void itAccountsForMaxHistoricalTaskUsage() {
    try {
        configuration.getMesosConfiguration().setScoringStrategy(SingularityUsageScoringStrategy.PROBABLE_MAX_USAGE);
        initRequest();
        double cpuReserved = 2;
        double memMbReserved = 1000;
        initFirstDeployWithResources(cpuReserved, memMbReserved);
        saveAndSchedule(requestManager.getRequest(requestId).get().getRequest().toBuilder().setInstances(Optional.of(1)));
        resourceOffers(3);
        SingularityTaskId taskId = taskManager.getActiveTaskIds().get(0);
        String t1 = taskId.getId();
        // 2 cpus used
        MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 10, TimeUnit.MILLISECONDS.toSeconds(taskId.getStartedAt()) + 5, 1000);
        mesosClient.setSlaveResourceUsage("host1", Collections.singletonList(t1u1));
        usagePoller.runActionOnPoll();
        // 1 cpus used
        MesosTaskMonitorObject t1u2 = getTaskMonitor(t1, 11, TimeUnit.MILLISECONDS.toSeconds(taskId.getStartedAt()) + 6, 1000);
        mesosClient.setSlaveResourceUsage("host1", Collections.singletonList(t1u2));
        usagePoller.runActionOnPoll();
        Map<ResourceUsageType, Number> longRunningTasksUsage = new HashMap<>();
        longRunningTasksUsage.put(ResourceUsageType.CPU_USED, 0.1);
        longRunningTasksUsage.put(ResourceUsageType.MEMORY_BYTES_USED, 0.1);
        longRunningTasksUsage.put(ResourceUsageType.DISK_BYTES_USED, 0.1);
        SingularitySlaveUsage smallUsage = new SingularitySlaveUsage(0.1, 0.1, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), longRunningTasksUsage, 1, System.currentTimeMillis(), 1, 30000, 10, 0, 0, 0, 0, 107374182);
        usageManager.saveSpecificSlaveUsageAndSetCurrent("host1", smallUsage);
        usageManager.saveSpecificSlaveUsageAndSetCurrent("host2", smallUsage);
        usageManager.saveSpecificSlaveUsageAndSetCurrent("host3", smallUsage);
        requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(3), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()), SingularityUser.DEFAULT_USER);
        Assert.assertEquals(3.0, usageManager.getRequestUtilizations().get(requestId).getCpuUsed(), 0.001);
        Offer host2Offer = createOffer(6, 30000, 107374182, "host2", "host2");
        slaveAndRackManager.checkOffer(host2Offer);
        Offer host3Offer = createOffer(6, 30000, 107374182, "host3", "host3");
        slaveAndRackManager.checkOffer(host3Offer);
        Collection<SingularityOfferHolder> offerHolders = offerScheduler.checkOffers(Arrays.asList(host2Offer, host3Offer));
        Assert.assertEquals(2, offerHolders.size());
        // A single offer should only ever get a single task even though both have room for both tasks here. Adding a task should reduce the score for the next check
        for (SingularityOfferHolder offerHolder : offerHolders) {
            Assert.assertEquals(1, offerHolder.getAcceptedTasks().size());
        }
    } finally {
        configuration.getMesosConfiguration().setScoringStrategy(SingularityUsageScoringStrategy.SPREAD_TASK_USAGE);
    }
}

Example 37 with SingularityTaskId

use of com.hubspot.singularity.SingularityTaskId in project Singularity by HubSpot.

the class SingularityMesosTaskBuilder method buildTask.

public SingularityMesosTaskHolder buildTask(SingularityOfferHolder offerHolder, List<Resource> availableResources, SingularityTaskRequest taskRequest, Resources desiredTaskResources, Resources desiredExecutorResources) {
    final String sanitizedRackId = offerHolder.getSanitizedRackId();
    final String sanitizedHost = offerHolder.getSanitizedHost();
    final SingularityTaskId taskId = new SingularityTaskId(taskRequest.getPendingTask().getPendingTaskId().getRequestId(), taskRequest.getDeploy().getId(), System.currentTimeMillis(), taskRequest.getPendingTask().getPendingTaskId().getInstanceNo(), sanitizedHost, sanitizedRackId);
    final TaskInfo.Builder bldr = TaskInfo.newBuilder().setTaskId(TaskID.newBuilder().setValue(taskId.toString()));
    Optional<long[]> ports = Optional.absent();
    Optional<Resource> portsResource = Optional.absent();
    final Optional<SingularityContainerInfo> containerInfo = taskRequest.getDeploy().getContainerInfo();
    if (desiredTaskResources.getNumPorts() > 0 || hasLiteralPortMapping(containerInfo)) {
        List<Long> requestedPorts = new ArrayList<>();
        if (hasLiteralPortMapping(containerInfo)) {
            requestedPorts.addAll(containerInfo.get().getDocker().get().getLiteralHostPorts());
        }
        portsResource = Optional.of(MesosUtils.getPortsResource(desiredTaskResources.getNumPorts(), availableResources, requestedPorts));
        ports = Optional.of(MesosUtils.getPorts(portsResource.get(), desiredTaskResources.getNumPorts()));
    }
    if (containerInfo.isPresent()) {
        prepareContainerInfo(offerHolder, taskId, bldr, containerInfo.get(), ports);
    }
    if (taskRequest.getDeploy().getCustomExecutorCmd().isPresent()) {
        prepareCustomExecutor(bldr, taskId, taskRequest, offerHolder, ports, desiredExecutorResources);
    } else {
        prepareCommand(bldr, taskId, taskRequest, offerHolder, ports);
    }
    if (portsResource.isPresent()) {
        bldr.addResources(portsResource.get());
    }
    Optional<String> requiredRole = taskRequest.getRequest().getRequiredRole();
    bldr.addResources(MesosUtils.getCpuResource(desiredTaskResources.getCpus(), requiredRole));
    bldr.addResources(MesosUtils.getMemoryResource(desiredTaskResources.getMemoryMb(), requiredRole));
    bldr.addResources(MesosUtils.getDiskResource(desiredTaskResources.getDiskMb(), requiredRole));
    bldr.setAgentId(offerHolder.getOffers().get(0).getAgentId());
    bldr.setName(taskRequest.getRequest().getId());
    final Builder labelsBuilder = Labels.newBuilder();
    // apply request-specific labels, if any
    if (taskRequest.getDeploy().getMesosLabels().isPresent() && !taskRequest.getDeploy().getMesosLabels().get().isEmpty()) {
        for (SingularityMesosTaskLabel label : taskRequest.getDeploy().getMesosLabels().get()) {
            org.apache.mesos.v1.Protos.Label.Builder labelBuilder = Label.newBuilder();
            labelBuilder.setKey(label.getKey());
            if ((label.getValue().isPresent())) {
                labelBuilder.setValue(label.getValue().get());
            }
            labelsBuilder.addLabels(labelBuilder.build());
        }
    }
    // apply task-specific labels, if any
    final int taskInstanceNo = taskRequest.getPendingTask().getPendingTaskId().getInstanceNo();
    if (taskRequest.getDeploy().getMesosTaskLabels().isPresent() && taskRequest.getDeploy().getMesosTaskLabels().get().containsKey(taskInstanceNo) && !taskRequest.getDeploy().getMesosTaskLabels().get().get(taskInstanceNo).isEmpty()) {
        for (SingularityMesosTaskLabel label : taskRequest.getDeploy().getMesosTaskLabels().get().get(taskInstanceNo)) {
            org.apache.mesos.v1.Protos.Label.Builder labelBuilder = Label.newBuilder();
            labelBuilder.setKey(label.getKey());
            if ((label.getValue().isPresent())) {
                labelBuilder.setValue(label.getValue().get());
            }
            labelsBuilder.addLabels(labelBuilder.build());
        }
    }
    bldr.setLabels(labelsBuilder);
    TaskInfo task = bldr.build();
    return new SingularityMesosTaskHolder(new SingularityTask(taskRequest, taskId, offerHolder.getOffers().stream().map((o) -> mesosProtosUtils.offerFromProtos(o)).collect(Collectors.toList()), mesosProtosUtils.taskFromProtos(task), Optional.of(offerHolder.getRackId())), task);
}

Example 38 with SingularityTaskId

use of com.hubspot.singularity.SingularityTaskId in project Singularity by HubSpot.

the class SingularityDeployChecker method getDeployResult.

private SingularityDeployResult getDeployResult(final SingularityRequest request, final RequestState requestState, final Optional<SingularityDeployMarker> cancelRequest, final SingularityPendingDeploy pendingDeploy, final Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, final Optional<SingularityDeploy> deploy, final Collection<SingularityTaskId> deployActiveTasks, final Collection<SingularityTaskId> otherActiveTasks, final Collection<SingularityTaskId> inactiveDeployMatchingTasks) {
    if (!request.isDeployable() || (configuration.isAllowDeployOfPausedRequests() && requestState == RequestState.PAUSED)) {
        LOG.info("Succeeding a deploy {} because the request {} was not deployable", pendingDeploy, request);
        return new SingularityDeployResult(DeployState.SUCCEEDED, "Request not deployable");
    }
    if (!pendingDeploy.getDeployProgress().isPresent()) {
        return new SingularityDeployResult(DeployState.FAILED, "No deploy progress data present in Zookeeper. Please reattempt your deploy");
    }
    Set<SingularityTaskId> newInactiveDeployTasks = getNewInactiveDeployTasks(pendingDeploy, inactiveDeployMatchingTasks);
    if (!newInactiveDeployTasks.isEmpty()) {
        if (canRetryTasks(deploy, inactiveDeployMatchingTasks)) {
            SingularityDeployProgress newProgress = pendingDeploy.getDeployProgress().get().withFailedTasks(new HashSet<>(inactiveDeployMatchingTasks));
            updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), DeployState.WAITING, Optional.of(newProgress));
            requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean>absent(), pendingDeploy.getDeployMarker().getMessage()));
            return new SingularityDeployResult(DeployState.WAITING);
        }
        if (request.isLoadBalanced() && shouldCancelLoadBalancer(pendingDeploy)) {
            LOG.info("Attempting to cancel pending load balancer request, failing deploy {} regardless", pendingDeploy);
            sendCancelToLoadBalancer(pendingDeploy);
        }
        int maxRetries = deploy.get().getMaxTaskRetries().or(configuration.getDefaultDeployMaxTaskRetries());
        return getDeployResultWithFailures(request, deploy, pendingDeploy, DeployState.FAILED, String.format("%s task(s) for this deploy failed", inactiveDeployMatchingTasks.size() - maxRetries), inactiveDeployMatchingTasks);
    }
    return checkDeployProgress(request, cancelRequest, pendingDeploy, updatePendingDeployRequest, deploy, deployActiveTasks, otherActiveTasks);
}

Example 39 with SingularityTaskId

use of com.hubspot.singularity.SingularityTaskId in project Singularity by HubSpot.

the class SingularityDeployChecker method checkDeployProgress.

private SingularityDeployResult checkDeployProgress(final SingularityRequest request, final Optional<SingularityDeployMarker> cancelRequest, final SingularityPendingDeploy pendingDeploy, final Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, final Optional<SingularityDeploy> deploy, final Collection<SingularityTaskId> deployActiveTasks, final Collection<SingularityTaskId> otherActiveTasks) {
    SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get();
    if (cancelRequest.isPresent()) {
        LOG.info("Canceling a deploy {} due to cancel request {}", pendingDeploy, cancelRequest.get());
        String userMessage = cancelRequest.get().getUser().isPresent() ? String.format(" by %s", cancelRequest.get().getUser().get()) : "";
        return new SingularityDeployResult(DeployState.CANCELED, Optional.of(String.format("Canceled due to request%s at %s", userMessage, cancelRequest.get().getTimestamp())), pendingDeploy.getLastLoadBalancerUpdate(), Collections.<SingularityDeployFailure>emptyList(), System.currentTimeMillis());
    }
    if (deployProgress.isStepComplete()) {
        return checkCanMoveToNextDeployStep(request, deploy, pendingDeploy, updatePendingDeployRequest);
    }
    final boolean isDeployOverdue = isDeployOverdue(pendingDeploy, deploy);
    if (deployActiveTasks.size() < deployProgress.getTargetActiveInstances()) {
        maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest);
        return checkOverdue(request, deploy, pendingDeploy, deployActiveTasks, isDeployOverdue);
    }
    if (shouldCheckLbState(pendingDeploy)) {
        final SingularityLoadBalancerUpdate lbUpdate = lbClient.getState(getLoadBalancerRequestId(pendingDeploy));
        return processLbState(request, deploy, pendingDeploy, updatePendingDeployRequest, deployActiveTasks, otherActiveTasks, tasksToShutDown(deployProgress, otherActiveTasks, request), lbUpdate);
    }
    if (isDeployOverdue && request.isLoadBalanced() && shouldCancelLoadBalancer(pendingDeploy)) {
        return cancelLoadBalancer(pendingDeploy, getDeployFailures(request, deploy, pendingDeploy, DeployState.OVERDUE, deployActiveTasks));
    }
    if (isWaitingForCurrentLbRequest(pendingDeploy)) {
        return new SingularityDeployResult(DeployState.WAITING, Optional.of("Waiting on load balancer API"), pendingDeploy.getLastLoadBalancerUpdate());
    }
    final DeployHealth deployHealth = deployHealthHelper.getDeployHealth(request, deploy, deployActiveTasks, true);
    switch(deployHealth) {
        case WAITING:
            maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest);
            return checkOverdue(request, deploy, pendingDeploy, deployActiveTasks, isDeployOverdue);
        case HEALTHY:
            if (!request.isLoadBalanced()) {
                return markStepFinished(pendingDeploy, deploy, deployActiveTasks, otherActiveTasks, request, updatePendingDeployRequest);
            }
            if (updatePendingDeployRequest.isPresent() && updatePendingDeployRequest.get().getTargetActiveInstances() != deployProgress.getTargetActiveInstances()) {
                maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest);
                return new SingularityDeployResult(DeployState.WAITING);
            }
            if (configuration.getLoadBalancerUri() == null) {
                LOG.warn("Deploy {} required a load balancer URI but it wasn't set", pendingDeploy);
                return new SingularityDeployResult(DeployState.FAILED, Optional.of("No valid load balancer URI was present"), Optional.<SingularityLoadBalancerUpdate>absent(), Collections.<SingularityDeployFailure>emptyList(), System.currentTimeMillis());
            }
            for (SingularityTaskId activeTaskId : deployActiveTasks) {
                taskManager.markHealthchecksFinished(activeTaskId);
                taskManager.clearStartupHealthchecks(activeTaskId);
            }
            return enqueueAndProcessLbRequest(request, deploy, pendingDeploy, updatePendingDeployRequest, deployActiveTasks, otherActiveTasks);
        case UNHEALTHY:
        default:
            for (SingularityTaskId activeTaskId : deployActiveTasks) {
                taskManager.markHealthchecksFinished(activeTaskId);
                taskManager.clearStartupHealthchecks(activeTaskId);
            }
            return getDeployResultWithFailures(request, deploy, pendingDeploy, DeployState.FAILED, "Not all tasks for deploy were healthy", deployActiveTasks);
    }
}

Example 40 with SingularityTaskId

use of com.hubspot.singularity.SingularityTaskId in project Singularity by HubSpot.

the class SingularityDeployChecker method finishDeploy.

private void finishDeploy(SingularityRequestWithState requestWithState, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, Iterable<SingularityTaskId> tasksToKill, SingularityDeployResult deployResult) {
    SingularityRequest request = requestWithState.getRequest();
    if (!request.isOneOff() && !(request.getRequestType() == RequestType.RUN_ONCE)) {
        cleanupTasks(pendingDeploy, request, deployResult, tasksToKill);
    }
    if (deploy.isPresent() && deploy.get().getRunImmediately().isPresent()) {
        String requestId = deploy.get().getRequestId();
        String deployId = deploy.get().getId();
        SingularityRunNowRequest runNowRequest = deploy.get().getRunImmediately().get();
        List<SingularityTaskId> activeTasks = taskManager.getActiveTaskIdsForRequest(requestId);
        List<SingularityPendingTaskId> pendingTasks = taskManager.getPendingTaskIdsForRequest(requestId);
        SingularityPendingRequestBuilder builder = new SingularityPendingRequestBuilder().setRequestId(requestId).setDeployId(deployId).setTimestamp(deployResult.getTimestamp()).setUser(pendingDeploy.getDeployMarker().getUser()).setCmdLineArgsList(runNowRequest.getCommandLineArgs()).setRunId(runNowRequest.getRunId().or(Optional.of(UUID.randomUUID().toString()))).setSkipHealthchecks(runNowRequest.getSkipHealthchecks().or(request.getSkipHealthchecks())).setMessage(runNowRequest.getMessage().or(pendingDeploy.getDeployMarker().getMessage())).setResources(runNowRequest.getResources()).setRunAsUserOverride(runNowRequest.getRunAsUserOverride()).setEnvOverrides(runNowRequest.getEnvOverrides()).setExtraArtifacts(runNowRequest.getExtraArtifacts()).setRunAt(runNowRequest.getRunAt());
        PendingType pendingType = null;
        if (request.isScheduled()) {
            if (activeTasks.isEmpty()) {
                pendingType = PendingType.IMMEDIATE;
            } else {
                // Don't run scheduled task over a running task. Will be picked up on the next run.
                pendingType = PendingType.NEW_DEPLOY;
            }
        } else if (!request.isLongRunning()) {
            if (request.getInstances().isPresent() && (activeTasks.size() + pendingTasks.size() < request.getInstances().get())) {
                pendingType = PendingType.ONEOFF;
            } else {
                // Don't run one-off / on-demand task when already at instance count cap
                pendingType = PendingType.NEW_DEPLOY;
            }
        }
        if (pendingType != null) {
            builder.setPendingType(canceledOr(deployResult.getDeployState(), pendingType));
            requestManager.addToPendingQueue(builder.build());
        } else {
            LOG.warn("Could not determine pending type for deploy {}.", deployId);
        }
    } else if (!request.isDeployable() && !request.isOneOff()) {
        PendingType pendingType = canceledOr(deployResult.getDeployState(), PendingType.NEW_DEPLOY);
        requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), pendingType, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.absent(), pendingDeploy.getDeployMarker().getMessage()));
    }
    if (deployResult.getDeployState() == DeployState.SUCCEEDED) {
        if (request.isDeployable() && !request.isOneOff()) {
            // remove the lock on bounces in case we deployed during a bounce
            requestManager.markBounceComplete(request.getId());
        }
        if (requestWithState.getState() == RequestState.FINISHED) {
            // A FINISHED request is moved to ACTIVE state so we can reevaluate the schedule
            requestManager.activate(request, RequestHistoryType.UPDATED, System.currentTimeMillis(), deploy.isPresent() ? deploy.get().getUser() : Optional.absent(), Optional.absent());
        }
    }
    deployManager.saveDeployResult(pendingDeploy.getDeployMarker(), deploy, deployResult);
    if (request.isDeployable() && (deployResult.getDeployState() == DeployState.CANCELED || deployResult.getDeployState() == DeployState.FAILED || deployResult.getDeployState() == DeployState.OVERDUE)) {
        Optional<SingularityRequestDeployState> maybeRequestDeployState = deployManager.getRequestDeployState(request.getId());
        if (maybeRequestDeployState.isPresent() && maybeRequestDeployState.get().getActiveDeploy().isPresent() && !(requestWithState.getState() == RequestState.PAUSED || requestWithState.getState() == RequestState.DEPLOYING_TO_UNPAUSE)) {
            requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), maybeRequestDeployState.get().getActiveDeploy().get().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), deployResult.getDeployState() == DeployState.CANCELED ? PendingType.DEPLOY_CANCELLED : PendingType.DEPLOY_FAILED, request.getSkipHealthchecks(), pendingDeploy.getDeployMarker().getMessage()));
        }
    }
    if (request.isDeployable() && deployResult.getDeployState() == DeployState.SUCCEEDED && pendingDeploy.getDeployProgress().isPresent() && requestWithState.getState() != RequestState.PAUSED) {
        if (pendingDeploy.getDeployProgress().get().getTargetActiveInstances() != request.getInstancesSafe()) {
            requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), PendingType.UPDATED_REQUEST, request.getSkipHealthchecks(), pendingDeploy.getDeployMarker().getMessage()));
        }
    }
    if (requestWithState.getState() == RequestState.DEPLOYING_TO_UNPAUSE) {
        if (deployResult.getDeployState() == DeployState.SUCCEEDED) {
            requestManager.activate(request, RequestHistoryType.DEPLOYED_TO_UNPAUSE, deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), Optional.<String>absent());
            requestManager.deleteExpiringObject(SingularityExpiringPause.class, request.getId());
        } else {
            requestManager.pause(request, deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), Optional.<String>absent());
        }
    }
    if (pendingDeploy.getUpdatedRequest().isPresent() && deployResult.getDeployState() == DeployState.SUCCEEDED) {
        requestManager.update(pendingDeploy.getUpdatedRequest().get(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), Optional.<String>absent());
        requestManager.deleteExpiringObject(SingularityExpiringScale.class, request.getId());
    }
    removePendingDeploy(pendingDeploy);
}