Examples with SingularityPendingRequest com.hubspot.singularity.SingularityPendingRequest used on opensource projects

Example 1 with SingularityPendingRequest

use of com.hubspot.singularity.SingularityPendingRequest in project Singularity by HubSpot.

the class SingularityValidator method checkRunNowRequest.

public SingularityPendingRequest checkRunNowRequest(String deployId, Optional<String> userEmail, SingularityRequest request, Optional<SingularityRunNowRequest> maybeRunNowRequest, List<SingularityTaskId> activeTasks, List<SingularityPendingTaskId> pendingTasks) {
    SingularityRunNowRequest runNowRequest = fillRunNowRequest(maybeRunNowRequest);
    PendingType pendingType;
    if (request.isScheduled()) {
        pendingType = PendingType.IMMEDIATE;
        checkConflict(activeTasks.isEmpty(), "Cannot request immediate run of a scheduled job which is currently running (%s)", activeTasks);
    } else if (request.isOneOff()) {
        pendingType = PendingType.ONEOFF;
        if (request.getInstances().isPresent()) {
            checkRateLimited(activeTasks.size() + pendingTasks.size() < request.getInstances().get(), "No more than %s tasks allowed to run concurrently for request %s (%s active, %s pending)", request.getInstances().get(), request, activeTasks.size(), pendingTasks.size());
        }
    } else {
        throw badRequest("Can not request an immediate run of a non-scheduled / always running request (%s)", request);
    }
    if (runNowRequest.getRunAt().isPresent() && runNowRequest.getRunAt().get() > (System.currentTimeMillis() + TimeUnit.DAYS.toMillis(maxRunNowTaskLaunchDelay))) {
        throw badRequest("Task launch delay can be at most %d days from now.", maxRunNowTaskLaunchDelay);
    }
    return new SingularityPendingRequest(request.getId(), deployId, System.currentTimeMillis(), userEmail, pendingType, runNowRequest.getCommandLineArgs(), Optional.of(getRunId(runNowRequest.getRunId())), runNowRequest.getSkipHealthchecks(), runNowRequest.getMessage(), Optional.absent(), runNowRequest.getResources(), runNowRequest.getS3UploaderAdditionalFiles(), runNowRequest.getRunAsUserOverride(), runNowRequest.getEnvOverrides(), runNowRequest.getExtraArtifacts(), runNowRequest.getRunAt());
}

Example 2 with SingularityPendingRequest

use of com.hubspot.singularity.SingularityPendingRequest in project Singularity by HubSpot.

the class PendingRequestDataMigration method applyMigration.

@Override
public void applyMigration() {
    String basePath = "/requests/pending";
    LOG.warn("Starting migration to re-write pending request paths");
    long start = System.currentTimeMillis();
    int rewrittenPaths = 0;
    try {
        if (curator.checkExists().forPath(basePath) == null) {
            return;
        }
    } catch (Exception exn) {
        LOG.error("Could not check existence of pending request path", exn);
        throw new RuntimeException(exn);
    }
    try {
        List<String> childPaths = curator.getChildren().forPath(basePath);
        for (String childPath : childPaths) {
            SingularityPendingRequest pendingRequest = requestTranscoder.fromBytes(curator.getData().forPath(String.format("%s/%s", basePath, childPath)));
            if (pendingRequest.getPendingType() == PendingType.IMMEDIATE) {
                String rewrittenPath = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()).getId();
                LOG.warn("Rewriting path {} to {}", childPath, String.format("%s%s", rewrittenPath, pendingRequest.getTimestamp()));
                requestManager.addToPendingQueue(pendingRequest);
                curator.delete().forPath(String.format("%s/%s", basePath, childPath));
                rewrittenPaths += 1;
            } else {
                LOG.warn("Not rewriting path {}, already correct", childPath);
            }
        }
    } catch (Exception exn) {
        LOG.error("Connection to Zookeeper failed while running migration", exn);
        throw new RuntimeException(exn);
    }
    LOG.warn("Applied PendingRequestDataMigration to {} requests in {}", rewrittenPaths, JavaUtils.duration(start));
}

Example 3 with SingularityPendingRequest

use of com.hubspot.singularity.SingularityPendingRequest in project Singularity by HubSpot.

the class RequestHelper method checkReschedule.

private void checkReschedule(SingularityRequest newRequest, Optional<SingularityRequest> maybeOldRequest, Optional<String> user, long timestamp, Optional<Boolean> skipHealthchecks, Optional<String> message, Optional<SingularityBounceRequest> maybeBounceRequest) {
    if (!maybeOldRequest.isPresent()) {
        return;
    }
    if (shouldReschedule(newRequest, maybeOldRequest.get())) {
        Optional<String> maybeDeployId = deployManager.getInUseDeployId(newRequest.getId());
        if (maybeDeployId.isPresent()) {
            if (maybeBounceRequest.isPresent()) {
                Optional<String> actionId = maybeBounceRequest.get().getActionId().or(Optional.of(UUID.randomUUID().toString()));
                Optional<Boolean> removeFromLoadBalancer = Optional.absent();
                SingularityCreateResult createResult = requestManager.createCleanupRequest(new SingularityRequestCleanup(user, maybeBounceRequest.get().getIncremental().or(true) ? RequestCleanupType.INCREMENTAL_BOUNCE : RequestCleanupType.BOUNCE, System.currentTimeMillis(), Optional.<Boolean>absent(), removeFromLoadBalancer, newRequest.getId(), Optional.of(maybeDeployId.get()), skipHealthchecks, message, actionId, maybeBounceRequest.get().getRunShellCommandBeforeKill()));
                if (createResult != SingularityCreateResult.EXISTED) {
                    requestManager.bounce(newRequest, System.currentTimeMillis(), user, Optional.of("Bouncing due to bounce after scale"));
                    final SingularityBounceRequest validatedBounceRequest = validator.checkBounceRequest(maybeBounceRequest.get());
                    requestManager.saveExpiringObject(new SingularityExpiringBounce(newRequest.getId(), maybeDeployId.get(), user, System.currentTimeMillis(), validatedBounceRequest, actionId.get()));
                } else {
                    requestManager.addToPendingQueue(new SingularityPendingRequest(newRequest.getId(), maybeDeployId.get(), timestamp, user, PendingType.UPDATED_REQUEST, skipHealthchecks, message));
                }
            } else {
                requestManager.addToPendingQueue(new SingularityPendingRequest(newRequest.getId(), maybeDeployId.get(), timestamp, user, PendingType.UPDATED_REQUEST, skipHealthchecks, message));
            }
        }
    }
}

Example 4 with SingularityPendingRequest

use of com.hubspot.singularity.SingularityPendingRequest in project Singularity by HubSpot.

the class SingularitySchedulerTest method testMaxOnDemandTasks.

@Test
public void testMaxOnDemandTasks() {
    SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND);
    bldr.setInstances(Optional.of(1));
    requestResource.postRequest(bldr.build(), singularityUser);
    deploy("on_demand_deploy");
    deployChecker.checkDeploys();
    requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent()));
    requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent()));
    scheduler.drainPendingQueue();
    resourceOffers();
    Assert.assertEquals(1, taskManager.getActiveTaskIds().size());
}

Example 5 with SingularityPendingRequest

use of com.hubspot.singularity.SingularityPendingRequest in project Singularity by HubSpot.

the class SingularitySchedulerTest method testQueueMultipleOneOffs.

@Test
public void testQueueMultipleOneOffs() {
    SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND);
    requestResource.postRequest(bldr.build(), singularityUser);
    deploy("on_demand_deploy");
    deployChecker.checkDeploys();
    requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent()));
    requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent()));
    scheduler.drainPendingQueue();
    Assert.assertEquals(2, taskManager.getPendingTaskIds().size());
}