Examples with SingularityPendingRequest com.hubspot.singularity.SingularityPendingRequest used on opensource projects

Example 41 with SingularityPendingRequest

use of com.hubspot.singularity.SingularityPendingRequest in project Singularity by HubSpot.

the class SingularityPendingRequestWithRunIdMigration method applyMigration.

@Override
public void applyMigration() {
    String basePath = "/requests/pending";
    LOG.info("Starting migration to rewrite one-off pending request paths to include run IDs");
    long start = System.currentTimeMillis();
    int rewrittenPaths = 0;
    try {
        if (curator.checkExists().forPath(basePath) == null) {
            LOG.info("Unable to run migration because pending requests base path doesn't exist!");
            return;
        }
    } catch (Exception exn) {
        LOG.error("Could not check existence of pending request path", exn);
        throw new RuntimeException(exn);
    }
    try {
        List<String> childPaths = curator.getChildren().forPath(basePath);
        for (String originalBasename : childPaths) {
            SingularityPendingRequest pendingRequest = requestTranscoder.fromBytes(curator.getData().forPath(String.format("%s/%s", basePath, originalBasename)));
            if (pendingRequest.getPendingType() == PendingType.IMMEDIATE || pendingRequest.getPendingType() == PendingType.ONEOFF) {
                String deployKey = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()).getId();
                String rewrittenBasename = String.format("%s%s%s", deployKey, pendingRequest.getTimestamp(), pendingRequest.getRunId().orElse(""));
                if (originalBasename.equals(rewrittenBasename)) {
                    LOG.warn("Not rewriting znode {}, because it had no runId and was therefore already correct", originalBasename);
                } else {
                    LOG.warn("Rewriting znode {} to {}", originalBasename, rewrittenBasename);
                    requestManager.addToPendingQueue(pendingRequest);
                    curator.delete().forPath(String.format("%s/%s", basePath, originalBasename));
                    rewrittenPaths += 1;
                }
            } else {
                LOG.warn("Not rewriting znode {}, already correct", originalBasename);
            }
        }
    } catch (Exception exn) {
        LOG.error("Connect to Zookeeper failed while running migration", exn);
        throw new RuntimeException(exn);
    }
    LOG.warn("Applied PendingRequestDataMigration to {} requests in {}", rewrittenPaths, JavaUtils.duration(start));
}

Example 42 with SingularityPendingRequest

use of com.hubspot.singularity.SingularityPendingRequest in project Singularity by HubSpot.

the class SingularityCmdLineArgsMigration method checkPendingRequests.

private void checkPendingRequests() {
    try {
        if (curator.checkExists().forPath(REQUEST_PENDING_PATH) == null) {
            return;
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    try {
        for (String pendingRequest : curator.getChildren().forPath(REQUEST_PENDING_PATH)) {
            SingularityPendingRequestPrevious previous = objectMapper.readValue(curator.getData().forPath(ZKPaths.makePath(REQUEST_PENDING_PATH, pendingRequest)), SingularityPendingRequestPrevious.class);
            SingularityPendingRequest newRequest = new SingularityPendingRequest(previous.requestId, previous.deployId, previous.timestamp, previous.user, previous.pendingType, getCmdLineArgs(previous.cmdLineArgs), Optional.<String>empty(), Optional.<Boolean>empty(), Optional.<String>empty(), Optional.<String>empty());
            LOG.info("Re-saving {}", newRequest);
            curator.setData().forPath(ZKPaths.makePath(REQUEST_PENDING_PATH, pendingRequest), pendingRequestTranscoder.toBytes(newRequest));
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 43 with SingularityPendingRequest

use of com.hubspot.singularity.SingularityPendingRequest in project Singularity by HubSpot.

the class SingularityUsageTest method scheduleTask.

protected void scheduleTask(String rqid, double requiredCpus, double requiredMemoryMb, int instances) {
    Resources rs = new Resources(requiredCpus, requiredMemoryMb, 0);
    SingularityRequest rq = new SingularityRequestBuilder(rqid, RequestType.WORKER).setInstances(Optional.of(instances)).build();
    SingularityDeploy dp = new SingularityDeployBuilder(rq.getId(), "deployId1").setCommand(Optional.of("sleep 100")).setResources(Optional.of(rs)).build();
    SingularityDeployMarker marker = new SingularityDeployMarker(dp.getRequestId(), dp.getId(), System.currentTimeMillis(), Optional.empty(), Optional.empty());
    SingularityPendingRequest pending = new SingularityPendingRequest(rq.getId(), dp.getId(), System.currentTimeMillis(), Optional.empty(), SingularityPendingRequest.PendingType.UPDATED_REQUEST, Optional.empty(), Optional.empty());
    saveRequest(rq);
    deployManager.saveDeploy(rq, marker, dp);
    deployManager.deletePendingDeploy(marker.getRequestId());
    deployManager.saveDeployResult(marker, Optional.of(dp), new SingularityDeployResult(DeployState.SUCCEEDED));
    deployManager.saveNewRequestDeployState(new SingularityRequestDeployState(marker.getRequestId(), Optional.of(marker), Optional.empty()));
    requestManager.addToPendingQueue(pending);
    scheduler.drainPendingQueue();
}

Example 44 with SingularityPendingRequest

use of com.hubspot.singularity.SingularityPendingRequest in project Singularity by HubSpot.

the class SingularitySchedulerTest method testSchedulerIsolatesPendingTasksBasedOnDeploy.

@Test
public void testSchedulerIsolatesPendingTasksBasedOnDeploy() {
    initRequest();
    initFirstDeploy();
    initSecondDeploy();
    SingularityPendingTask p1 = pendingTask(requestId, firstDeployId, PendingType.ONEOFF);
    SingularityPendingTask p2 = pendingTask(requestId, firstDeployId, PendingType.TASK_DONE);
    SingularityPendingTask p3 = pendingTask(requestId, secondDeployId, PendingType.TASK_DONE);
    taskManager.savePendingTask(p1);
    taskManager.savePendingTask(p2);
    taskManager.savePendingTask(p3);
    requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, secondDeployId, System.currentTimeMillis(), Optional.<String>empty(), PendingType.NEW_DEPLOY, Optional.<Boolean>empty(), Optional.<String>empty()));
    scheduler.drainPendingQueue();
    // we expect there to be 3 pending tasks :
    List<SingularityPendingTask> returnedScheduledTasks = taskManager.getPendingTasks();
    Assertions.assertEquals(3, returnedScheduledTasks.size());
    Assertions.assertTrue(returnedScheduledTasks.contains(p1));
    Assertions.assertTrue(returnedScheduledTasks.contains(p2));
    Assertions.assertTrue(!returnedScheduledTasks.contains(p3));
    boolean found = false;
    for (SingularityPendingTask pendingTask : returnedScheduledTasks) {
        if (pendingTask.getPendingTaskId().getDeployId().equals(secondDeployId)) {
            found = true;
            Assertions.assertEquals(PendingType.NEW_DEPLOY, pendingTask.getPendingTaskId().getPendingType());
        }
    }
    Assertions.assertTrue(found);
}

Example 45 with SingularityPendingRequest

use of com.hubspot.singularity.SingularityPendingRequest in project Singularity by HubSpot.

the class SingularitySchedulerTest method testImmediateRunReplacesScheduledTask.

@Test
public void testImmediateRunReplacesScheduledTask() {
    initScheduledRequest();
    SingularityDeploy deploy = SingularityDeploy.newBuilder(requestId, firstDeployId).setCommand(Optional.of("sleep 100")).build();
    SingularityDeployRequest singularityDeployRequest = new SingularityDeployRequest(deploy, Optional.empty(), Optional.empty(), Optional.empty());
    deployResource.deploy(singularityDeployRequest, singularityUser);
    scheduler.drainPendingQueue();
    SingularityPendingTask task1 = createAndSchedulePendingTask(firstDeployId);
    Assertions.assertEquals(1, taskManager.getPendingTaskIds().size());
    Assertions.assertEquals(PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType());
    requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, deploy.getId(), System.currentTimeMillis(), Optional.empty(), PendingType.IMMEDIATE, deploy.getSkipHealthchecksOnDeploy(), Optional.empty()));
    scheduler.drainPendingQueue();
    Assertions.assertEquals(1, taskManager.getPendingTaskIds().size());
    Assertions.assertEquals(PendingType.IMMEDIATE, taskManager.getPendingTaskIds().get(0).getPendingType());
}