Examples with FlightState bio.terra.stairway.FlightState used on opensource projects

Example 1 with FlightState

use of bio.terra.stairway.FlightState in project jade-data-repo by DataBiosphere.

the class IngestDriverStep method getLoadCandidates.

private LoadCandidates getLoadCandidates(FlightContext context, UUID loadId, int concurrentLoads) throws DatabaseOperationException, InterruptedException {
    // We start by getting the database view of the state of loads.
    // For the running loads, we ask Stairway what the actual state is.
    // If they have completed, we mark them as such.
    // We then update the failure count and runnings loads list in the
    // LoadCandidates so it correctly reflects the running state
    // right now (more or less).
    LoadCandidates candidates = loadService.findCandidates(loadId, concurrentLoads);
    logger.debug("Candidates from db: failedLoads={}  runningLoads={}  candidateFiles={}", candidates.getFailedLoads(), candidates.getRunningLoads().size(), candidates.getCandidateFiles().size());
    int failureCount = candidates.getFailedLoads();
    List<LoadFile> realRunningLoads = new LinkedList<>();
    for (LoadFile loadFile : candidates.getRunningLoads()) {
        FlightState flightState = context.getStairway().getFlightState(loadFile.getFlightId());
        switch(flightState.getFlightStatus()) {
            case RUNNING:
            case WAITING:
            case READY:
            case QUEUED:
                realRunningLoads.add(loadFile);
                break;
            case ERROR:
            case FATAL:
                {
                    String error = "unknown error";
                    if (flightState.getException().isPresent()) {
                        error = flightState.getException().get().toString();
                    }
                    loadService.setLoadFileFailed(loadId, loadFile.getTargetPath(), error);
                    failureCount++;
                    break;
                }
            case SUCCESS:
                {
                    FlightMap resultMap = flightState.getResultMap().orElse(null);
                    if (resultMap == null) {
                        throw new FileSystemCorruptException("no result map in flight state");
                    }
                    String fileId = resultMap.get(FileMapKeys.FILE_ID, String.class);
                    FSFileInfo fileInfo = resultMap.get(FileMapKeys.FILE_INFO, FSFileInfo.class);
                    loadService.setLoadFileSucceeded(loadId, loadFile.getTargetPath(), fileId, fileInfo);
                    break;
                }
        }
    }
    candidates.failedLoads(failureCount).runningLoads(realRunningLoads);
    logger.debug("Candidates resolved: failedLoads={}  runningLoads={}  candidateFiles={}", candidates.getFailedLoads(), candidates.getRunningLoads().size(), candidates.getCandidateFiles().size());
    return candidates;
}

Example 2 with FlightState

use of bio.terra.stairway.FlightState in project jade-data-repo by DataBiosphere.

the class JobService method retrieveJobResultWorker.

private <T> JobResultWithStatus<T> retrieveJobResultWorker(String jobId, Class<T> resultClass) throws StairwayException, InterruptedException {
    FlightState flightState = stairway.getFlightState(jobId);
    FlightMap resultMap = flightState.getResultMap().orElse(null);
    if (resultMap == null) {
        throw new InvalidResultStateException("No result map returned from flight");
    }
    switch(flightState.getFlightStatus()) {
        case FATAL:
        case ERROR:
            if (flightState.getException().isPresent()) {
                Exception exception = flightState.getException().get();
                if (exception instanceof RuntimeException) {
                    throw (RuntimeException) exception;
                } else {
                    throw new JobResponseException("wrap non-runtime exception", exception);
                }
            }
            throw new InvalidResultStateException("Failed operation with no exception reported");
        case SUCCESS:
            HttpStatus statusCode = resultMap.get(JobMapKeys.STATUS_CODE.getKeyName(), HttpStatus.class);
            if (statusCode == null) {
                statusCode = HttpStatus.OK;
            }
            return new JobResultWithStatus<T>().statusCode(statusCode).result(resultMap.get(JobMapKeys.RESPONSE.getKeyName(), resultClass));
        case RUNNING:
            throw new JobNotCompleteException("Attempt to retrieve job result before job is complete; job id: " + flightState.getFlightId());
        default:
            throw new InvalidResultStateException("Impossible case reached");
    }
}

Example 3 with FlightState

use of bio.terra.stairway.FlightState in project jade-data-repo by DataBiosphere.

the class FlightStates method makeFlightCompletedState.

public static FlightState makeFlightCompletedState() {
    DatasetSummaryModel req = buildMinimalDatasetSummary();
    FlightMap resultMap = new FlightMap();
    resultMap.put(JobMapKeys.RESPONSE.getKeyName(), req);
    resultMap.put(JobMapKeys.STATUS_CODE.getKeyName(), HttpStatus.I_AM_A_TEAPOT);
    resultMap.put(JobMapKeys.DESCRIPTION.getKeyName(), req.getDescription());
    FlightState flightState = new FlightState();
    flightState.setFlightId(testFlightId);
    flightState.setFlightStatus(FlightStatus.SUCCESS);
    flightState.setSubmitted(submittedTime);
    flightState.setInputParameters(resultMap);
    flightState.setResultMap(resultMap);
    flightState.setCompleted(completedTime);
    return flightState;
}

Example 4 with FlightState

use of bio.terra.stairway.FlightState in project jade-data-repo by DataBiosphere.

the class JobServiceShutdownTest method testShutdown.

@Test
public void testShutdown() throws Exception {
    // Segregated into its own test, since the Stairway object is not usable after it completes
    // scenario:
    // - start a thread that creates one flight every X second; keeping track of the flights
    // - each flight sleeps Y secs and then returns; we want to get a backlog of flights in the queue
    // when thread catches JobServiceShutdown exception, it stops launching and returns its flight list
    // - sleep Z seconds to let the system fill up
    // - call shutdown
    // - validate that flights are either SUCCESS or READY
    // Test Control Parameters
    final int flightPerSecond = 1;
    final int flightWaitSeconds = 30;
    int maxStairwayThreads = appConfig.getMaxStairwayThreads();
    logger.info("maxStairwayThreads: " + maxStairwayThreads);
    JobTestShutdownFlightLauncher launcher = new JobTestShutdownFlightLauncher(flightPerSecond, flightWaitSeconds, testUser, jobService);
    Thread launchThread = new Thread(launcher);
    launchThread.start();
    // Build a backlog in the queue
    TimeUnit.SECONDS.sleep(2 * maxStairwayThreads);
    jobService.shutdown();
    launchThread.join();
    List<String> flightIdList = launcher.getFlightIdList();
    Stairway stairway = jobService.getStairway();
    for (String flightId : flightIdList) {
        FlightState state = stairway.getFlightState(flightId);
        FlightStatus status = state.getFlightStatus();
        logger.info("Flightid: " + flightId + "; status: " + status);
        assertTrue(status == FlightStatus.SUCCESS || status == FlightStatus.READY);
    }
}

Example 5 with FlightState

use of bio.terra.stairway.FlightState in project terra-workspace-manager by DataBiosphere.

the class CreateGcpContextFlightV2Test method successCreatesProjectAndContext.

@Test
@DisabledIfEnvironmentVariable(named = "TEST_ENV", matches = BUFFER_SERVICE_DISABLED_ENVS_REG_EX)
void successCreatesProjectAndContext() throws Exception {
    UUID workspaceId = createWorkspace(spendUtils.defaultSpendId());
    AuthenticatedUserRequest userRequest = userAccessUtils.defaultUserAuthRequest();
    assertTrue(testUtils.getAuthorizedGcpCloudContext(workspaceId, userRequest).isEmpty());
    // Retry steps once to validate idempotency.
    Map<String, StepStatus> retrySteps = getStepNameToStepStatusMap();
    FlightDebugInfo debugInfo = FlightDebugInfo.newBuilder().doStepFailures(retrySteps).build();
    FlightState flightState = StairwayTestUtils.blockUntilFlightCompletes(jobService.getStairway(), CreateGcpContextFlightV2.class, createInputParameters(workspaceId, userRequest), STAIRWAY_FLIGHT_TIMEOUT, debugInfo);
    assertEquals(FlightStatus.SUCCESS, flightState.getFlightStatus());
    String projectId = flightState.getResultMap().get().get(WorkspaceFlightMapKeys.GCP_PROJECT_ID, String.class);
    assertTrue(testUtils.getAuthorizedGcpCloudContext(workspaceId, userRequest).isPresent());
    String contextProjectId = workspaceService.getAuthorizedRequiredGcpProject(workspaceId, userRequest);
    assertEquals(projectId, contextProjectId);
    // Verify that the policies were properly stored
    Optional<GcpCloudContext> optionalCloudContext = testUtils.getAuthorizedGcpCloudContext(workspaceId, userRequest);
    assertTrue(optionalCloudContext.isPresent(), "has cloud context");
    GcpCloudContext cloudContext = optionalCloudContext.get();
    assertTrue(cloudContext.getSamPolicyOwner().isPresent(), "has owner policy");
    assertTrue(cloudContext.getSamPolicyWriter().isPresent(), "has writer policy");
    assertTrue(cloudContext.getSamPolicyReader().isPresent(), "has reader policy");
    assertTrue(cloudContext.getSamPolicyApplication().isPresent(), "has application policy");
    Project project = crl.getCloudResourceManagerCow().projects().get(projectId).execute();
    assertEquals(projectId, project.getProjectId());
    assertEquals("billingAccounts/" + spendUtils.defaultBillingAccountId(), crl.getCloudBillingClientCow().getProjectBillingInfo("projects/" + projectId).getBillingAccountName());
    assertRolesExist(project);
    assertPolicyGroupsSynced(workspaceId, project);
}