Examples with Stairway bio.terra.stairway.Stairway used on opensource projects

Example 1 with Stairway

use of bio.terra.stairway.Stairway in project jade-data-repo by DataBiosphere.

the class IngestDriverStep method launchLoads.

private void launchLoads(FlightContext context, int launchCount, List<LoadFile> loadFiles, String profileId, UUID loadId, GoogleBucketResource bucketInfo) throws DatabaseOperationException, StairwayExecutionException, InterruptedException {
    Stairway stairway = context.getStairway();
    for (int i = 0; i < launchCount; i++) {
        LoadFile loadFile = loadFiles.get(i);
        String flightId = stairway.createFlightId();
        FileLoadModel fileLoadModel = new FileLoadModel().sourcePath(loadFile.getSourcePath()).targetPath(loadFile.getTargetPath()).mimeType(loadFile.getMimeType()).profileId(profileId).loadTag(loadTag).description(loadFile.getDescription());
        FlightMap inputParameters = new FlightMap();
        inputParameters.put(FileMapKeys.DATASET_ID, datasetId);
        inputParameters.put(FileMapKeys.REQUEST, fileLoadModel);
        inputParameters.put(FileMapKeys.BUCKET_INFO, bucketInfo);
        loadService.setLoadFileRunning(loadId, loadFile.getTargetPath(), flightId);
        // NOTE: this is the window where we have recorded a flight as RUNNING in the load_file
        // table, but it has not yet been launched. A failure in this window leaves "orphan"
        // loads that are marked running, but not actually started. We handle this
        // with the check for launch orphans at the beginning of the do() method.
        // We use submitToQueue to spread the file loaders across multiple instances of datarepo.
        stairway.submitToQueue(flightId, FileIngestWorkerFlight.class, inputParameters);
    }
}

Example 2 with Stairway

use of bio.terra.stairway.Stairway in project jade-data-repo by DataBiosphere.

the class JobServiceShutdownTest method testShutdown.

@Test
public void testShutdown() throws Exception {
    // Segregated into its own test, since the Stairway object is not usable after it completes
    // scenario:
    // - start a thread that creates one flight every X second; keeping track of the flights
    // - each flight sleeps Y secs and then returns; we want to get a backlog of flights in the queue
    // when thread catches JobServiceShutdown exception, it stops launching and returns its flight list
    // - sleep Z seconds to let the system fill up
    // - call shutdown
    // - validate that flights are either SUCCESS or READY
    // Test Control Parameters
    final int flightPerSecond = 1;
    final int flightWaitSeconds = 30;
    int maxStairwayThreads = appConfig.getMaxStairwayThreads();
    logger.info("maxStairwayThreads: " + maxStairwayThreads);
    JobTestShutdownFlightLauncher launcher = new JobTestShutdownFlightLauncher(flightPerSecond, flightWaitSeconds, testUser, jobService);
    Thread launchThread = new Thread(launcher);
    launchThread.start();
    // Build a backlog in the queue
    TimeUnit.SECONDS.sleep(2 * maxStairwayThreads);
    jobService.shutdown();
    launchThread.join();
    List<String> flightIdList = launcher.getFlightIdList();
    Stairway stairway = jobService.getStairway();
    for (String flightId : flightIdList) {
        FlightState state = stairway.getFlightState(flightId);
        FlightStatus status = state.getFlightStatus();
        logger.info("Flightid: " + flightId + "; status: " + status);
        assertTrue(status == FlightStatus.SUCCESS || status == FlightStatus.READY);
    }
}

Example 3 with Stairway

use of bio.terra.stairway.Stairway in project terra-workspace-manager by DataBiosphere.

the class LaunchCloneAllResourcesFlightStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException, RetryException {
    validateRequiredEntries(context.getInputParameters(), JobMapKeys.AUTH_USER_INFO.getKeyName(), JobMapKeys.REQUEST.getKeyName());
    validateRequiredEntries(context.getWorkingMap(), ControlledResourceKeys.RESOURCES_TO_CLONE, ControlledResourceKeys.CLONE_ALL_RESOURCES_FLIGHT_ID);
    final var userRequest = context.getInputParameters().get(JobMapKeys.AUTH_USER_INFO.getKeyName(), AuthenticatedUserRequest.class);
    @Nullable final var location = context.getInputParameters().get(ControlledResourceKeys.LOCATION, String.class);
    final var cloneAllResourcesFlightId = context.getWorkingMap().get(ControlledResourceKeys.CLONE_ALL_RESOURCES_FLIGHT_ID, String.class);
    final List<ResourceWithFlightId> resourcesAndFlightIds = context.getWorkingMap().get(ControlledResourceKeys.RESOURCES_TO_CLONE, new TypeReference<>() {
    });
    final var destinationWorkspace = context.getInputParameters().get(JobMapKeys.REQUEST.getKeyName(), Workspace.class);
    final Stairway stairway = context.getStairway();
    final FlightMap subflightInputParameters = new FlightMap();
    subflightInputParameters.put(JobMapKeys.AUTH_USER_INFO.getKeyName(), userRequest);
    subflightInputParameters.put(ControlledResourceKeys.RESOURCES_TO_CLONE, resourcesAndFlightIds);
    subflightInputParameters.put(ControlledResourceKeys.DESTINATION_WORKSPACE_ID, destinationWorkspace.getWorkspaceId());
    subflightInputParameters.put(ControlledResourceKeys.LOCATION, location);
    // Build a CloneAllResourcesFlight
    try {
        stairway.submit(cloneAllResourcesFlightId, CloneAllResourcesFlight.class, subflightInputParameters);
    } catch (DuplicateFlightIdException e) {
        // exit early if flight is already going
        return StepResult.getStepResultSuccess();
    } catch (StairwayException e) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
    }
    return StepResult.getStepResultSuccess();
}