Examples with StairwayExecutionException bio.terra.stairway.exception.StairwayExecutionException used on opensource projects

Example 1 with StairwayExecutionException

use of bio.terra.stairway.exception.StairwayExecutionException in project jade-data-repo by DataBiosphere.

the class IngestDriverStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
    // Gather inputs
    FlightMap workingMap = context.getWorkingMap();
    String loadIdString = workingMap.get(LoadMapKeys.LOAD_ID, String.class);
    UUID loadId = UUID.fromString(loadIdString);
    GoogleBucketResource bucketResource = workingMap.get(FileMapKeys.BUCKET_INFO, GoogleBucketResource.class);
    try {
        // Check for launch orphans - these are loads in the RUNNING state that never
        // got recorded by stairway.
        checkForOrphans(context, loadId);
        // Load Loop
        while (true) {
            int podCount = kubeService.getActivePodCount();
            int concurrentFiles = configurationService.getParameterValue(ConfigEnum.LOAD_CONCURRENT_FILES);
            int scaledConcurrentFiles = podCount * concurrentFiles;
            // Get the state of active and failed loads
            LoadCandidates candidates = getLoadCandidates(context, loadId, scaledConcurrentFiles);
            int currentRunning = candidates.getRunningLoads().size();
            int candidateCount = candidates.getCandidateFiles().size();
            if (currentRunning == 0 && candidateCount == 0) {
                // Nothing doing and nothing to do
                break;
            }
            // Test for exceeding max failed loads; if so, wait for all RUNNINGs to finish
            if (candidates.getFailedLoads() > maxFailedFileLoads) {
                waitForAll(context, loadId, scaledConcurrentFiles);
                break;
            }
            // Launch new loads
            if (currentRunning < scaledConcurrentFiles) {
                // Compute how many loads to launch
                int launchCount = scaledConcurrentFiles - currentRunning;
                if (candidateCount < launchCount) {
                    launchCount = candidateCount;
                }
                launchLoads(context, launchCount, candidates.getCandidateFiles(), profileId, loadId, bucketResource);
                currentRunning += launchCount;
            }
            // Wait until some loads complete
            waitForAny(context, loadId, scaledConcurrentFiles, currentRunning);
        }
    } catch (DatabaseOperationException | StairwayExecutionException ex) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, ex);
    }
    return StepResult.getStepResultSuccess();
}

Example 2 with StairwayExecutionException

use of bio.terra.stairway.exception.StairwayExecutionException in project terra-workspace-manager by DataBiosphere.

the class LaunchCloneGcsBucketResourceFlightStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException, RetryException {
    validateRequiredEntries(context.getInputParameters(), JobMapKeys.AUTH_USER_INFO.getKeyName(), ControlledResourceKeys.DESTINATION_WORKSPACE_ID);
    @Nullable final var location = context.getInputParameters().get(ControlledResourceKeys.LOCATION, String.class);
    final var userRequest = context.getInputParameters().get(JobMapKeys.AUTH_USER_INFO.getKeyName(), AuthenticatedUserRequest.class);
    final var destinationWorkspaceId = context.getInputParameters().get(ControlledResourceKeys.DESTINATION_WORKSPACE_ID, UUID.class);
    // Gather input parameters for flight. See
    // bio.terra.workspace.service.resource.controlled.ControlledResourceService#cloneGcsBucket.
    final FlightMap subflightInputParameters = new FlightMap();
    subflightInputParameters.put(ControlledResourceKeys.LOCATION, location);
    subflightInputParameters.put(ControlledResourceKeys.DESTINATION_WORKSPACE_ID, destinationWorkspaceId);
    subflightInputParameters.put(JobMapKeys.AUTH_USER_INFO.getKeyName(), userRequest);
    subflightInputParameters.put(ResourceKeys.RESOURCE, resource);
    subflightInputParameters.put(ControlledResourceKeys.CLONING_INSTRUCTIONS, resource.getCloningInstructions());
    // submit flight
    try {
        context.getStairway().submit(subflightId, CloneControlledGcsBucketResourceFlight.class, subflightInputParameters);
    } catch (DuplicateFlightIdException unused) {
        return StepResult.getStepResultSuccess();
    } catch (DatabaseOperationException | StairwayExecutionException e) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
    }
    return StepResult.getStepResultSuccess();
}

Example 3 with StairwayExecutionException

use of bio.terra.stairway.exception.StairwayExecutionException in project terra-workspace-manager by DataBiosphere.

the class LaunchCreateReferenceResourceFlightStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException, RetryException {
    if (CloningInstructions.COPY_REFERENCE != resource.getCloningInstructions()) {
        // Nothing to do -- don't launch flight
        return StepResult.getStepResultSuccess();
    }
    FlightUtils.validateRequiredEntries(context.getInputParameters(), ControlledResourceKeys.DESTINATION_WORKSPACE_ID, JobMapKeys.AUTH_USER_INFO.getKeyName());
    final var destinationWorkspaceId = context.getInputParameters().get(ControlledResourceKeys.DESTINATION_WORKSPACE_ID, UUID.class);
    final var userRequest = context.getInputParameters().get(JobMapKeys.AUTH_USER_INFO.getKeyName(), AuthenticatedUserRequest.class);
    final String description = String.format("Clone of Referenced Resource %s", resource.getResourceId());
    final ReferencedResource destinationResource = WorkspaceCloneUtils.buildDestinationReferencedResource(resource, destinationWorkspaceId, null, description);
    // put the destination resource in the map, because it's not communicated
    // from the flight as the response (and we need the workspace ID)
    context.getWorkingMap().put(ControlledResourceKeys.DESTINATION_REFERENCED_RESOURCE, destinationResource);
    final FlightMap subflightInputParameters = new FlightMap();
    subflightInputParameters.put(ResourceKeys.RESOURCE, destinationResource);
    subflightInputParameters.put(JobMapKeys.AUTH_USER_INFO.getKeyName(), userRequest);
    subflightInputParameters.put(WorkspaceFlightMapKeys.ResourceKeys.RESOURCE_TYPE, resource.getResourceType().name());
    try {
        context.getStairway().submit(subflightId, CreateReferenceResourceFlight.class, subflightInputParameters);
    } catch (DatabaseOperationException | StairwayExecutionException e) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
    } catch (DuplicateFlightIdException unused) {
        // already submitted the flight - treat as success
        return StepResult.getStepResultSuccess();
    }
    FlightUtils.validateRequiredEntries(context.getWorkingMap(), ControlledResourceKeys.DESTINATION_REFERENCED_RESOURCE);
    return StepResult.getStepResultSuccess();
}

Example 4 with StairwayExecutionException

use of bio.terra.stairway.exception.StairwayExecutionException in project terra-resource-buffer by DataBiosphere.

the class FlightManagerTest method setup.

@BeforeEach
public void setup() throws Exception {
    when(mockStairwayComponent.get()).thenReturn(mockStairway);
    when(mockStairway.createFlightId()).thenReturn("flightId");
    doThrow(new StairwayExecutionException("test")).when(mockStairway).submitToQueue(anyString(), any(), any());
    flightManager = new FlightManager(bufferDao, flightSubmissionFactory, mockStairwayComponent, transactionTemplate);
}

Example 5 with StairwayExecutionException

use of bio.terra.stairway.exception.StairwayExecutionException in project terra-workspace-manager by DataBiosphere.

the class LaunchCloneControlledGcpBigQueryDatasetResourceFlightStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException, RetryException {
    validateRequiredEntries(context.getInputParameters(), JobMapKeys.AUTH_USER_INFO.getKeyName(), ControlledResourceKeys.DESTINATION_WORKSPACE_ID);
    final var userRequest = context.getInputParameters().get(JobMapKeys.AUTH_USER_INFO.getKeyName(), AuthenticatedUserRequest.class);
    final var destinationWorkspaceId = context.getInputParameters().get(ControlledResourceKeys.DESTINATION_WORKSPACE_ID, UUID.class);
    @Nullable final var location = context.getInputParameters().get(ControlledResourceKeys.LOCATION, String.class);
    // build input parameter map. Leave out resource name, description, and dataset name so that
    // they will take values from the source dataset.
    final var subflightInputParameters = new FlightMap();
    subflightInputParameters.put(JobMapKeys.AUTH_USER_INFO.getKeyName(), userRequest);
    subflightInputParameters.put(ControlledResourceKeys.DESTINATION_WORKSPACE_ID, destinationWorkspaceId);
    subflightInputParameters.put(ControlledResourceKeys.LOCATION, location);
    subflightInputParameters.put(ControlledResourceKeys.CLONING_INSTRUCTIONS, resource.getCloningInstructions());
    subflightInputParameters.put(ResourceKeys.RESOURCE, resource);
    subflightInputParameters.put(ControlledResourceKeys.LOCATION, location);
    // launch the flight
    try {
        context.getStairway().submit(subflightId, CloneControlledGcpBigQueryDatasetResourceFlight.class, subflightInputParameters);
    } catch (DuplicateFlightIdException unused) {
        return StepResult.getStepResultSuccess();
    } catch (DatabaseOperationException | StairwayExecutionException e) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
    }
    return StepResult.getStepResultSuccess();
}