Examples with FlightMap bio.terra.stairway.FlightMap used on opensource projects

Example 6 with FlightMap

use of bio.terra.stairway.FlightMap in project jade-data-repo by DataBiosphere.

the class CreateDatasetMetadataStep method doStep.

@Override
public StepResult doStep(FlightContext context) {
    try {
        Dataset newDataset = DatasetUtils.convertRequestWithGeneratedNames(datasetRequest);
        UUID datasetId = datasetDao.createAndLock(newDataset, context.getFlightId());
        FlightMap workingMap = context.getWorkingMap();
        workingMap.put(DatasetWorkingMapKeys.DATASET_ID, datasetId);
        DatasetSummaryModel datasetSummary = DatasetJsonConversion.datasetSummaryModelFromDatasetSummary(newDataset.getDatasetSummary());
        workingMap.put(JobMapKeys.RESPONSE.getKeyName(), datasetSummary);
        return StepResult.getStepResultSuccess();
    } catch (InvalidDatasetException idEx) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_FATAL, idEx);
    } catch (Exception ex) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_FATAL, new InvalidDatasetException("Cannot create dataset: " + datasetRequest.getName(), ex));
    }
}

Example 7 with FlightMap

use of bio.terra.stairway.FlightMap in project jade-data-repo by DataBiosphere.

the class CreateDatasetPrimaryDataStep method getDataset.

private Dataset getDataset(FlightContext context) {
    FlightMap workingMap = context.getWorkingMap();
    UUID datasetId = workingMap.get(DatasetWorkingMapKeys.DATASET_ID, UUID.class);
    return datasetDao.retrieve(datasetId);
}

Example 8 with FlightMap

use of bio.terra.stairway.FlightMap in project jade-data-repo by DataBiosphere.

the class DeleteDatasetPrimaryDataStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
    Dataset dataset = datasetService.retrieve(datasetId);
    bigQueryPdao.deleteDataset(dataset);
    if (configService.testInsertFault(ConfigEnum.LOAD_SKIP_FILE_LOAD)) {
        // If we didn't load files, don't try to delete them
        fileDao.deleteFilesFromDataset(dataset, fireStoreFile -> {
        });
    } else {
        fileDao.deleteFilesFromDataset(dataset, fireStoreFile -> gcsPdao.deleteFile(fireStoreFile));
    }
    // this fault is used by the DatasetConnectedTest > testOverlappingDeletes
    if (configService.testInsertFault(ConfigEnum.DATASET_DELETE_LOCK_CONFLICT_STOP_FAULT)) {
        logger.info("DATASET_DELETE_LOCK_CONFLICT_STOP_FAULT");
        while (!configService.testInsertFault(ConfigEnum.DATASET_DELETE_LOCK_CONFLICT_CONTINUE_FAULT)) {
            logger.info("Sleeping for CONTINUE FAULT");
            TimeUnit.SECONDS.sleep(5);
        }
        logger.info("DATASET_DELETE_LOCK_CONFLICT_CONTINUE_FAULT");
    }
    FlightMap map = context.getWorkingMap();
    map.put(JobMapKeys.STATUS_CODE.getKeyName(), HttpStatus.NO_CONTENT);
    return StepResult.getStepResultSuccess();
}

Example 9 with FlightMap

use of bio.terra.stairway.FlightMap in project jade-data-repo by DataBiosphere.

the class IngestDriverStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
    // Gather inputs
    FlightMap workingMap = context.getWorkingMap();
    String loadIdString = workingMap.get(LoadMapKeys.LOAD_ID, String.class);
    UUID loadId = UUID.fromString(loadIdString);
    GoogleBucketResource bucketResource = workingMap.get(FileMapKeys.BUCKET_INFO, GoogleBucketResource.class);
    try {
        // Check for launch orphans - these are loads in the RUNNING state that never
        // got recorded by stairway.
        checkForOrphans(context, loadId);
        // Load Loop
        while (true) {
            int podCount = kubeService.getActivePodCount();
            int concurrentFiles = configurationService.getParameterValue(ConfigEnum.LOAD_CONCURRENT_FILES);
            int scaledConcurrentFiles = podCount * concurrentFiles;
            // Get the state of active and failed loads
            LoadCandidates candidates = getLoadCandidates(context, loadId, scaledConcurrentFiles);
            int currentRunning = candidates.getRunningLoads().size();
            int candidateCount = candidates.getCandidateFiles().size();
            if (currentRunning == 0 && candidateCount == 0) {
                // Nothing doing and nothing to do
                break;
            }
            // Test for exceeding max failed loads; if so, wait for all RUNNINGs to finish
            if (candidates.getFailedLoads() > maxFailedFileLoads) {
                waitForAll(context, loadId, scaledConcurrentFiles);
                break;
            }
            // Launch new loads
            if (currentRunning < scaledConcurrentFiles) {
                // Compute how many loads to launch
                int launchCount = scaledConcurrentFiles - currentRunning;
                if (candidateCount < launchCount) {
                    launchCount = candidateCount;
                }
                launchLoads(context, launchCount, candidates.getCandidateFiles(), profileId, loadId, bucketResource);
                currentRunning += launchCount;
            }
            // Wait until some loads complete
            waitForAny(context, loadId, scaledConcurrentFiles, currentRunning);
        }
    } catch (DatabaseOperationException | StairwayExecutionException ex) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, ex);
    }
    return StepResult.getStepResultSuccess();
}

Example 10 with FlightMap

use of bio.terra.stairway.FlightMap in project jade-data-repo by DataBiosphere.

the class IngestDriverStep method getLoadCandidates.

private LoadCandidates getLoadCandidates(FlightContext context, UUID loadId, int concurrentLoads) throws DatabaseOperationException, InterruptedException {
    // We start by getting the database view of the state of loads.
    // For the running loads, we ask Stairway what the actual state is.
    // If they have completed, we mark them as such.
    // We then update the failure count and runnings loads list in the
    // LoadCandidates so it correctly reflects the running state
    // right now (more or less).
    LoadCandidates candidates = loadService.findCandidates(loadId, concurrentLoads);
    logger.debug("Candidates from db: failedLoads={}  runningLoads={}  candidateFiles={}", candidates.getFailedLoads(), candidates.getRunningLoads().size(), candidates.getCandidateFiles().size());
    int failureCount = candidates.getFailedLoads();
    List<LoadFile> realRunningLoads = new LinkedList<>();
    for (LoadFile loadFile : candidates.getRunningLoads()) {
        FlightState flightState = context.getStairway().getFlightState(loadFile.getFlightId());
        switch(flightState.getFlightStatus()) {
            case RUNNING:
            case WAITING:
            case READY:
            case QUEUED:
                realRunningLoads.add(loadFile);
                break;
            case ERROR:
            case FATAL:
                {
                    String error = "unknown error";
                    if (flightState.getException().isPresent()) {
                        error = flightState.getException().get().toString();
                    }
                    loadService.setLoadFileFailed(loadId, loadFile.getTargetPath(), error);
                    failureCount++;
                    break;
                }
            case SUCCESS:
                {
                    FlightMap resultMap = flightState.getResultMap().orElse(null);
                    if (resultMap == null) {
                        throw new FileSystemCorruptException("no result map in flight state");
                    }
                    String fileId = resultMap.get(FileMapKeys.FILE_ID, String.class);
                    FSFileInfo fileInfo = resultMap.get(FileMapKeys.FILE_INFO, FSFileInfo.class);
                    loadService.setLoadFileSucceeded(loadId, loadFile.getTargetPath(), fileId, fileInfo);
                    break;
                }
        }
    }
    candidates.failedLoads(failureCount).runningLoads(realRunningLoads);
    logger.debug("Candidates resolved: failedLoads={}  runningLoads={}  candidateFiles={}", candidates.getFailedLoads(), candidates.getRunningLoads().size(), candidates.getCandidateFiles().size());
    return candidates;
}