Examples with StepResult bio.terra.stairway.StepResult used on opensource projects

Example 1 with StepResult

use of bio.terra.stairway.StepResult in project jade-data-repo by DataBiosphere.

the class CreateDatasetAssetStep method doStep.

@Override
public StepResult doStep(FlightContext context) {
    // TODO: Asset columns and tables need to match things in the dataset schema
    Dataset dataset = getDataset(context);
    FlightMap map = context.getWorkingMap();
    // get the dataset assets that already exist --asset name needs to be unique
    AssetSpecification newAssetSpecification = getNewAssetSpec(context, dataset);
    // add a fault that forces an exception to make sure the undo works
    try {
        configService.fault(ConfigEnum.CREATE_ASSET_FAULT, () -> {
            throw new RuntimeException("fault insertion");
        });
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    try {
        assetDao.create(newAssetSpecification, dataset.getId());
    } catch (InvalidAssetException e) {
        FlightUtils.setErrorResponse(context, e.getMessage(), HttpStatus.BAD_REQUEST);
        map.put(DatasetWorkingMapKeys.ASSET_NAME_COLLISION, true);
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_FATAL, e);
    }
    map.put(JobMapKeys.STATUS_CODE.getKeyName(), HttpStatus.CREATED);
    return StepResult.getStepResultSuccess();
}

Example 2 with StepResult

use of bio.terra.stairway.StepResult in project jade-data-repo by DataBiosphere.

the class CreateDatasetMetadataStep method doStep.

@Override
public StepResult doStep(FlightContext context) {
    try {
        Dataset newDataset = DatasetUtils.convertRequestWithGeneratedNames(datasetRequest);
        UUID datasetId = datasetDao.createAndLock(newDataset, context.getFlightId());
        FlightMap workingMap = context.getWorkingMap();
        workingMap.put(DatasetWorkingMapKeys.DATASET_ID, datasetId);
        DatasetSummaryModel datasetSummary = DatasetJsonConversion.datasetSummaryModelFromDatasetSummary(newDataset.getDatasetSummary());
        workingMap.put(JobMapKeys.RESPONSE.getKeyName(), datasetSummary);
        return StepResult.getStepResultSuccess();
    } catch (InvalidDatasetException idEx) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_FATAL, idEx);
    } catch (Exception ex) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_FATAL, new InvalidDatasetException("Cannot create dataset: " + datasetRequest.getName(), ex));
    }
}

Example 3 with StepResult

use of bio.terra.stairway.StepResult in project jade-data-repo by DataBiosphere.

the class IngestDriverStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
    // Gather inputs
    FlightMap workingMap = context.getWorkingMap();
    String loadIdString = workingMap.get(LoadMapKeys.LOAD_ID, String.class);
    UUID loadId = UUID.fromString(loadIdString);
    GoogleBucketResource bucketResource = workingMap.get(FileMapKeys.BUCKET_INFO, GoogleBucketResource.class);
    try {
        // Check for launch orphans - these are loads in the RUNNING state that never
        // got recorded by stairway.
        checkForOrphans(context, loadId);
        // Load Loop
        while (true) {
            int podCount = kubeService.getActivePodCount();
            int concurrentFiles = configurationService.getParameterValue(ConfigEnum.LOAD_CONCURRENT_FILES);
            int scaledConcurrentFiles = podCount * concurrentFiles;
            // Get the state of active and failed loads
            LoadCandidates candidates = getLoadCandidates(context, loadId, scaledConcurrentFiles);
            int currentRunning = candidates.getRunningLoads().size();
            int candidateCount = candidates.getCandidateFiles().size();
            if (currentRunning == 0 && candidateCount == 0) {
                // Nothing doing and nothing to do
                break;
            }
            // Test for exceeding max failed loads; if so, wait for all RUNNINGs to finish
            if (candidates.getFailedLoads() > maxFailedFileLoads) {
                waitForAll(context, loadId, scaledConcurrentFiles);
                break;
            }
            // Launch new loads
            if (currentRunning < scaledConcurrentFiles) {
                // Compute how many loads to launch
                int launchCount = scaledConcurrentFiles - currentRunning;
                if (candidateCount < launchCount) {
                    launchCount = candidateCount;
                }
                launchLoads(context, launchCount, candidates.getCandidateFiles(), profileId, loadId, bucketResource);
                currentRunning += launchCount;
            }
            // Wait until some loads complete
            waitForAny(context, loadId, scaledConcurrentFiles, currentRunning);
        }
    } catch (DatabaseOperationException | StairwayExecutionException ex) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, ex);
    }
    return StepResult.getStepResultSuccess();
}

Example 4 with StepResult

use of bio.terra.stairway.StepResult in project jade-data-repo by DataBiosphere.

the class IngestFileDirectoryStep method doStep.

@Override
public StepResult doStep(FlightContext context) {
    FlightMap inputParameters = context.getInputParameters();
    FileLoadModel loadModel = inputParameters.get(JobMapKeys.REQUEST.getKeyName(), FileLoadModel.class);
    FlightMap workingMap = context.getWorkingMap();
    String fileId = workingMap.get(FileMapKeys.FILE_ID, String.class);
    workingMap.put(FileMapKeys.LOAD_COMPLETED, false);
    String datasetId = dataset.getId().toString();
    String targetPath = loadModel.getTargetPath();
    try {
        // The state logic goes like this:
        // 1. the directory entry doesn't exist. We need to create the directory entry for it.
        // 2. the directory entry exists. There are three cases:
        // a. If loadTags do not match, then we throw FileAlreadyExistsException.
        // b. directory entry loadTag matches our loadTag AND entry fileId matches our fileId:
        // means we are recovering and need to complete the file creation work.
        // c. directory entry loadTag matches our loadTag AND entry fileId does NOT match our fileId
        // means this is a re-run of a load job. We update the fileId in the working map. We don't
        // know if we are recovering or already finished. We try to retrieve the file object for
        // the entry fileId:
        // i. If that is successful, then we already loaded this file. We store "completed=true"
        // in the working map, so other steps do nothing.
        // ii. If that fails, then we are recovering: we leave completed unset (=false) in the working map.
        // 
        // Lookup the file - on a recovery, we may have already created it, but not
        // finished. Or it might already exist, created by someone else.
        FireStoreDirectoryEntry existingEntry = fileDao.lookupDirectoryEntryByPath(dataset, targetPath);
        if (existingEntry == null) {
            // Not there - create it
            FireStoreDirectoryEntry newEntry = new FireStoreDirectoryEntry().fileId(fileId).isFileRef(true).path(fireStoreUtils.getDirectoryPath(loadModel.getTargetPath())).name(fireStoreUtils.getName(loadModel.getTargetPath())).datasetId(datasetId).loadTag(loadModel.getLoadTag());
            fileDao.createDirectoryEntry(dataset, newEntry);
        } else {
            if (!StringUtils.equals(existingEntry.getLoadTag(), loadModel.getLoadTag())) {
                // (a) Exists and is not our file
                throw new FileAlreadyExistsException("Path already exists: " + targetPath);
            }
            // (b) or (c) Load tags match - check file ids
            if (!StringUtils.equals(existingEntry.getFileId(), fileId)) {
                // (c) We are in a re-run of a load job. Try to get the file entry.
                fileId = existingEntry.getFileId();
                workingMap.put(FileMapKeys.FILE_ID, fileId);
                FireStoreFile fileEntry = fileDao.lookupFile(dataset, fileId);
                if (fileEntry != null) {
                    // (c)(i) We successfully loaded this file already
                    workingMap.put(FileMapKeys.LOAD_COMPLETED, true);
                }
            // (c)(ii) We are recovering and should continue this load; leave load completed false/unset
            }
        }
    } catch (FileSystemAbortTransactionException rex) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, rex);
    }
    return StepResult.getStepResultSuccess();
}

Example 5 with StepResult

use of bio.terra.stairway.StepResult in project jade-data-repo by DataBiosphere.

the class IngestFilePrimaryDataLocationStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
    FlightMap workingMap = context.getWorkingMap();
    Boolean loadComplete = workingMap.get(FileMapKeys.LOAD_COMPLETED, Boolean.class);
    if (loadComplete == null || !loadComplete) {
        try {
            GoogleBucketResource bucketForFile = locationService.getOrCreateBucketForFile(profileId, context.getFlightId());
            workingMap.put(FileMapKeys.BUCKET_INFO, bucketForFile);
        } catch (BucketLockException blEx) {
            return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, blEx);
        }
    }
    return StepResult.getStepResultSuccess();
}