Examples with GoogleBucketResource bio.terra.service.resourcemanagement.google.GoogleBucketResource used on opensource projects

Example 1 with GoogleBucketResource

use of bio.terra.service.resourcemanagement.google.GoogleBucketResource in project jade-data-repo by DataBiosphere.

the class IngestDriverStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
    // Gather inputs
    FlightMap workingMap = context.getWorkingMap();
    String loadIdString = workingMap.get(LoadMapKeys.LOAD_ID, String.class);
    UUID loadId = UUID.fromString(loadIdString);
    GoogleBucketResource bucketResource = workingMap.get(FileMapKeys.BUCKET_INFO, GoogleBucketResource.class);
    try {
        // Check for launch orphans - these are loads in the RUNNING state that never
        // got recorded by stairway.
        checkForOrphans(context, loadId);
        // Load Loop
        while (true) {
            int podCount = kubeService.getActivePodCount();
            int concurrentFiles = configurationService.getParameterValue(ConfigEnum.LOAD_CONCURRENT_FILES);
            int scaledConcurrentFiles = podCount * concurrentFiles;
            // Get the state of active and failed loads
            LoadCandidates candidates = getLoadCandidates(context, loadId, scaledConcurrentFiles);
            int currentRunning = candidates.getRunningLoads().size();
            int candidateCount = candidates.getCandidateFiles().size();
            if (currentRunning == 0 && candidateCount == 0) {
                // Nothing doing and nothing to do
                break;
            }
            // Test for exceeding max failed loads; if so, wait for all RUNNINGs to finish
            if (candidates.getFailedLoads() > maxFailedFileLoads) {
                waitForAll(context, loadId, scaledConcurrentFiles);
                break;
            }
            // Launch new loads
            if (currentRunning < scaledConcurrentFiles) {
                // Compute how many loads to launch
                int launchCount = scaledConcurrentFiles - currentRunning;
                if (candidateCount < launchCount) {
                    launchCount = candidateCount;
                }
                launchLoads(context, launchCount, candidates.getCandidateFiles(), profileId, loadId, bucketResource);
                currentRunning += launchCount;
            }
            // Wait until some loads complete
            waitForAny(context, loadId, scaledConcurrentFiles, currentRunning);
        }
    } catch (DatabaseOperationException | StairwayExecutionException ex) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, ex);
    }
    return StepResult.getStepResultSuccess();
}

Example 2 with GoogleBucketResource

use of bio.terra.service.resourcemanagement.google.GoogleBucketResource in project jade-data-repo by DataBiosphere.

the class IngestFilePrimaryDataLocationStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
    FlightMap workingMap = context.getWorkingMap();
    Boolean loadComplete = workingMap.get(FileMapKeys.LOAD_COMPLETED, Boolean.class);
    if (loadComplete == null || !loadComplete) {
        try {
            GoogleBucketResource bucketForFile = locationService.getOrCreateBucketForFile(profileId, context.getFlightId());
            workingMap.put(FileMapKeys.BUCKET_INFO, bucketForFile);
        } catch (BucketLockException blEx) {
            return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, blEx);
        }
    }
    return StepResult.getStepResultSuccess();
}

Example 3 with GoogleBucketResource

use of bio.terra.service.resourcemanagement.google.GoogleBucketResource in project jade-data-repo by DataBiosphere.

the class IngestFilePrimaryDataStep method undoStep.

@Override
public StepResult undoStep(FlightContext context) {
    FlightMap workingMap = context.getWorkingMap();
    String fileId = workingMap.get(FileMapKeys.FILE_ID, String.class);
    GoogleBucketResource bucketResource = workingMap.get(FileMapKeys.BUCKET_INFO, GoogleBucketResource.class);
    gcsPdao.deleteFileById(dataset, fileId, bucketResource);
    return StepResult.getStepResultSuccess();
}

Example 4 with GoogleBucketResource

use of bio.terra.service.resourcemanagement.google.GoogleBucketResource in project jade-data-repo by DataBiosphere.

the class DeleteFilePrimaryDataStep method doStep.

@Override
public StepResult doStep(FlightContext context) {
    FlightMap workingMap = context.getWorkingMap();
    FireStoreFile fireStoreFile = workingMap.get(FileMapKeys.FIRESTORE_FILE, FireStoreFile.class);
    if (fireStoreFile != null) {
        GoogleBucketResource bucketResource = locationService.lookupBucket(fireStoreFile.getBucketResourceId());
        gcsPdao.deleteFileByGspath(fireStoreFile.getGspath(), bucketResource);
    }
    return StepResult.getStepResultSuccess();
}

Example 5 with GoogleBucketResource

use of bio.terra.service.resourcemanagement.google.GoogleBucketResource in project jade-data-repo by DataBiosphere.

the class GcsPdao method deleteFile.

// Consumer method for deleting GCS files driven from a scan over the firestore files
public void deleteFile(FireStoreFile fireStoreFile) {
    if (fireStoreFile != null) {
        GoogleBucketResource bucketResource = dataLocationService.lookupBucket(fireStoreFile.getBucketResourceId());
        deleteFileByGspath(fireStoreFile.getGspath(), bucketResource);
    }
}