Examples with GoogleBucketResource bio.terra.service.resourcemanagement.google.GoogleBucketResource used on opensource projects

Example 6 with GoogleBucketResource

use of bio.terra.service.resourcemanagement.google.GoogleBucketResource in project jade-data-repo by DataBiosphere.

the class GcsPdao method fileAclOp.

private void fileAclOp(AclOp op, Dataset dataset, List<String> fileIds, String readersPolicyEmail) {
    Acl.Group readerGroup = new Acl.Group(readersPolicyEmail);
    Acl acl = Acl.newBuilder(readerGroup, Acl.Role.READER).build();
    for (String fileId : fileIds) {
        FSItem fsItem = fileDao.retrieveById(dataset, fileId, 0, true);
        if (fsItem instanceof FSFile) {
            FSFile fsFile = (FSFile) fsItem;
            GoogleBucketResource bucketForFile = dataLocationService.lookupBucket(fsFile.getBucketResourceId());
            Storage storage = storageForBucket(bucketForFile);
            URI gsUri = URI.create(fsFile.getGspath());
            String bucketPath = StringUtils.removeStart(gsUri.getPath(), "/");
            BlobId blobId = BlobId.of(bucketForFile.getName(), bucketPath);
            switch(op) {
                case ACL_OP_CREATE:
                    storage.createAcl(blobId, acl);
                    break;
                case ACL_OP_DELETE:
                    storage.deleteAcl(blobId, readerGroup);
                    break;
            }
        }
    }
}

Example 7 with GoogleBucketResource

use of bio.terra.service.resourcemanagement.google.GoogleBucketResource in project jade-data-repo by DataBiosphere.

the class BucketResourceTest method noBucketButMetadataExistsTest.

@Test
public // bucket_resource metadata row exists, but the corresponding bucket cloud resource does not
void noBucketButMetadataExistsTest() throws Exception {
    logger.info("app property allowReuseExistingBuckets = " + resourceService.getAllowReuseExistingBuckets());
    String bucketName = "testbucket_nobucketbutmetadataexiststest";
    String flightIdA = "noBucketButMetadataExistsTestA";
    bucketNames.add(bucketName);
    // create the bucket and metadata
    GoogleBucketRequest googleBucketRequest = buildBucketRequest(bucketName);
    GoogleBucketResource bucketResource = resourceService.getOrCreateBucket(googleBucketRequest, flightIdA);
    checkBucketExists(bucketResource.getResourceId());
    // delete the bucket cloud resource only
    Bucket bucket = storage.get(bucketName);
    boolean bucketDeleted = bucket.delete();
    assertTrue("bucket cloud resource deleted", bucketDeleted);
    // try to fetch the bucket again, check fails with corrupt metadata exception
    boolean caughtCorruptMetadataException = false;
    try {
        resourceService.getBucketResourceById(bucketResource.getResourceId(), true);
    } catch (CorruptMetadataException cmEx) {
        caughtCorruptMetadataException = true;
    }
    assertTrue("fetch failed when cloud resource does not exist", caughtCorruptMetadataException);
    // try to getOrCreate bucket again, check fails with corrupt metadata exception
    String flightIdB = "bucketExistsBeforeMetadataTestB";
    caughtCorruptMetadataException = false;
    try {
        resourceService.getOrCreateBucket(googleBucketRequest, flightIdB);
    } catch (CorruptMetadataException cmEx) {
        caughtCorruptMetadataException = true;
    }
    assertTrue("getOrCreate failed when cloud resource does not exist", caughtCorruptMetadataException);
    // update the metadata to match the cloud state, check that everything is deleted
    resourceService.updateBucketMetadata(bucketName, null);
    checkBucketDeleted(bucketResource.getName(), bucketResource.getResourceId());
}

Example 8 with GoogleBucketResource

use of bio.terra.service.resourcemanagement.google.GoogleBucketResource in project jade-data-repo by DataBiosphere.

the class BucketResourceTest method checkBucketDeleted.

private void checkBucketDeleted(String bucketName, UUID bucketResourceId) {
    // confirm the bucket and metadata row no longer exist
    Bucket bucket = storage.get(bucketName);
    assertNull("bucket no longer exists", bucket);
    boolean exceptionThrown = false;
    try {
        GoogleBucketResource bucketResource = resourceService.getBucketResourceById(bucketResourceId, false);
        logger.info("bucketResource = " + bucketResource);
    } catch (GoogleResourceNotFoundException grnfEx) {
        exceptionThrown = true;
    }
    assertTrue("bucket metadata row no longer exists", exceptionThrown);
}

Example 9 with GoogleBucketResource

use of bio.terra.service.resourcemanagement.google.GoogleBucketResource in project jade-data-repo by DataBiosphere.

the class IngestFilePrimaryDataStep method doStep.

@Override
public StepResult doStep(FlightContext context) {
    FlightMap inputParameters = context.getInputParameters();
    FileLoadModel fileLoadModel = inputParameters.get(JobMapKeys.REQUEST.getKeyName(), FileLoadModel.class);
    FlightMap workingMap = context.getWorkingMap();
    String fileId = workingMap.get(FileMapKeys.FILE_ID, String.class);
    Boolean loadComplete = workingMap.get(FileMapKeys.LOAD_COMPLETED, Boolean.class);
    if (loadComplete == null || !loadComplete) {
        // The bucket has been selected for this file. In the single file load case, the info
        // is stored in the working map. In the bulk load case, the info is stored in the input
        // parameters.
        GoogleBucketResource bucketResource = inputParameters.get(FileMapKeys.BUCKET_INFO, GoogleBucketResource.class);
        if (bucketResource == null) {
            bucketResource = workingMap.get(FileMapKeys.BUCKET_INFO, GoogleBucketResource.class);
        }
        FSFileInfo fsFileInfo;
        if (configService.testInsertFault(ConfigEnum.LOAD_SKIP_FILE_LOAD)) {
            fsFileInfo = new FSFileInfo().fileId(fileId).bucketResourceId(bucketResource.getResourceId().toString()).checksumCrc32c(null).checksumMd5("baaaaaad").createdDate(Instant.now().toString()).gspath("gs://path").size(100L);
        } else {
            fsFileInfo = gcsPdao.copyFile(dataset, fileLoadModel, fileId, bucketResource);
        }
        workingMap.put(FileMapKeys.FILE_INFO, fsFileInfo);
    }
    return StepResult.getStepResultSuccess();
}

Example 10 with GoogleBucketResource

use of bio.terra.service.resourcemanagement.google.GoogleBucketResource in project jade-data-repo by DataBiosphere.

the class BucketResourceTest method bucketExistsBeforeMetadataTest.

@Test
public // bucket cloud resource exists, but the corresponding bucket_resource metadata row does not
void bucketExistsBeforeMetadataTest() throws Exception {
    logger.info("app property allowReuseExistingBuckets = " + resourceService.getAllowReuseExistingBuckets());
    String bucketName = "testbucket_bucketexistsbeforemetadatatest";
    String flightIdA = "bucketExistsBeforeMetadataTestA";
    bucketNames.add(bucketName);
    // create the bucket and metadata
    GoogleBucketRequest googleBucketRequest = buildBucketRequest(bucketName);
    GoogleBucketResource bucketResource = resourceService.getOrCreateBucket(googleBucketRequest, flightIdA);
    checkBucketExists(bucketResource.getResourceId());
    // delete the metadata only
    boolean rowDeleted = resourceDao.deleteBucketMetadata(bucketName, flightIdA);
    assertTrue("metadata row deleted", rowDeleted);
    // try to fetch the bucket again, check fails with not found exception
    boolean caughtNotFoundException = false;
    try {
        resourceService.getBucketResourceById(bucketResource.getResourceId(), true);
    } catch (GoogleResourceNotFoundException cmEx) {
        caughtNotFoundException = true;
    }
    assertTrue("fetch failed when metadata does not exist", caughtNotFoundException);
    // set application property allowReuseExistingBuckets=false
    // try to create bucket again, check fails with corrupt metadata exception
    resourceService.setAllowReuseExistingBuckets(false);
    String flightIdB = "bucketExistsBeforeMetadataTestB";
    boolean caughtCorruptMetadataException = false;
    try {
        resourceService.getOrCreateBucket(googleBucketRequest, flightIdB);
    } catch (CorruptMetadataException cmEx) {
        caughtCorruptMetadataException = true;
    }
    assertTrue("create failed when cloud resource already exists", caughtCorruptMetadataException);
    // set application property allowReuseExistingBuckets=true
    // try to create bucket again, check succeeds
    resourceService.setAllowReuseExistingBuckets(true);
    String flightIdC = "bucketExistsBeforeMetadataTestC";
    bucketResource = resourceService.getOrCreateBucket(googleBucketRequest, flightIdC);
    // check the bucket and metadata exist
    checkBucketExists(bucketResource.getResourceId());
    // delete the bucket and metadata
    deleteBucket(bucketResource.getName());
    checkBucketDeleted(bucketResource.getName(), bucketResource.getResourceId());
    // restore original value of application property allowReuseExistingBuckets, which was saved in setup
    // (this is also done in cleanup after all tests, in case this test errors out before reaching this line)
    resourceService.setAllowReuseExistingBuckets(allowReuseExistingBuckets);
}