use of bio.terra.model.DRSObject in project jade-data-repo by DataBiosphere.
the class DrsService method drsObjectFromFSDir.
private DRSObject drsObjectFromFSDir(FSDir fsDir, String snapshotId) {
DRSObject dirObject = makeCommonDrsObject(fsDir, snapshotId);
DRSChecksum drsChecksum = new DRSChecksum().type("crc32c").checksum("0");
dirObject.size(0L).addChecksumsItem(drsChecksum).contents(makeContentsList(fsDir, snapshotId));
return dirObject;
}
use of bio.terra.model.DRSObject in project jade-data-repo by DataBiosphere.
the class DrsService method makeCommonDrsObject.
private DRSObject makeCommonDrsObject(FSItem fsObject, String snapshotId) {
// Compute the time once; used for both created and updated times as per DRS spec for immutable objects
String theTime = fsObject.getCreatedDate().toString();
DrsId drsId = drsIdService.makeDrsId(fsObject, snapshotId);
return new DRSObject().id(drsId.toDrsObjectId()).name(getLastNameFromPath(fsObject.getPath())).createdTime(theTime).updatedTime(theTime).version(DRS_OBJECT_VERSION).description(fsObject.getDescription()).aliases(Collections.singletonList(fsObject.getPath())).size(fsObject.getSize()).checksums(fileService.makeChecksums(fsObject));
}
use of bio.terra.model.DRSObject in project jade-data-repo by DataBiosphere.
the class EncodeFileTest method encodeFileTest.
// NOTES ABOUT THIS TEST: this test requires create access to the jade-testdata bucket in order to
// re-write the json source data replacing the gs paths with the Jade object id.
@Test
public void encodeFileTest() throws Exception {
DatasetSummaryModel datasetSummary = connectedOperations.createDataset(profileModel, "encodefiletest-dataset.json");
// Load all of the files into the dataset
String targetPath = loadFiles(datasetSummary.getId(), false, false);
String gsPath = "gs://" + testConfig.getIngestbucket() + "/" + targetPath;
IngestRequestModel ingestRequest = new IngestRequestModel().format(IngestRequestModel.FormatEnum.JSON).table("file").path(gsPath);
connectedOperations.ingestTableSuccess(datasetSummary.getId(), ingestRequest);
// Delete the scratch blob
Blob scratchBlob = storage.get(BlobId.of(testConfig.getIngestbucket(), targetPath));
if (scratchBlob != null) {
scratchBlob.delete();
}
// Load donor success
ingestRequest.table("donor").path("gs://" + testConfig.getIngestbucket() + "/encodetest/donor.json");
connectedOperations.ingestTableSuccess(datasetSummary.getId(), ingestRequest);
// At this point, we have files and tabular data. Let's make a snapshot!
SnapshotSummaryModel snapshotSummary = connectedOperations.createSnapshot(datasetSummary, "encodefiletest-snapshot.json", "");
String fileUri = getFileRefIdFromSnapshot(snapshotSummary);
DrsId drsId = drsIdService.fromUri(fileUri);
DRSObject drsObject = connectedOperations.drsGetObjectSuccess(drsId.toDrsObjectId(), false);
String filePath = drsObject.getAliases().get(0);
FileModel fsObjById = connectedOperations.lookupSnapshotFileSuccess(snapshotSummary.getId(), drsId.getFsObjectId());
FileModel fsObjByPath = connectedOperations.lookupSnapshotFileByPathSuccess(snapshotSummary.getId(), filePath, 0);
assertThat("Retrieve snapshot file objects match", fsObjById, equalTo(fsObjByPath));
assertThat("Load tag is stored", fsObjById.getFileDetail().getLoadTag(), equalTo(loadTag));
// Build the reference directory name map
String datasetPath = "/" + datasetSummary.getName();
Map<String, List<String>> dirmap = makeDirectoryMap(datasetPath);
testSnapEnum(dirmap, snapshotSummary.getId(), datasetPath, -1);
testSnapEnum(dirmap, snapshotSummary.getId(), datasetPath, 0);
testSnapEnum(dirmap, snapshotSummary.getId(), datasetPath, 6);
testSnapEnum(dirmap, snapshotSummary.getId(), datasetPath, 3);
// Try to delete a file with a dependency
MvcResult result = mvc.perform(delete("/api/repository/v1/datasets/" + datasetSummary.getId() + "/files/" + drsId.getFsObjectId())).andReturn();
MockHttpServletResponse response = connectedOperations.validateJobModelAndWait(result);
assertThat(response.getStatus(), equalTo(HttpStatus.BAD_REQUEST.value()));
ErrorModel errorModel = connectedOperations.handleFailureCase(response);
assertThat("correct dependency error message", errorModel.getMessage(), containsString("used by at least one snapshot"));
}
use of bio.terra.model.DRSObject in project jade-data-repo by DataBiosphere.
the class SnapshotConnectedTest method testExcludeLockedFromSnapshotFileLookups.
@Test
public void testExcludeLockedFromSnapshotFileLookups() throws Exception {
// create a dataset
DatasetSummaryModel datasetRefSummary = createTestDataset("simple-with-filerefs-dataset.json");
// ingest a file
URI sourceUri = new URI("gs", "jade-testdata", "/fileloadprofiletest/1KBfile.txt", null, null);
String targetFilePath = "/mm/" + Names.randomizeName("testdir") + "/testExcludeLockedFromSnapshotFileLookups.txt";
FileLoadModel fileLoadModel = new FileLoadModel().sourcePath(sourceUri.toString()).description("testExcludeLockedFromSnapshotFileLookups").mimeType("text/plain").targetPath(targetFilePath).profileId(billingProfile.getId());
FileModel fileModel = connectedOperations.ingestFileSuccess(datasetRefSummary.getId(), fileLoadModel);
// generate a JSON file with the fileref
String jsonLine = "{\"name\":\"name1\", \"file_ref\":\"" + fileModel.getFileId() + "\"}\n";
// load a JSON file that contains the table rows to load into the test bucket
String jsonFileName = "this-better-pass.json";
String dirInCloud = "scratch/testExcludeLockedFromSnapshotFileLookups/" + UUID.randomUUID().toString();
BlobInfo ingestTableBlob = BlobInfo.newBuilder(testConfig.getIngestbucket(), dirInCloud + "/" + jsonFileName).build();
Storage storage = StorageOptions.getDefaultInstance().getService();
storage.create(ingestTableBlob, jsonLine.getBytes(StandardCharsets.UTF_8));
// make sure the JSON file gets cleaned up on test teardown
connectedOperations.addScratchFile(dirInCloud + "/" + jsonFileName);
// ingest the tabular data from the JSON file we just generated
String gsPath = "gs://" + testConfig.getIngestbucket() + "/" + dirInCloud + "/" + jsonFileName;
IngestRequestModel ingestRequest1 = new IngestRequestModel().format(IngestRequestModel.FormatEnum.JSON).table("tableA").path(gsPath);
connectedOperations.ingestTableSuccess(datasetRefSummary.getId(), ingestRequest1);
// create a snapshot
SnapshotSummaryModel snapshotSummary = connectedOperations.createSnapshot(datasetRefSummary, "simple-with-filerefs-snapshot.json", "");
// check that the snapshot metadata row is unlocked
String exclusiveLock = snapshotDao.getExclusiveLockState(UUID.fromString(snapshotSummary.getId()));
assertNull("snapshot row is unlocked", exclusiveLock);
String fileUri = getFileRefIdFromSnapshot(snapshotSummary);
DrsId drsId = drsIdService.fromUri(fileUri);
DRSObject drsObject = connectedOperations.drsGetObjectSuccess(drsId.toDrsObjectId(), false);
String filePath = drsObject.getAliases().get(0);
// lookup the snapshot file by DRS id, make sure it's returned (lookupSnapshotFileSuccess will already check)
FileModel fsObjById = connectedOperations.lookupSnapshotFileSuccess(snapshotSummary.getId(), drsId.getFsObjectId());
assertEquals("Retrieve snapshot file by DRS id matches desc", fsObjById.getDescription(), fileLoadModel.getDescription());
// lookup the snapshot file by DRS path and check that it's found
FileModel fsObjByPath = connectedOperations.lookupSnapshotFileByPathSuccess(snapshotSummary.getId(), filePath, 0);
assertEquals("Retrieve snapshot file by path matches desc", fsObjByPath.getDescription(), fileLoadModel.getDescription());
assertThat("Retrieve snapshot file objects match", fsObjById, CoreMatchers.equalTo(fsObjByPath));
// now the snapshot exists....let's get it locked!
// NO ASSERTS inside the block below where hang is enabled to reduce chance of failing before disabling the hang
// ====================================================
// enable hang in DeleteSnapshotPrimaryDataStep
configService.setFault(ConfigEnum.SNAPSHOT_DELETE_LOCK_CONFLICT_STOP_FAULT.name(), true);
// kick off a request to delete the snapshot. this should hang before unlocking the snapshot object.
// note: asserts are below outside the hang block
MvcResult deleteResult = mvc.perform(delete("/api/repository/v1/snapshots/" + snapshotSummary.getId())).andReturn();
// give the flight time to launch and get to the hang
TimeUnit.SECONDS.sleep(5);
// check that the snapshot metadata row has an exclusive lock
exclusiveLock = snapshotDao.getExclusiveLockState(UUID.fromString(snapshotSummary.getId()));
// lookup the snapshot file by id and check that it's NOT found
MockHttpServletResponse failedGetSnapshotByIdResponse = connectedOperations.lookupSnapshotFileRaw(snapshotSummary.getId(), drsId.getFsObjectId());
// lookup the snapshot file by path and check that it's NOT found
MockHttpServletResponse failedGetSnapshotByPathResponse = connectedOperations.lookupSnapshotFileByPathRaw(snapshotSummary.getId(), filePath, 0);
// disable hang in DeleteSnapshotPrimaryDataStep
configService.setFault(ConfigEnum.SNAPSHOT_DELETE_LOCK_CONFLICT_CONTINUE_FAULT.name(), true);
// ====================================================
// check that the snapshot metadata row has an exclusive lock after kicking off the delete
assertNotNull("snapshot row is exclusively locked", exclusiveLock);
assertEquals("Snapshot file NOT found by DRS id lookup", HttpStatus.NOT_FOUND, HttpStatus.valueOf(failedGetSnapshotByIdResponse.getStatus()));
assertEquals("Snapshot file NOT found by path lookup", HttpStatus.NOT_FOUND, HttpStatus.valueOf(failedGetSnapshotByPathResponse.getStatus()));
// check the response from the snapshot delete request
MockHttpServletResponse deleteResponse = connectedOperations.validateJobModelAndWait(deleteResult);
DeleteResponseModel deleteResponseModel = connectedOperations.handleSuccessCase(deleteResponse, DeleteResponseModel.class);
assertEquals("Snapshot delete returned successfully", DeleteResponseModel.ObjectStateEnum.DELETED, deleteResponseModel.getObjectState());
// delete the dataset and check that it succeeds
connectedOperations.deleteTestDataset(datasetRefSummary.getId());
// remove the file from the connectedoperation bookkeeping list
connectedOperations.removeFile(datasetRefSummary.getId(), fileModel.getFileId());
// try to fetch the snapshot again and confirm nothing is returned
connectedOperations.getSnapshotExpectError(snapshotSummary.getId(), HttpStatus.NOT_FOUND);
// try to fetch the dataset again and confirm nothing is returned
connectedOperations.getDatasetExpectError(datasetRefSummary.getId(), HttpStatus.NOT_FOUND);
}
use of bio.terra.model.DRSObject in project jade-data-repo by DataBiosphere.
the class AccessTest method fileAclTest.
@Test
public void fileAclTest() throws Exception {
makeAclTestDataset();
dataRepoFixtures.addDatasetPolicyMember(steward(), datasetSummaryModel.getId(), IamRole.CUSTODIAN, custodian().getEmail());
// Ingest a file into the dataset
String gsPath = "gs://" + testConfiguration.getIngestbucket();
FileModel fileModel = dataRepoFixtures.ingestFile(steward(), datasetSummaryModel.getId(), profileId, gsPath + "/files/File Design Notes.pdf", "/foo/bar");
// Ingest one row into the study 'file' table with a reference to that ingested file
String json = String.format("{\"file_id\":\"foo\",\"file_ref\":\"%s\"}", fileModel.getFileId());
String targetPath = "scratch/file" + UUID.randomUUID().toString() + ".json";
BlobInfo targetBlobInfo = BlobInfo.newBuilder(BlobId.of(testConfiguration.getIngestbucket(), targetPath)).build();
Storage storage = StorageOptions.getDefaultInstance().getService();
try (WriteChannel writer = storage.writer(targetBlobInfo)) {
writer.write(ByteBuffer.wrap(json.getBytes(StandardCharsets.UTF_8)));
}
IngestRequestModel request = dataRepoFixtures.buildSimpleIngest("file", targetPath);
IngestResponseModel ingestResponseModel = dataRepoFixtures.ingestJsonData(steward(), datasetSummaryModel.getId(), request);
assertThat("1 Row was ingested", ingestResponseModel.getRowCount(), equalTo(1L));
// Create a snapshot exposing the one row and grant read access to our reader.
SnapshotSummaryModel snapshotSummaryModel = dataRepoFixtures.createSnapshot(custodian(), datasetSummaryModel, "file-acl-test-snapshot.json");
snapshotIds.add(snapshotSummaryModel.getId());
SnapshotModel snapshotModel = dataRepoFixtures.getSnapshot(custodian(), snapshotSummaryModel.getId());
dataRepoFixtures.addSnapshotPolicyMember(custodian(), snapshotModel.getId(), IamRole.READER, reader().getEmail());
AuthenticatedUserRequest authenticatedReaderRequest = new AuthenticatedUserRequest().email(reader().getEmail()).token(Optional.of(readerToken));
boolean authorized = iamService.isAuthorized(authenticatedReaderRequest, IamResourceType.DATASNAPSHOT, snapshotModel.getId(), IamAction.READ_DATA);
assertTrue("correctly added reader", authorized);
// The reader does not have permission to make queries in any project,
// so we have to use the custodian to look up the DRS id.
BigQuery bigQueryCustodian = BigQueryFixtures.getBigQuery(snapshotModel.getDataProject(), custodianToken);
BigQueryFixtures.hasAccess(bigQueryCustodian, snapshotModel.getDataProject(), snapshotModel.getName());
// Read and validate the DRS URI from the file ref column in the 'file' table.
String drsObjectId = BigQueryFixtures.queryForDrsId(bigQueryCustodian, snapshotModel, "file", "file_ref");
// Use DRS API to lookup the file by DRS ID (pulled out of the URI).
DRSObject drsObject = dataRepoFixtures.drsGetObject(reader(), drsObjectId);
String gsuri = TestUtils.validateDrsAccessMethods(drsObject.getAccessMethods());
// Try to read the file of the gs path as reader and discoverer
String[] strings = gsuri.split("/", 4);
String bucketName = strings[2];
String blobName = strings[3];
BlobId blobId = BlobId.of(bucketName, blobName);
Storage readerStorage = getStorage(readerToken);
assertTrue("Reader can read some bytes of the file", canReadBlobRetry(readerStorage, blobId));
Storage discovererStorage = getStorage(discovererToken);
assertFalse("Discoverer can not read the file", canReadBlob(discovererStorage, blobId));
}
Aggregations