use of bio.terra.model.SnapshotSummaryModel in project jade-data-repo by DataBiosphere.
the class SnapshotTest method snapshotRowIdsHappyPathTest.
@Test
public void snapshotRowIdsHappyPathTest() throws Exception {
// fetch rowIds from the ingested dataset by querying the participant table
DatasetModel dataset = dataRepoFixtures.getDataset(steward(), datasetId);
String datasetProject = dataset.getDataProject();
String bqDatasetName = PdaoConstant.PDAO_PREFIX + dataset.getName();
String participantTable = "participant";
String sampleTable = "sample";
BigQuery bigQuery = BigQueryFixtures.getBigQuery(dataset.getDataProject(), stewardToken);
String sql = String.format("SELECT %s FROM `%s.%s.%s`", PdaoConstant.PDAO_ROW_ID_COLUMN, datasetProject, bqDatasetName, participantTable);
TableResult participantIds = BigQueryFixtures.query(sql, bigQuery);
List<String> participantIdList = StreamSupport.stream(participantIds.getValues().spliterator(), false).map(v -> v.get(0).getStringValue()).collect(Collectors.toList());
sql = String.format("SELECT %s FROM `%s.%s.%s`", PdaoConstant.PDAO_ROW_ID_COLUMN, datasetProject, bqDatasetName, sampleTable);
TableResult sampleIds = BigQueryFixtures.query(sql, bigQuery);
List<String> sampleIdList = StreamSupport.stream(sampleIds.getValues().spliterator(), false).map(v -> v.get(0).getStringValue()).collect(Collectors.toList());
// swap in these row ids in the request
SnapshotRequestModel requestModel = jsonLoader.loadObject("ingest-test-snapshot-row-ids-test.json", SnapshotRequestModel.class);
requestModel.getContents().get(0).getRowIdSpec().getTables().get(0).setRowIds(participantIdList);
requestModel.getContents().get(0).getRowIdSpec().getTables().get(1).setRowIds(sampleIdList);
SnapshotSummaryModel snapshotSummary = dataRepoFixtures.createSnapshotWithRequest(steward(), dataset.getName(), requestModel);
TimeUnit.SECONDS.sleep(10);
createdSnapshotIds.add(snapshotSummary.getId());
SnapshotModel snapshot = dataRepoFixtures.getSnapshot(steward(), snapshotSummary.getId());
assertEquals("new snapshot has been created", snapshot.getName(), requestModel.getName());
assertEquals("new snapshot has the correct number of tables", requestModel.getContents().get(0).getRowIdSpec().getTables().size(), snapshot.getTables().size());
// TODO: get the snapshot and make sure the number of rows matches with the row ids input
assertThat("one relationship comes through", snapshot.getRelationships().size(), equalTo(1));
assertThat("the right relationship comes through", snapshot.getRelationships().get(0).getName(), equalTo("sample_participants"));
}
use of bio.terra.model.SnapshotSummaryModel in project jade-data-repo by DataBiosphere.
the class SnapshotTest method snapshotByQueryHappyPathTest.
@Test
public void snapshotByQueryHappyPathTest() throws Exception {
DatasetModel dataset = dataRepoFixtures.getDataset(steward(), datasetId);
String datasetName = dataset.getName();
SnapshotRequestModel requestModel = jsonLoader.loadObject("ingest-test-snapshot-query.json", SnapshotRequestModel.class);
// swap in the correct dataset name (with the id at the end)
requestModel.getContents().get(0).setDatasetName(datasetName);
requestModel.getContents().get(0).getQuerySpec().setQuery("SELECT " + datasetName + ".sample.datarepo_row_id FROM " + datasetName + ".sample WHERE " + datasetName + ".sample.id ='sample6'");
SnapshotSummaryModel snapshotSummary = dataRepoFixtures.createSnapshotWithRequest(steward(), datasetName, requestModel);
TimeUnit.SECONDS.sleep(10);
createdSnapshotIds.add(snapshotSummary.getId());
SnapshotModel snapshot = dataRepoFixtures.getSnapshot(steward(), snapshotSummary.getId());
assertEquals("new snapshot has been created", snapshot.getName(), requestModel.getName());
}
use of bio.terra.model.SnapshotSummaryModel in project jade-data-repo by DataBiosphere.
the class BigQueryPdaoTest method testGetFullViews.
@Test
public void testGetFullViews() throws Exception {
Dataset dataset = readDataset("ingest-test-dataset.json");
// Stage tabular data for ingest.
String targetPath = "scratch/file" + UUID.randomUUID().toString() + "/";
String bucket = testConfig.getIngestbucket();
BlobInfo participantBlob = BlobInfo.newBuilder(bucket, targetPath + "ingest-test-participant.json").build();
BlobInfo sampleBlob = BlobInfo.newBuilder(bucket, targetPath + "ingest-test-sample.json").build();
BlobInfo fileBlob = BlobInfo.newBuilder(bucket, targetPath + "ingest-test-file.json").build();
try {
bigQueryPdao.createDataset(dataset);
storage.create(participantBlob, readFile("ingest-test-participant.json"));
storage.create(sampleBlob, readFile("ingest-test-sample.json"));
storage.create(fileBlob, readFile("ingest-test-file.json"));
// Ingest staged data into the new dataset.
IngestRequestModel ingestRequest = new IngestRequestModel().format(IngestRequestModel.FormatEnum.JSON);
String datasetId = dataset.getId().toString();
connectedOperations.ingestTableSuccess(datasetId, ingestRequest.table("participant").path(gsPath(participantBlob)));
connectedOperations.ingestTableSuccess(datasetId, ingestRequest.table("sample").path(gsPath(sampleBlob)));
connectedOperations.ingestTableSuccess(datasetId, ingestRequest.table("file").path(gsPath(fileBlob)));
// Create a full-view snapshot!
DatasetSummaryModel datasetSummary = DatasetJsonConversion.datasetSummaryModelFromDatasetSummary(dataset.getDatasetSummary());
SnapshotSummaryModel snapshotSummary = connectedOperations.createSnapshot(datasetSummary, "snapshot-fullviews-test-snapshot.json", "");
SnapshotModel snapshot = connectedOperations.getSnapshot(snapshotSummary.getId());
BigQueryProject bigQueryProject = TestUtils.bigQueryProjectForDatasetName(datasetDao, dataLocationService, dataset.getName());
Assert.assertThat(snapshot.getTables().size(), is(equalTo(3)));
List<String> participantIds = queryForIds(snapshot.getName(), "participant", bigQueryProject);
List<String> sampleIds = queryForIds(snapshot.getName(), "sample", bigQueryProject);
List<String> fileIds = queryForIds(snapshot.getName(), "file", bigQueryProject);
Assert.assertThat(participantIds, containsInAnyOrder("participant_1", "participant_2", "participant_3", "participant_4", "participant_5"));
Assert.assertThat(sampleIds, containsInAnyOrder("sample1", "sample2", "sample3", "sample4", "sample5", "sample6", "sample7"));
Assert.assertThat(fileIds, is(equalTo(Collections.singletonList("file1"))));
} finally {
storage.delete(participantBlob.getBlobId(), sampleBlob.getBlobId(), fileBlob.getBlobId());
bigQueryPdao.deleteDataset(dataset);
// Need to manually clean up the DAO because `readDataset` bypasses the
// `connectedOperations` object, so we can't rely on its auto-teardown logic.
datasetDao.delete(dataset.getId());
}
}
use of bio.terra.model.SnapshotSummaryModel in project jade-data-repo by DataBiosphere.
the class SnapshotConnectedTest method testOverlappingDeletes.
@Test
public void testOverlappingDeletes() throws Exception {
// create a snapshot
SnapshotSummaryModel summaryModel = connectedOperations.createSnapshot(datasetSummary, "snapshot-test-snapshot.json", "_d2_");
// NO ASSERTS inside the block below where hang is enabled to reduce chance of failing before disabling the hang
// ====================================================
// enable hang in DeleteSnapshotPrimaryDataStep
configService.setFault(ConfigEnum.SNAPSHOT_DELETE_LOCK_CONFLICT_STOP_FAULT.name(), true);
// try to delete the snapshot
MvcResult result1 = mvc.perform(delete("/api/repository/v1/snapshots/" + summaryModel.getId())).andReturn();
// try to delete the snapshot again, this should fail with a lock exception
// note: asserts are below outside the hang block
MvcResult result2 = mvc.perform(delete("/api/repository/v1/snapshots/" + summaryModel.getId())).andReturn();
// disable hang in DeleteSnapshotPrimaryDataStep
configService.setFault(ConfigEnum.SNAPSHOT_DELETE_LOCK_CONFLICT_CONTINUE_FAULT.name(), true);
// ====================================================
// check the response from the first delete request
MockHttpServletResponse response1 = connectedOperations.validateJobModelAndWait(result1);
DeleteResponseModel deleteResponseModel = connectedOperations.handleSuccessCase(response1, DeleteResponseModel.class);
assertEquals("First delete returned successfully", DeleteResponseModel.ObjectStateEnum.DELETED, deleteResponseModel.getObjectState());
// check the response from the second delete request
MockHttpServletResponse response2 = connectedOperations.validateJobModelAndWait(result2);
ErrorModel errorModel2 = connectedOperations.handleFailureCase(response2, HttpStatus.INTERNAL_SERVER_ERROR);
assertThat("delete failed on lock exception", errorModel2.getMessage(), startsWith("Failed to lock the snapshot"));
// confirm deleted
connectedOperations.getSnapshotExpectError(summaryModel.getId(), HttpStatus.NOT_FOUND);
}
use of bio.terra.model.SnapshotSummaryModel in project jade-data-repo by DataBiosphere.
the class SnapshotConnectedTest method testExcludeLockedFromSnapshotFileLookups.
@Test
public void testExcludeLockedFromSnapshotFileLookups() throws Exception {
// create a dataset
DatasetSummaryModel datasetRefSummary = createTestDataset("simple-with-filerefs-dataset.json");
// ingest a file
URI sourceUri = new URI("gs", "jade-testdata", "/fileloadprofiletest/1KBfile.txt", null, null);
String targetFilePath = "/mm/" + Names.randomizeName("testdir") + "/testExcludeLockedFromSnapshotFileLookups.txt";
FileLoadModel fileLoadModel = new FileLoadModel().sourcePath(sourceUri.toString()).description("testExcludeLockedFromSnapshotFileLookups").mimeType("text/plain").targetPath(targetFilePath).profileId(billingProfile.getId());
FileModel fileModel = connectedOperations.ingestFileSuccess(datasetRefSummary.getId(), fileLoadModel);
// generate a JSON file with the fileref
String jsonLine = "{\"name\":\"name1\", \"file_ref\":\"" + fileModel.getFileId() + "\"}\n";
// load a JSON file that contains the table rows to load into the test bucket
String jsonFileName = "this-better-pass.json";
String dirInCloud = "scratch/testExcludeLockedFromSnapshotFileLookups/" + UUID.randomUUID().toString();
BlobInfo ingestTableBlob = BlobInfo.newBuilder(testConfig.getIngestbucket(), dirInCloud + "/" + jsonFileName).build();
Storage storage = StorageOptions.getDefaultInstance().getService();
storage.create(ingestTableBlob, jsonLine.getBytes(StandardCharsets.UTF_8));
// make sure the JSON file gets cleaned up on test teardown
connectedOperations.addScratchFile(dirInCloud + "/" + jsonFileName);
// ingest the tabular data from the JSON file we just generated
String gsPath = "gs://" + testConfig.getIngestbucket() + "/" + dirInCloud + "/" + jsonFileName;
IngestRequestModel ingestRequest1 = new IngestRequestModel().format(IngestRequestModel.FormatEnum.JSON).table("tableA").path(gsPath);
connectedOperations.ingestTableSuccess(datasetRefSummary.getId(), ingestRequest1);
// create a snapshot
SnapshotSummaryModel snapshotSummary = connectedOperations.createSnapshot(datasetRefSummary, "simple-with-filerefs-snapshot.json", "");
// check that the snapshot metadata row is unlocked
String exclusiveLock = snapshotDao.getExclusiveLockState(UUID.fromString(snapshotSummary.getId()));
assertNull("snapshot row is unlocked", exclusiveLock);
String fileUri = getFileRefIdFromSnapshot(snapshotSummary);
DrsId drsId = drsIdService.fromUri(fileUri);
DRSObject drsObject = connectedOperations.drsGetObjectSuccess(drsId.toDrsObjectId(), false);
String filePath = drsObject.getAliases().get(0);
// lookup the snapshot file by DRS id, make sure it's returned (lookupSnapshotFileSuccess will already check)
FileModel fsObjById = connectedOperations.lookupSnapshotFileSuccess(snapshotSummary.getId(), drsId.getFsObjectId());
assertEquals("Retrieve snapshot file by DRS id matches desc", fsObjById.getDescription(), fileLoadModel.getDescription());
// lookup the snapshot file by DRS path and check that it's found
FileModel fsObjByPath = connectedOperations.lookupSnapshotFileByPathSuccess(snapshotSummary.getId(), filePath, 0);
assertEquals("Retrieve snapshot file by path matches desc", fsObjByPath.getDescription(), fileLoadModel.getDescription());
assertThat("Retrieve snapshot file objects match", fsObjById, CoreMatchers.equalTo(fsObjByPath));
// now the snapshot exists....let's get it locked!
// NO ASSERTS inside the block below where hang is enabled to reduce chance of failing before disabling the hang
// ====================================================
// enable hang in DeleteSnapshotPrimaryDataStep
configService.setFault(ConfigEnum.SNAPSHOT_DELETE_LOCK_CONFLICT_STOP_FAULT.name(), true);
// kick off a request to delete the snapshot. this should hang before unlocking the snapshot object.
// note: asserts are below outside the hang block
MvcResult deleteResult = mvc.perform(delete("/api/repository/v1/snapshots/" + snapshotSummary.getId())).andReturn();
// give the flight time to launch and get to the hang
TimeUnit.SECONDS.sleep(5);
// check that the snapshot metadata row has an exclusive lock
exclusiveLock = snapshotDao.getExclusiveLockState(UUID.fromString(snapshotSummary.getId()));
// lookup the snapshot file by id and check that it's NOT found
MockHttpServletResponse failedGetSnapshotByIdResponse = connectedOperations.lookupSnapshotFileRaw(snapshotSummary.getId(), drsId.getFsObjectId());
// lookup the snapshot file by path and check that it's NOT found
MockHttpServletResponse failedGetSnapshotByPathResponse = connectedOperations.lookupSnapshotFileByPathRaw(snapshotSummary.getId(), filePath, 0);
// disable hang in DeleteSnapshotPrimaryDataStep
configService.setFault(ConfigEnum.SNAPSHOT_DELETE_LOCK_CONFLICT_CONTINUE_FAULT.name(), true);
// ====================================================
// check that the snapshot metadata row has an exclusive lock after kicking off the delete
assertNotNull("snapshot row is exclusively locked", exclusiveLock);
assertEquals("Snapshot file NOT found by DRS id lookup", HttpStatus.NOT_FOUND, HttpStatus.valueOf(failedGetSnapshotByIdResponse.getStatus()));
assertEquals("Snapshot file NOT found by path lookup", HttpStatus.NOT_FOUND, HttpStatus.valueOf(failedGetSnapshotByPathResponse.getStatus()));
// check the response from the snapshot delete request
MockHttpServletResponse deleteResponse = connectedOperations.validateJobModelAndWait(deleteResult);
DeleteResponseModel deleteResponseModel = connectedOperations.handleSuccessCase(deleteResponse, DeleteResponseModel.class);
assertEquals("Snapshot delete returned successfully", DeleteResponseModel.ObjectStateEnum.DELETED, deleteResponseModel.getObjectState());
// delete the dataset and check that it succeeds
connectedOperations.deleteTestDataset(datasetRefSummary.getId());
// remove the file from the connectedoperation bookkeeping list
connectedOperations.removeFile(datasetRefSummary.getId(), fileModel.getFileId());
// try to fetch the snapshot again and confirm nothing is returned
connectedOperations.getSnapshotExpectError(snapshotSummary.getId(), HttpStatus.NOT_FOUND);
// try to fetch the dataset again and confirm nothing is returned
connectedOperations.getDatasetExpectError(datasetRefSummary.getId(), HttpStatus.NOT_FOUND);
}
Aggregations