Examples with SnapshotSummaryModel bio.terra.model.SnapshotSummaryModel used on opensource projects

Example 1 with SnapshotSummaryModel

use of bio.terra.model.SnapshotSummaryModel in project jade-data-repo by DataBiosphere.

the class CreateSnapshotMetadataStep method doStep.

@Override
public StepResult doStep(FlightContext context) {
    try {
        Snapshot snapshot = snapshotService.makeSnapshotFromSnapshotRequest(snapshotReq);
        UUID snapshotId = snapshotDao.createAndLock(snapshot, context.getFlightId());
        FlightMap workingMap = context.getWorkingMap();
        workingMap.put(SnapshotWorkingMapKeys.SNAPSHOT_ID, snapshotId);
        SnapshotSummary snapshotSummary = snapshotDao.retrieveSummaryById(snapshot.getId());
        SnapshotSummaryModel response = snapshotService.makeSummaryModelFromSummary(snapshotSummary);
        FlightUtils.setResponse(context, response, HttpStatus.CREATED);
        return StepResult.getStepResultSuccess();
    } catch (InvalidSnapshotException isEx) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_FATAL, isEx);
    } catch (SnapshotNotFoundException ex) {
        FlightUtils.setErrorResponse(context, ex.toString(), HttpStatus.BAD_REQUEST);
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_FATAL, ex);
    }
}

Example 2 with SnapshotSummaryModel

use of bio.terra.model.SnapshotSummaryModel in project jade-data-repo by DataBiosphere.

the class IngestTest method ingestBuildSnapshot.

@Test
public void ingestBuildSnapshot() throws Exception {
    IngestRequestModel ingestRequest = dataRepoFixtures.buildSimpleIngest("participant", "ingest-test/ingest-test-participant.json");
    IngestResponseModel ingestResponse = dataRepoFixtures.ingestJsonData(steward(), datasetId, ingestRequest);
    assertThat("correct participant row count", ingestResponse.getRowCount(), equalTo(5L));
    ingestRequest = dataRepoFixtures.buildSimpleIngest("sample", "ingest-test/ingest-test-sample.json");
    ingestResponse = dataRepoFixtures.ingestJsonData(steward(), datasetId, ingestRequest);
    assertThat("correct sample row count", ingestResponse.getRowCount(), equalTo(7L));
    ingestRequest = dataRepoFixtures.buildSimpleIngest("file", "ingest-test/ingest-test-file.json");
    ingestResponse = dataRepoFixtures.ingestJsonData(steward(), datasetId, ingestRequest);
    assertThat("correct file row count", ingestResponse.getRowCount(), equalTo(1L));
    SnapshotSummaryModel snapshotSummary = dataRepoFixtures.createSnapshot(custodian(), datasetSummaryModel, "ingest-test-snapshot.json");
    createdSnapshotIds.add(snapshotSummary.getId());
}

Example 3 with SnapshotSummaryModel

use of bio.terra.model.SnapshotSummaryModel in project jade-data-repo by DataBiosphere.

the class ConnectedOperations method handleCreateSnapshotSuccessCase.

public SnapshotSummaryModel handleCreateSnapshotSuccessCase(MockHttpServletResponse response) throws Exception {
    SnapshotSummaryModel summaryModel = handleSuccessCase(response, SnapshotSummaryModel.class);
    addSnapshot(summaryModel.getId());
    return summaryModel;
}

Example 4 with SnapshotSummaryModel

use of bio.terra.model.SnapshotSummaryModel in project jade-data-repo by DataBiosphere.

the class EncodeFileTest method encodeFileTest.

// NOTES ABOUT THIS TEST: this test requires create access to the jade-testdata bucket in order to
// re-write the json source data replacing the gs paths with the Jade object id.
@Test
public void encodeFileTest() throws Exception {
    DatasetSummaryModel datasetSummary = connectedOperations.createDataset(profileModel, "encodefiletest-dataset.json");
    // Load all of the files into the dataset
    String targetPath = loadFiles(datasetSummary.getId(), false, false);
    String gsPath = "gs://" + testConfig.getIngestbucket() + "/" + targetPath;
    IngestRequestModel ingestRequest = new IngestRequestModel().format(IngestRequestModel.FormatEnum.JSON).table("file").path(gsPath);
    connectedOperations.ingestTableSuccess(datasetSummary.getId(), ingestRequest);
    // Delete the scratch blob
    Blob scratchBlob = storage.get(BlobId.of(testConfig.getIngestbucket(), targetPath));
    if (scratchBlob != null) {
        scratchBlob.delete();
    }
    // Load donor success
    ingestRequest.table("donor").path("gs://" + testConfig.getIngestbucket() + "/encodetest/donor.json");
    connectedOperations.ingestTableSuccess(datasetSummary.getId(), ingestRequest);
    // At this point, we have files and tabular data. Let's make a snapshot!
    SnapshotSummaryModel snapshotSummary = connectedOperations.createSnapshot(datasetSummary, "encodefiletest-snapshot.json", "");
    String fileUri = getFileRefIdFromSnapshot(snapshotSummary);
    DrsId drsId = drsIdService.fromUri(fileUri);
    DRSObject drsObject = connectedOperations.drsGetObjectSuccess(drsId.toDrsObjectId(), false);
    String filePath = drsObject.getAliases().get(0);
    FileModel fsObjById = connectedOperations.lookupSnapshotFileSuccess(snapshotSummary.getId(), drsId.getFsObjectId());
    FileModel fsObjByPath = connectedOperations.lookupSnapshotFileByPathSuccess(snapshotSummary.getId(), filePath, 0);
    assertThat("Retrieve snapshot file objects match", fsObjById, equalTo(fsObjByPath));
    assertThat("Load tag is stored", fsObjById.getFileDetail().getLoadTag(), equalTo(loadTag));
    // Build the reference directory name map
    String datasetPath = "/" + datasetSummary.getName();
    Map<String, List<String>> dirmap = makeDirectoryMap(datasetPath);
    testSnapEnum(dirmap, snapshotSummary.getId(), datasetPath, -1);
    testSnapEnum(dirmap, snapshotSummary.getId(), datasetPath, 0);
    testSnapEnum(dirmap, snapshotSummary.getId(), datasetPath, 6);
    testSnapEnum(dirmap, snapshotSummary.getId(), datasetPath, 3);
    // Try to delete a file with a dependency
    MvcResult result = mvc.perform(delete("/api/repository/v1/datasets/" + datasetSummary.getId() + "/files/" + drsId.getFsObjectId())).andReturn();
    MockHttpServletResponse response = connectedOperations.validateJobModelAndWait(result);
    assertThat(response.getStatus(), equalTo(HttpStatus.BAD_REQUEST.value()));
    ErrorModel errorModel = connectedOperations.handleFailureCase(response);
    assertThat("correct dependency error message", errorModel.getMessage(), containsString("used by at least one snapshot"));
}

Example 5 with SnapshotSummaryModel

use of bio.terra.model.SnapshotSummaryModel in project jade-data-repo by DataBiosphere.

the class EncodeFixture method setupEncode.

// Create dataset, load files and tables. Create and return snapshot.
// Steward owns dataset; custodian is custodian on dataset; reader has access to the snapshot.
public SetupResult setupEncode(TestConfiguration.User steward, TestConfiguration.User custodian, TestConfiguration.User reader) throws Exception {
    DatasetSummaryModel datasetSummary = dataRepoFixtures.createDataset(steward, "encodefiletest-dataset.json");
    String datasetId = datasetSummary.getId();
    dataRepoFixtures.addDatasetPolicyMember(steward, datasetId, IamRole.CUSTODIAN, custodian.getEmail());
    // Parse the input data and load the files; generate revised data file
    String stewardToken = authService.getDirectAccessAuthToken(steward.getEmail());
    Storage stewardStorage = dataRepoFixtures.getStorage(stewardToken);
    BillingProfileModel billingProfile = dataRepoFixtures.createBillingProfile(steward);
    String targetPath = loadFiles(datasetSummary.getId(), billingProfile.getId(), steward, stewardStorage);
    // Load the tables
    IngestRequestModel request = dataRepoFixtures.buildSimpleIngest("file", targetPath);
    dataRepoFixtures.ingestJsonData(steward, datasetId, request);
    // Delete the targetPath file
    deleteLoadFile(steward, targetPath);
    request = dataRepoFixtures.buildSimpleIngest("donor", "encodetest/donor.json");
    dataRepoFixtures.ingestJsonData(steward, datasetId, request);
    // Delete the scratch blob
    Blob scratchBlob = stewardStorage.get(BlobId.of(testConfiguration.getIngestbucket(), targetPath));
    if (scratchBlob != null) {
        scratchBlob.delete();
    }
    // At this point, we have files and tabular data. Let's make a data snapshot!
    SnapshotSummaryModel snapshotSummary = dataRepoFixtures.createSnapshot(custodian, datasetSummary, "encodefiletest-snapshot.json");
    // TODO: Fix use of IamProviderInterface - see DR-494
    dataRepoFixtures.addSnapshotPolicyMember(custodian, snapshotSummary.getId(), IamRole.READER, reader.getEmail());
    // We wait here for SAM to sync. We expect this to take 5 minutes. It can take more as recent
    // issues have shown. We make a BigQuery request as the test to see that READER has access.
    // We need to get the snapshot, rather than the snapshot summary in order to make a query.
    // TODO: Add dataProject to SnapshotSummaryModel?
    SnapshotModel snapshotModel = dataRepoFixtures.getSnapshot(custodian, snapshotSummary.getId());
    String readerToken = authService.getDirectAccessAuthToken(reader.getEmail());
    BigQuery bigQueryReader = BigQueryFixtures.getBigQuery(snapshotModel.getDataProject(), readerToken);
    BigQueryFixtures.hasAccess(bigQueryReader, snapshotModel.getDataProject(), snapshotModel.getName());
    return new SetupResult(datasetId, snapshotSummary);
}