Examples with SnapshotModel bio.terra.model.SnapshotModel used on opensource projects

Example 1 with SnapshotModel

use of bio.terra.model.SnapshotModel in project jade-data-repo by DataBiosphere.

the class EncodeFixture method setupEncode.

// Create dataset, load files and tables. Create and return snapshot.
// Steward owns dataset; custodian is custodian on dataset; reader has access to the snapshot.
public SetupResult setupEncode(TestConfiguration.User steward, TestConfiguration.User custodian, TestConfiguration.User reader) throws Exception {
    DatasetSummaryModel datasetSummary = dataRepoFixtures.createDataset(steward, "encodefiletest-dataset.json");
    String datasetId = datasetSummary.getId();
    dataRepoFixtures.addDatasetPolicyMember(steward, datasetId, IamRole.CUSTODIAN, custodian.getEmail());
    // Parse the input data and load the files; generate revised data file
    String stewardToken = authService.getDirectAccessAuthToken(steward.getEmail());
    Storage stewardStorage = dataRepoFixtures.getStorage(stewardToken);
    BillingProfileModel billingProfile = dataRepoFixtures.createBillingProfile(steward);
    String targetPath = loadFiles(datasetSummary.getId(), billingProfile.getId(), steward, stewardStorage);
    // Load the tables
    IngestRequestModel request = dataRepoFixtures.buildSimpleIngest("file", targetPath);
    dataRepoFixtures.ingestJsonData(steward, datasetId, request);
    // Delete the targetPath file
    deleteLoadFile(steward, targetPath);
    request = dataRepoFixtures.buildSimpleIngest("donor", "encodetest/donor.json");
    dataRepoFixtures.ingestJsonData(steward, datasetId, request);
    // Delete the scratch blob
    Blob scratchBlob = stewardStorage.get(BlobId.of(testConfiguration.getIngestbucket(), targetPath));
    if (scratchBlob != null) {
        scratchBlob.delete();
    }
    // At this point, we have files and tabular data. Let's make a data snapshot!
    SnapshotSummaryModel snapshotSummary = dataRepoFixtures.createSnapshot(custodian, datasetSummary, "encodefiletest-snapshot.json");
    // TODO: Fix use of IamProviderInterface - see DR-494
    dataRepoFixtures.addSnapshotPolicyMember(custodian, snapshotSummary.getId(), IamRole.READER, reader.getEmail());
    // We wait here for SAM to sync. We expect this to take 5 minutes. It can take more as recent
    // issues have shown. We make a BigQuery request as the test to see that READER has access.
    // We need to get the snapshot, rather than the snapshot summary in order to make a query.
    // TODO: Add dataProject to SnapshotSummaryModel?
    SnapshotModel snapshotModel = dataRepoFixtures.getSnapshot(custodian, snapshotSummary.getId());
    String readerToken = authService.getDirectAccessAuthToken(reader.getEmail());
    BigQuery bigQueryReader = BigQueryFixtures.getBigQuery(snapshotModel.getDataProject(), readerToken);
    BigQueryFixtures.hasAccess(bigQueryReader, snapshotModel.getDataProject(), snapshotModel.getName());
    return new SetupResult(datasetId, snapshotSummary);
}

Example 2 with SnapshotModel

use of bio.terra.model.SnapshotModel in project jade-data-repo by DataBiosphere.

the class SnapshotTest method snapshotRowIdsHappyPathTest.

@Test
public void snapshotRowIdsHappyPathTest() throws Exception {
    // fetch rowIds from the ingested dataset by querying the participant table
    DatasetModel dataset = dataRepoFixtures.getDataset(steward(), datasetId);
    String datasetProject = dataset.getDataProject();
    String bqDatasetName = PdaoConstant.PDAO_PREFIX + dataset.getName();
    String participantTable = "participant";
    String sampleTable = "sample";
    BigQuery bigQuery = BigQueryFixtures.getBigQuery(dataset.getDataProject(), stewardToken);
    String sql = String.format("SELECT %s FROM `%s.%s.%s`", PdaoConstant.PDAO_ROW_ID_COLUMN, datasetProject, bqDatasetName, participantTable);
    TableResult participantIds = BigQueryFixtures.query(sql, bigQuery);
    List<String> participantIdList = StreamSupport.stream(participantIds.getValues().spliterator(), false).map(v -> v.get(0).getStringValue()).collect(Collectors.toList());
    sql = String.format("SELECT %s FROM `%s.%s.%s`", PdaoConstant.PDAO_ROW_ID_COLUMN, datasetProject, bqDatasetName, sampleTable);
    TableResult sampleIds = BigQueryFixtures.query(sql, bigQuery);
    List<String> sampleIdList = StreamSupport.stream(sampleIds.getValues().spliterator(), false).map(v -> v.get(0).getStringValue()).collect(Collectors.toList());
    // swap in these row ids in the request
    SnapshotRequestModel requestModel = jsonLoader.loadObject("ingest-test-snapshot-row-ids-test.json", SnapshotRequestModel.class);
    requestModel.getContents().get(0).getRowIdSpec().getTables().get(0).setRowIds(participantIdList);
    requestModel.getContents().get(0).getRowIdSpec().getTables().get(1).setRowIds(sampleIdList);
    SnapshotSummaryModel snapshotSummary = dataRepoFixtures.createSnapshotWithRequest(steward(), dataset.getName(), requestModel);
    TimeUnit.SECONDS.sleep(10);
    createdSnapshotIds.add(snapshotSummary.getId());
    SnapshotModel snapshot = dataRepoFixtures.getSnapshot(steward(), snapshotSummary.getId());
    assertEquals("new snapshot has been created", snapshot.getName(), requestModel.getName());
    assertEquals("new snapshot has the correct number of tables", requestModel.getContents().get(0).getRowIdSpec().getTables().size(), snapshot.getTables().size());
    // TODO: get the snapshot and make sure the number of rows matches with the row ids input
    assertThat("one relationship comes through", snapshot.getRelationships().size(), equalTo(1));
    assertThat("the right relationship comes through", snapshot.getRelationships().get(0).getName(), equalTo("sample_participants"));
}

Example 3 with SnapshotModel

use of bio.terra.model.SnapshotModel in project jade-data-repo by DataBiosphere.

the class SnapshotTest method snapshotByQueryHappyPathTest.

@Test
public void snapshotByQueryHappyPathTest() throws Exception {
    DatasetModel dataset = dataRepoFixtures.getDataset(steward(), datasetId);
    String datasetName = dataset.getName();
    SnapshotRequestModel requestModel = jsonLoader.loadObject("ingest-test-snapshot-query.json", SnapshotRequestModel.class);
    // swap in the correct dataset name (with the id at the end)
    requestModel.getContents().get(0).setDatasetName(datasetName);
    requestModel.getContents().get(0).getQuerySpec().setQuery("SELECT " + datasetName + ".sample.datarepo_row_id FROM " + datasetName + ".sample WHERE " + datasetName + ".sample.id ='sample6'");
    SnapshotSummaryModel snapshotSummary = dataRepoFixtures.createSnapshotWithRequest(steward(), datasetName, requestModel);
    TimeUnit.SECONDS.sleep(10);
    createdSnapshotIds.add(snapshotSummary.getId());
    SnapshotModel snapshot = dataRepoFixtures.getSnapshot(steward(), snapshotSummary.getId());
    assertEquals("new snapshot has been created", snapshot.getName(), requestModel.getName());
}

Example 4 with SnapshotModel

use of bio.terra.model.SnapshotModel in project jade-data-repo by DataBiosphere.

the class BigQueryPdaoTest method testGetFullViews.

@Test
public void testGetFullViews() throws Exception {
    Dataset dataset = readDataset("ingest-test-dataset.json");
    // Stage tabular data for ingest.
    String targetPath = "scratch/file" + UUID.randomUUID().toString() + "/";
    String bucket = testConfig.getIngestbucket();
    BlobInfo participantBlob = BlobInfo.newBuilder(bucket, targetPath + "ingest-test-participant.json").build();
    BlobInfo sampleBlob = BlobInfo.newBuilder(bucket, targetPath + "ingest-test-sample.json").build();
    BlobInfo fileBlob = BlobInfo.newBuilder(bucket, targetPath + "ingest-test-file.json").build();
    try {
        bigQueryPdao.createDataset(dataset);
        storage.create(participantBlob, readFile("ingest-test-participant.json"));
        storage.create(sampleBlob, readFile("ingest-test-sample.json"));
        storage.create(fileBlob, readFile("ingest-test-file.json"));
        // Ingest staged data into the new dataset.
        IngestRequestModel ingestRequest = new IngestRequestModel().format(IngestRequestModel.FormatEnum.JSON);
        String datasetId = dataset.getId().toString();
        connectedOperations.ingestTableSuccess(datasetId, ingestRequest.table("participant").path(gsPath(participantBlob)));
        connectedOperations.ingestTableSuccess(datasetId, ingestRequest.table("sample").path(gsPath(sampleBlob)));
        connectedOperations.ingestTableSuccess(datasetId, ingestRequest.table("file").path(gsPath(fileBlob)));
        // Create a full-view snapshot!
        DatasetSummaryModel datasetSummary = DatasetJsonConversion.datasetSummaryModelFromDatasetSummary(dataset.getDatasetSummary());
        SnapshotSummaryModel snapshotSummary = connectedOperations.createSnapshot(datasetSummary, "snapshot-fullviews-test-snapshot.json", "");
        SnapshotModel snapshot = connectedOperations.getSnapshot(snapshotSummary.getId());
        BigQueryProject bigQueryProject = TestUtils.bigQueryProjectForDatasetName(datasetDao, dataLocationService, dataset.getName());
        Assert.assertThat(snapshot.getTables().size(), is(equalTo(3)));
        List<String> participantIds = queryForIds(snapshot.getName(), "participant", bigQueryProject);
        List<String> sampleIds = queryForIds(snapshot.getName(), "sample", bigQueryProject);
        List<String> fileIds = queryForIds(snapshot.getName(), "file", bigQueryProject);
        Assert.assertThat(participantIds, containsInAnyOrder("participant_1", "participant_2", "participant_3", "participant_4", "participant_5"));
        Assert.assertThat(sampleIds, containsInAnyOrder("sample1", "sample2", "sample3", "sample4", "sample5", "sample6", "sample7"));
        Assert.assertThat(fileIds, is(equalTo(Collections.singletonList("file1"))));
    } finally {
        storage.delete(participantBlob.getBlobId(), sampleBlob.getBlobId(), fileBlob.getBlobId());
        bigQueryPdao.deleteDataset(dataset);
        // Need to manually clean up the DAO because `readDataset` bypasses the
        // `connectedOperations` object, so we can't rely on its auto-teardown logic.
        datasetDao.delete(dataset.getId());
    }
}

Example 5 with SnapshotModel

use of bio.terra.model.SnapshotModel in project jade-data-repo by DataBiosphere.

the class SnapshotConnectedTest method getTestSnapshot.

private SnapshotModel getTestSnapshot(String id, SnapshotRequestModel snapshotRequest, DatasetSummaryModel datasetSummary) throws Exception {
    MvcResult result = mvc.perform(get("/api/repository/v1/snapshots/" + id)).andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8_VALUE)).andReturn();
    MockHttpServletResponse response = result.getResponse();
    SnapshotModel snapshotModel = objectMapper.readValue(response.getContentAsString(), SnapshotModel.class);
    assertThat(snapshotModel.getDescription(), equalTo(snapshotRequest.getDescription()));
    assertThat(snapshotModel.getName(), startsWith(snapshotRequest.getName()));
    assertThat("source array has one element", snapshotModel.getSource().size(), equalTo(1));
    SnapshotSourceModel sourceModel = snapshotModel.getSource().get(0);
    assertThat("snapshot dataset summary is the same as from dataset", sourceModel.getDataset(), equalTo(datasetSummary));
    return snapshotModel;
}