Examples with DatasetModel bio.terra.model.DatasetModel used on opensource projects

Example 11 with DatasetModel

use of bio.terra.model.DatasetModel in project jade-data-repo by DataBiosphere.

the class DatasetIntegrationTest method datasetHappyPath.

@Test
public void datasetHappyPath() throws Exception {
    DatasetSummaryModel summaryModel = dataRepoFixtures.createDataset(steward(), "it-dataset-omop.json");
    datasetId = summaryModel.getId();
    logger.info("dataset id is " + summaryModel.getId());
    assertThat(summaryModel.getName(), startsWith(omopDatasetName));
    assertThat(summaryModel.getDescription(), equalTo(omopDatasetDesc));
    DatasetModel datasetModel = dataRepoFixtures.getDataset(steward(), summaryModel.getId());
    assertThat(datasetModel.getName(), startsWith(omopDatasetName));
    assertThat(datasetModel.getDescription(), equalTo(omopDatasetDesc));
    // There is a delay from when a resource is created in SAM to when it is available in an enumerate call.
    boolean metExpectation = TestUtils.eventualExpect(5, 60, true, () -> {
        EnumerateDatasetModel enumerateDatasetModel = dataRepoFixtures.enumerateDatasets(steward());
        boolean found = false;
        for (DatasetSummaryModel oneDataset : enumerateDatasetModel.getItems()) {
            if (oneDataset.getId().equals(datasetModel.getId())) {
                assertThat(oneDataset.getName(), startsWith(omopDatasetName));
                assertThat(oneDataset.getDescription(), equalTo(omopDatasetDesc));
                found = true;
                break;
            }
        }
        return found;
    });
    assertTrue("dataset was found in enumeration", metExpectation);
    // test allowable permissions
    dataRepoFixtures.addDatasetPolicyMember(steward(), summaryModel.getId(), IamRole.CUSTODIAN, custodian().getEmail());
    DataRepoResponse<EnumerateDatasetModel> enumDatasets = dataRepoFixtures.enumerateDatasetsRaw(custodian());
    assertThat("Custodian is authorized to enumerate datasets", enumDatasets.getStatusCode(), equalTo(HttpStatus.OK));
}

Example 12 with DatasetModel

use of bio.terra.model.DatasetModel in project jade-data-repo by DataBiosphere.

the class DatasetIntegrationTest method getRowIds.

private List<String> getRowIds(BigQuery bigQuery, DatasetModel dataset, String tableName, Long n) throws InterruptedException {
    String tableRef = BigQueryFixtures.makeTableRef(dataset, tableName);
    String sql = String.format("SELECT %s FROM %s LIMIT %s", PdaoConstant.PDAO_ROW_ID_COLUMN, tableRef, n);
    TableResult result = BigQueryFixtures.queryWithRetry(sql, bigQuery);
    assertThat("got right num of row ids back", result.getTotalRows(), equalTo(n));
    return StreamSupport.stream(result.getValues().spliterator(), false).map(fieldValues -> fieldValues.get(0).getStringValue()).collect(Collectors.toList());
}

Example 13 with DatasetModel

use of bio.terra.model.DatasetModel in project jade-data-repo by DataBiosphere.

the class AccessTest method checkShared.

@Test
public void checkShared() throws Exception {
    makeIngestTestDataset();
    IngestRequestModel request = dataRepoFixtures.buildSimpleIngest("participant", "ingest-test/ingest-test-participant.json");
    dataRepoFixtures.ingestJsonData(steward(), datasetId, request);
    request = dataRepoFixtures.buildSimpleIngest("sample", "ingest-test/ingest-test-sample.json");
    dataRepoFixtures.ingestJsonData(steward(), datasetId, request);
    DatasetModel dataset = dataRepoFixtures.getDataset(steward(), datasetId);
    String datasetBqSnapshotName = "datarepo_" + dataset.getName();
    BigQuery custodianBigQuery = BigQueryFixtures.getBigQuery(dataset.getDataProject(), custodianToken);
    try {
        BigQueryFixtures.datasetExists(custodianBigQuery, dataset.getDataProject(), datasetBqSnapshotName);
        fail("custodian shouldn't be able to access bq dataset before it is shared with them");
    } catch (IllegalStateException e) {
        assertThat("checking message for pdao exception error", e.getMessage(), equalTo("existence check failed for " + datasetBqSnapshotName));
    }
    dataRepoFixtures.addDatasetPolicyMember(steward(), datasetId, IamRole.CUSTODIAN, custodian().getEmail());
    DataRepoResponse<EnumerateDatasetModel> enumDatasets = dataRepoFixtures.enumerateDatasetsRaw(custodian());
    assertThat("Custodian is authorized to enumerate datasets", enumDatasets.getStatusCode(), equalTo(HttpStatus.OK));
    boolean custodianHasAccess = BigQueryFixtures.hasAccess(custodianBigQuery, dataset.getDataProject(), datasetBqSnapshotName);
    assertThat("custodian can access the bq snapshot after it has been shared", custodianHasAccess, equalTo(true));
    SnapshotSummaryModel snapshotSummaryModel = dataRepoFixtures.createSnapshot(custodian(), datasetSummaryModel, "ingest-test-snapshot.json");
    SnapshotModel snapshotModel = dataRepoFixtures.getSnapshot(custodian(), snapshotSummaryModel.getId());
    BigQuery bigQuery = BigQueryFixtures.getBigQuery(snapshotModel.getDataProject(), readerToken);
    try {
        BigQueryFixtures.datasetExists(bigQuery, snapshotModel.getDataProject(), snapshotModel.getName());
        fail("reader shouldn't be able to access bq dataset before it is shared with them");
    } catch (IllegalStateException e) {
        assertThat("checking message for exception error", e.getMessage(), equalTo("existence check failed for ".concat(snapshotSummaryModel.getName())));
    }
    dataRepoFixtures.addSnapshotPolicyMember(custodian(), snapshotSummaryModel.getId(), IamRole.READER, reader().getEmail());
    AuthenticatedUserRequest authenticatedReaderRequest = new AuthenticatedUserRequest().email(reader().getEmail()).token(Optional.of(readerToken));
    assertThat("correctly added reader", iamService.isAuthorized(authenticatedReaderRequest, IamResourceType.DATASNAPSHOT, snapshotSummaryModel.getId(), IamAction.READ_DATA), equalTo(true));
    boolean readerHasAccess = BigQueryFixtures.hasAccess(bigQuery, snapshotModel.getDataProject(), snapshotModel.getName());
    assertThat("reader can access the snapshot after it has been shared", readerHasAccess, equalTo(true));
}

Example 14 with DatasetModel

use of bio.terra.model.DatasetModel in project jade-data-repo by DataBiosphere.

the class AccessTest method checkCustodianPermissions.

@Test
public void checkCustodianPermissions() throws Exception {
    makeIngestTestDataset();
    IngestRequestModel request = dataRepoFixtures.buildSimpleIngest("participant", "ingest-test/ingest-test-participant.json");
    dataRepoFixtures.ingestJsonData(steward(), datasetId, request);
    request = dataRepoFixtures.buildSimpleIngest("sample", "ingest-test/ingest-test-sample.json");
    dataRepoFixtures.ingestJsonData(steward(), datasetId, request);
    DatasetModel dataset = dataRepoFixtures.getDataset(steward(), datasetId);
    String datasetBqSnapshotName = PdaoConstant.PDAO_PREFIX + dataset.getName();
    BigQuery custodianBigQuery = BigQueryFixtures.getBigQuery(dataset.getDataProject(), custodianToken);
    try {
        BigQueryFixtures.datasetExists(custodianBigQuery, dataset.getDataProject(), datasetBqSnapshotName);
        fail("custodian shouldn't be able to access bq dataset before it is shared with them");
    } catch (IllegalStateException e) {
        assertThat("checking message for pdao exception error", e.getMessage(), equalTo("existence check failed for " + datasetBqSnapshotName));
    }
    dataRepoFixtures.addDatasetPolicyMember(steward(), datasetId, IamRole.CUSTODIAN, custodian().getEmail());
    DataRepoResponse<EnumerateDatasetModel> enumDatasets = dataRepoFixtures.enumerateDatasetsRaw(custodian());
    assertThat("Custodian is authorized to enumerate datasets", enumDatasets.getStatusCode(), equalTo(HttpStatus.OK));
    boolean custodianHasAccess = BigQueryFixtures.hasAccess(custodianBigQuery, dataset.getDataProject(), datasetBqSnapshotName);
    assertTrue("custodian can access the bq snapshot after it has been shared", custodianHasAccess);
    // gets the "sample" table and makes a table ref to use in the query
    String tableRef = BigQueryFixtures.makeTableRef(dataset, dataset.getSchema().getTables().get(1).getName());
    String sql = String.format("SELECT * FROM %s LIMIT %s", tableRef, 1000);
    TableResult results = BigQueryFixtures.query(sql, custodianBigQuery);
    Assert.assertEquals(7, results.getTotalRows());
}

Example 15 with DatasetModel

use of bio.terra.model.DatasetModel in project jade-data-repo by DataBiosphere.

the class SnapshotTest method snapshotByFullViewHappyPathTest.

@Test
public void snapshotByFullViewHappyPathTest() throws Exception {
    DatasetModel dataset = dataRepoFixtures.getDataset(steward(), datasetId);
    String datasetName = dataset.getName();
    SnapshotRequestModel requestModel = jsonLoader.loadObject("ingest-test-snapshot-fullviews.json", SnapshotRequestModel.class);
    // swap in the correct dataset name (with the id at the end)
    requestModel.getContents().get(0).setDatasetName(datasetName);
    SnapshotSummaryModel snapshotSummary = dataRepoFixtures.createSnapshotWithRequest(steward(), datasetName, requestModel);
    TimeUnit.SECONDS.sleep(10);
    createdSnapshotIds.add(snapshotSummary.getId());
    SnapshotModel snapshot = dataRepoFixtures.getSnapshot(steward(), snapshotSummary.getId());
    assertEquals("new snapshot has been created", snapshot.getName(), requestModel.getName());
    assertEquals("all 5 relationships come through", snapshot.getRelationships().size(), 5);
}