Examples with Dataset bio.terra.service.dataset.Dataset used on opensource projects

Example 26 with Dataset

use of bio.terra.service.dataset.Dataset in project jade-data-repo by DataBiosphere.

the class BigQueryPdaoTest method readDataset.

// NOTE: This method bypasses the `connectedOperations` object, and creates a dataset
// using lower-level method calls. This means that the dataset entry isn't auto-cleaned
// as part of `connectedOperations.teardown()`. If you forget to manually delete any
// datasets from the DAO at the end of a test, you'll see a FK violation when `connectedOperations`
// tries to delete the resource profile generated in `setup()`.
private Dataset readDataset(String requestFile) throws Exception {
    DatasetRequestModel datasetRequest = jsonLoader.loadObject(requestFile, DatasetRequestModel.class);
    datasetRequest.defaultProfileId(profileModel.getId()).name(datasetName());
    Dataset dataset = DatasetUtils.convertRequestWithGeneratedNames(datasetRequest);
    String createFlightId = UUID.randomUUID().toString();
    datasetDao.createAndLock(dataset, createFlightId);
    datasetDao.unlockExclusive(dataset.getId(), createFlightId);
    dataLocationService.getOrCreateProject(dataset);
    return dataset;
}

Example 27 with Dataset

use of bio.terra.service.dataset.Dataset in project jade-data-repo by DataBiosphere.

the class BigQueryPdaoTest method basicTest.

@Test
public void basicTest() throws Exception {
    Dataset dataset = readDataset("ingest-test-dataset.json");
    try {
        assertThatDatasetAndTablesShouldExist(dataset, false);
        bigQueryPdao.createDataset(dataset);
        assertThatDatasetAndTablesShouldExist(dataset, true);
        // Perform the redo, which should delete and re-create
        bigQueryPdao.createDataset(dataset);
        assertThatDatasetAndTablesShouldExist(dataset, true);
        // Now delete it and test that it is gone
        bigQueryPdao.deleteDataset(dataset);
        assertThatDatasetAndTablesShouldExist(dataset, false);
    } finally {
        datasetDao.delete(dataset.getId());
    }
}

Example 28 with Dataset

use of bio.terra.service.dataset.Dataset in project jade-data-repo by DataBiosphere.

the class BigQueryPdaoTest method testGetFullViews.

@Test
public void testGetFullViews() throws Exception {
    Dataset dataset = readDataset("ingest-test-dataset.json");
    // Stage tabular data for ingest.
    String targetPath = "scratch/file" + UUID.randomUUID().toString() + "/";
    String bucket = testConfig.getIngestbucket();
    BlobInfo participantBlob = BlobInfo.newBuilder(bucket, targetPath + "ingest-test-participant.json").build();
    BlobInfo sampleBlob = BlobInfo.newBuilder(bucket, targetPath + "ingest-test-sample.json").build();
    BlobInfo fileBlob = BlobInfo.newBuilder(bucket, targetPath + "ingest-test-file.json").build();
    try {
        bigQueryPdao.createDataset(dataset);
        storage.create(participantBlob, readFile("ingest-test-participant.json"));
        storage.create(sampleBlob, readFile("ingest-test-sample.json"));
        storage.create(fileBlob, readFile("ingest-test-file.json"));
        // Ingest staged data into the new dataset.
        IngestRequestModel ingestRequest = new IngestRequestModel().format(IngestRequestModel.FormatEnum.JSON);
        String datasetId = dataset.getId().toString();
        connectedOperations.ingestTableSuccess(datasetId, ingestRequest.table("participant").path(gsPath(participantBlob)));
        connectedOperations.ingestTableSuccess(datasetId, ingestRequest.table("sample").path(gsPath(sampleBlob)));
        connectedOperations.ingestTableSuccess(datasetId, ingestRequest.table("file").path(gsPath(fileBlob)));
        // Create a full-view snapshot!
        DatasetSummaryModel datasetSummary = DatasetJsonConversion.datasetSummaryModelFromDatasetSummary(dataset.getDatasetSummary());
        SnapshotSummaryModel snapshotSummary = connectedOperations.createSnapshot(datasetSummary, "snapshot-fullviews-test-snapshot.json", "");
        SnapshotModel snapshot = connectedOperations.getSnapshot(snapshotSummary.getId());
        BigQueryProject bigQueryProject = TestUtils.bigQueryProjectForDatasetName(datasetDao, dataLocationService, dataset.getName());
        Assert.assertThat(snapshot.getTables().size(), is(equalTo(3)));
        List<String> participantIds = queryForIds(snapshot.getName(), "participant", bigQueryProject);
        List<String> sampleIds = queryForIds(snapshot.getName(), "sample", bigQueryProject);
        List<String> fileIds = queryForIds(snapshot.getName(), "file", bigQueryProject);
        Assert.assertThat(participantIds, containsInAnyOrder("participant_1", "participant_2", "participant_3", "participant_4", "participant_5"));
        Assert.assertThat(sampleIds, containsInAnyOrder("sample1", "sample2", "sample3", "sample4", "sample5", "sample6", "sample7"));
        Assert.assertThat(fileIds, is(equalTo(Collections.singletonList("file1"))));
    } finally {
        storage.delete(participantBlob.getBlobId(), sampleBlob.getBlobId(), fileBlob.getBlobId());
        bigQueryPdao.deleteDataset(dataset);
        // Need to manually clean up the DAO because `readDataset` bypasses the
        // `connectedOperations` object, so we can't rely on its auto-teardown logic.
        datasetDao.delete(dataset.getId());
    }
}

Example 29 with Dataset

use of bio.terra.service.dataset.Dataset in project jade-data-repo by DataBiosphere.

the class BigQueryPdaoTest method testBadSoftDeletePath.

@Test
public void testBadSoftDeletePath() throws Exception {
    Dataset dataset = readDataset("ingest-test-dataset.json");
    String suffix = UUID.randomUUID().toString().replaceAll("-", "");
    String badGsUri = String.format("gs://%s/not/a/real/path/to/*files", testConfig.getIngestbucket());
    try {
        bigQueryPdao.createDataset(dataset);
        exceptionGrabber.expect(BadExternalFileException.class);
        bigQueryPdao.createSoftDeleteExternalTable(dataset, badGsUri, "participant", suffix);
    } finally {
        bigQueryPdao.deleteDataset(dataset);
        // Need to manually clean up the DAO because `readDataset` bypasses the
        // `connectedOperations` object, so we can't rely on its auto-teardown logic.
        datasetDao.delete(dataset.getId());
    }
}

Example 30 with Dataset

use of bio.terra.service.dataset.Dataset in project jade-data-repo by DataBiosphere.

the class OneProjectPerProfileIdSelectorTest method shouldGetCorrectIdForDataset.

@Test
public void shouldGetCorrectIdForDataset() throws Exception {
    String coreBillingAccountId = resourceConfiguration.getCoreBillingAccount();
    String profileName = ProfileFixtures.randomHex(16);
    BillingProfileRequestModel billingProfileRequestModel = ProfileFixtures.randomBillingProfileRequest().billingAccountId(coreBillingAccountId).profileName(profileName);
    BillingProfileModel profile = profileService.createProfile(billingProfileRequestModel);
    DatasetSummaryModel datasetSummaryModel = connectedOperations.createDataset(profile, "dataset-minimal.json");
    Dataset dataset = datasetService.retrieve(UUID.fromString(datasetSummaryModel.getId()));
    String projectId = oneProjectPerProfileIdSelector.projectIdForDataset(dataset);
    String expectedProfileId = resourceConfiguration.getProjectId() + "-" + profileName;
    assertThat("Project ID is what we expect", projectId, equalTo(expectedProfileId));
}