Examples with DatasetId co.cask.cdap.proto.id.DatasetId used on opensource projects

Example 96 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class HBaseMetricsTableTest method getTable.

@Override
protected MetricsTable getTable(String name) throws Exception {
    DatasetId metricsDatasetInstanceId = NamespaceId.SYSTEM.dataset(name);
    DatasetProperties props = TableProperties.builder().setReadlessIncrementSupport(true).build();
    return DatasetsUtil.getOrCreateDataset(dsFramework, metricsDatasetInstanceId, MetricsTable.class.getName(), props, null);
}

Example 97 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class DatasetSystemMetadataWriterTest method assertDatasetSchema.

private void assertDatasetSchema(String expected, DatasetProperties properties) {
    DatasetSystemMetadataWriter metadataWriter = new DatasetSystemMetadataWriter(new NoOpMetadataStore(), new DatasetId("ns1", "avro1"), properties, null, null, null);
    Assert.assertEquals(expected, metadataWriter.getSchemaToAdd());
}

Example 98 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class UsageDatasetTest method testProgramDatasetMapping.

@Test
public void testProgramDatasetMapping() throws Exception {
    final UsageDataset usageDataset = getUsageDataset("testProgramDatasetMapping");
    TransactionExecutor txnl = dsFrameworkUtil.newInMemoryTransactionExecutor((TransactionAware) usageDataset);
    // Add mappings
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            usageDataset.register(flow11, datasetInstance1);
            usageDataset.register(flow11, datasetInstance3);
            usageDataset.register(flow12, datasetInstance2);
            usageDataset.register(service11, datasetInstance1);
            usageDataset.register(flow21, datasetInstance2);
            usageDataset.register(service21, datasetInstance1);
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // Verify program mappings
            Assert.assertEquals(ImmutableSet.of(datasetInstance1, datasetInstance3), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.of(datasetInstance2), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.of(datasetInstance1), usageDataset.getDatasets(service11));
            Assert.assertEquals(ImmutableSet.of(datasetInstance2), usageDataset.getDatasets(flow21));
            Assert.assertEquals(ImmutableSet.of(datasetInstance1), usageDataset.getDatasets(service21));
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.of(datasetInstance1, datasetInstance2, datasetInstance3), usageDataset.getDatasets(flow11.getParent()));
            Assert.assertEquals(ImmutableSet.of(datasetInstance1, datasetInstance2), usageDataset.getDatasets(flow21.getParent()));
            // Verify dataset mappings
            Assert.assertEquals(ImmutableSet.of(flow11, service11, service21), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.of(flow12, flow21), usageDataset.getPrograms(datasetInstance2));
            Assert.assertEquals(ImmutableSet.of(flow11), usageDataset.getPrograms(datasetInstance3));
        }
    });
    // --------- Delete app1 -----------
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            usageDataset.unregister(flow11.getParent());
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // There should be no mappings for programs of app1 now
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(service11));
            Assert.assertEquals(ImmutableSet.of(datasetInstance2), usageDataset.getDatasets(flow21));
            Assert.assertEquals(ImmutableSet.of(datasetInstance1), usageDataset.getDatasets(service21));
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow11.getParent()));
            Assert.assertEquals(ImmutableSet.of(datasetInstance1, datasetInstance2), usageDataset.getDatasets(flow21.getParent()));
            // Verify dataset mappings
            Assert.assertEquals(ImmutableSet.of(service21), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.of(flow21), usageDataset.getPrograms(datasetInstance2));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(datasetInstance3));
        }
    });
}

Example 99 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class HiveExploreServiceTestRun method testDeployNotRecordScannable.

@Test
public void testDeployNotRecordScannable() throws Exception {
    // Try to deploy a dataset that is not record scannable, when explore is enabled.
    // This should be processed with no exception being thrown
    DatasetModuleId module2 = NAMESPACE_ID.datasetModule("module2");
    DatasetId myTableNotRecordScannable = NAMESPACE_ID.dataset("my_table_not_record_scannable");
    datasetFramework.addModule(module2, new NotRecordScannableTableDefinition.NotRecordScannableTableModule());
    datasetFramework.addInstance("NotRecordScannableTableDef", myTableNotRecordScannable, DatasetProperties.EMPTY);
    datasetFramework.deleteInstance(myTableNotRecordScannable);
    datasetFramework.deleteModule(module2);
}

Example 100 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class HiveExploreServiceTestRun method previewResultsTest.

@Test
public void previewResultsTest() throws Exception {
    DatasetId myTable2 = NAMESPACE_ID.dataset("my_table_2");
    DatasetId myTable3 = NAMESPACE_ID.dataset("my_table_3");
    DatasetId myTable4 = NAMESPACE_ID.dataset("my_table_4");
    DatasetId myTable5 = NAMESPACE_ID.dataset("my_table_5");
    DatasetId myTable6 = NAMESPACE_ID.dataset("my_table_6");
    datasetFramework.addInstance("keyStructValueTable", myTable2, DatasetProperties.EMPTY);
    datasetFramework.addInstance("keyStructValueTable", myTable3, DatasetProperties.EMPTY);
    datasetFramework.addInstance("keyStructValueTable", myTable4, DatasetProperties.EMPTY);
    datasetFramework.addInstance("keyStructValueTable", myTable5, DatasetProperties.EMPTY);
    datasetFramework.addInstance("keyStructValueTable", myTable6, DatasetProperties.EMPTY);
    try {
        QueryHandle handle = exploreService.execute(NAMESPACE_ID, "show tables");
        QueryStatus status = waitForCompletionStatus(handle, 200, TimeUnit.MILLISECONDS, 50);
        Assert.assertEquals(QueryStatus.OpStatus.FINISHED, status.getStatus());
        List<QueryResult> firstPreview = exploreService.previewResults(handle);
        Assert.assertEquals(ImmutableList.of(new QueryResult(ImmutableList.<Object>of(MY_TABLE_NAME)), new QueryResult(ImmutableList.<Object>of(getDatasetHiveName(myTable2))), new QueryResult(ImmutableList.<Object>of(getDatasetHiveName(myTable3))), new QueryResult(ImmutableList.<Object>of(getDatasetHiveName(myTable4))), new QueryResult(ImmutableList.<Object>of(getDatasetHiveName(myTable5)))), firstPreview);
        // test that preview results do not change even when the query cursor is updated by nextResults
        List<QueryResult> endResults = exploreService.nextResults(handle, 100);
        Assert.assertEquals(ImmutableList.of(new QueryResult(ImmutableList.<Object>of(getDatasetHiveName(myTable6)))), endResults);
        List<QueryResult> secondPreview = exploreService.previewResults(handle);
        Assert.assertEquals(firstPreview, secondPreview);
        Assert.assertEquals(ImmutableList.<QueryResult>of(), exploreService.nextResults(handle, 100));
        try {
            // All results are fetched, query should be inactive now
            exploreService.previewResults(handle);
            Assert.fail("HandleNotFoundException expected - query should be inactive.");
        } catch (HandleNotFoundException e) {
            Assert.assertTrue(e.isInactive());
        // Expected exception
        }
        // now test preview on a query that doesn't return any results
        handle = exploreService.execute(NAMESPACE_ID, "select * from " + getDatasetHiveName(myTable2));
        status = waitForCompletionStatus(handle, 200, TimeUnit.MILLISECONDS, 50);
        Assert.assertEquals(QueryStatus.OpStatus.FINISHED, status.getStatus());
        Assert.assertTrue(exploreService.previewResults(handle).isEmpty());
        // calling preview again should return the same thing. it should not throw an exception
        Assert.assertTrue(exploreService.previewResults(handle).isEmpty());
    } finally {
        datasetFramework.deleteInstance(myTable2);
        datasetFramework.deleteInstance(myTable3);
        datasetFramework.deleteInstance(myTable4);
        datasetFramework.deleteInstance(myTable5);
        datasetFramework.deleteInstance(myTable6);
    }
}