use of co.cask.cdap.data2.dataset2.lib.table.CoreDatasetsModule in project cdap by caskdata.
the class AbstractDatasetFrameworkTest method testCompositeDataset.
@Test
public void testCompositeDataset() throws Exception {
// Configuring Dataset types
DatasetFramework framework = getFramework();
framework.addModule(IN_MEMORY, new InMemoryTableModule());
framework.addModule(CORE, new CoreDatasetsModule());
Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
Assert.assertFalse(framework.hasType(SIMPLE_KV_TYPE));
framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
// Creating instance
Assert.assertFalse(framework.hasInstance(MY_TABLE));
framework.addInstance(SimpleKVTable.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
Assert.assertTrue(framework.hasInstance(MY_TABLE));
testCompositeDataset(framework);
// cleanup
framework.deleteInstance(MY_TABLE);
framework.deleteModule(KEY_VALUE);
framework.deleteModule(CORE);
framework.deleteModule(IN_MEMORY);
}
use of co.cask.cdap.data2.dataset2.lib.table.CoreDatasetsModule in project cdap by caskdata.
the class AbstractDatasetFrameworkTest method testBasicManagement.
@Test
public void testBasicManagement() throws Exception {
DatasetTypeId tableType = NAMESPACE_ID.datasetType(Table.class.getName());
// Adding modules
DatasetFramework framework = getFramework();
framework.addModule(IN_MEMORY, new InMemoryTableModule());
framework.addModule(CORE, new CoreDatasetsModule());
framework.addModule(FILE, new FileSetModule());
framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
// keyvalue has been added in the system namespace
Assert.assertTrue(framework.hasSystemType(Table.class.getName()));
Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
Assert.assertTrue(framework.hasType(tableType));
Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
// Creating instances
framework.addInstance(Table.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
Assert.assertTrue(framework.hasInstance(MY_TABLE));
DatasetSpecification spec = framework.getDatasetSpec(MY_TABLE);
Assert.assertNotNull(spec);
Assert.assertEquals(MY_TABLE.getEntityName(), spec.getName());
Assert.assertEquals(Table.class.getName(), spec.getType());
framework.addInstance(Table.class.getName(), MY_TABLE2, DatasetProperties.EMPTY);
Assert.assertTrue(framework.hasInstance(MY_TABLE2));
// Update instances
File baseDir = TMP_FOLDER.newFolder();
framework.addInstance(FileSet.class.getName(), MY_DS, FileSetProperties.builder().setBasePath(baseDir.getPath()).setDataExternal(true).build());
// this should fail because it would "internalize" external data
try {
framework.updateInstance(MY_DS, DatasetProperties.EMPTY);
Assert.fail("update should have thrown instance conflict");
} catch (InstanceConflictException e) {
// expected
}
baseDir = TMP_FOLDER.newFolder();
// this should succeed because it simply changes the external path
framework.updateInstance(MY_DS, FileSetProperties.builder().setBasePath(baseDir.getPath()).setDataExternal(true).build());
spec = framework.getDatasetSpec(MY_DS);
Assert.assertNotNull(spec);
Assert.assertEquals(baseDir.getPath(), FileSetProperties.getBasePath(spec.getProperties()));
// cleanup
try {
framework.deleteAllModules(NAMESPACE_ID);
Assert.fail("should not delete modules: there are datasets using their types");
} catch (DatasetManagementException e) {
// expected
}
// types are still there
Assert.assertTrue(framework.hasType(tableType));
Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
framework.deleteAllInstances(NAMESPACE_ID);
Assert.assertEquals(0, framework.getInstances(NAMESPACE_ID).size());
Assert.assertFalse(framework.hasInstance(MY_TABLE));
Assert.assertNull(framework.getDatasetSpec(MY_TABLE));
Assert.assertFalse(framework.hasInstance(MY_TABLE2));
Assert.assertNull(framework.getDatasetSpec(MY_TABLE2));
// now it should succeed
framework.deleteAllModules(NAMESPACE_ID);
Assert.assertTrue(framework.hasSystemType(Table.class.getName()));
Assert.assertFalse(framework.hasType(tableType));
Assert.assertFalse(framework.hasType(SIMPLE_KV_TYPE));
}
use of co.cask.cdap.data2.dataset2.lib.table.CoreDatasetsModule in project cdap by caskdata.
the class AbstractDatasetFrameworkTest method testDoubleCompositeDataset.
@Test
public void testDoubleCompositeDataset() throws Exception {
// Configuring Dataset types
DatasetFramework framework = getFramework();
framework.addModule(IN_MEMORY, new InMemoryTableModule());
framework.addModule(CORE, new CoreDatasetsModule());
framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
Assert.assertFalse(framework.hasSystemType(DoubleWrappedKVTable.class.getName()));
Assert.assertFalse(framework.hasType(DOUBLE_KV_TYPE));
framework.addModule(DOUBLE_KV, new SingleTypeModule(DoubleWrappedKVTable.class));
Assert.assertTrue(framework.hasType(DOUBLE_KV_TYPE));
Assert.assertFalse(framework.hasSystemType(DoubleWrappedKVTable.class.getName()));
// Creating instance
Assert.assertFalse(framework.hasInstance(MY_TABLE));
framework.addInstance(DoubleWrappedKVTable.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
Assert.assertTrue(framework.hasInstance(MY_TABLE));
testCompositeDataset(framework);
// cleanup
framework.deleteInstance(MY_TABLE);
framework.deleteModule(DOUBLE_KV);
framework.deleteModule(KEY_VALUE);
framework.deleteModule(CORE);
framework.deleteModule(IN_MEMORY);
}
use of co.cask.cdap.data2.dataset2.lib.table.CoreDatasetsModule in project cdap by caskdata.
the class SystemDatasetRuntimeModule method bindDefaultModules.
/**
* Add bindings for Dataset modules that are available by default
*/
private void bindDefaultModules(MapBinder<String, DatasetModule> mapBinder) {
mapBinder.addBinding("core").toInstance(new CoreDatasetsModule());
mapBinder.addBinding("fileSet").toInstance(new FileSetModule());
mapBinder.addBinding("timePartitionedFileSet").toInstance(new TimePartitionedFileSetModule());
mapBinder.addBinding("partitionedFileSet").toInstance(new PartitionedFileSetModule());
mapBinder.addBinding("objectMappedTable").toInstance(new ObjectMappedTableModule());
mapBinder.addBinding("cube").toInstance(new CubeModule());
mapBinder.addBinding("usage").toInstance(new UsageDatasetModule());
mapBinder.addBinding("metadata").toInstance(new MetadataDatasetModule());
mapBinder.addBinding("lineage").toInstance(new LineageDatasetModule());
mapBinder.addBinding("externalDataset").toInstance(new ExternalDatasetModule());
}
use of co.cask.cdap.data2.dataset2.lib.table.CoreDatasetsModule in project cdap by caskdata.
the class AbstractDatasetFrameworkTest method testMultipleTransitiveDependencies.
@Test
public void testMultipleTransitiveDependencies() throws DatasetManagementException, IOException {
// Adding modules
DatasetFramework framework = getFramework();
try {
framework.addModule(IN_MEMORY, new InMemoryTableModule());
framework.addModule(CORE, new CoreDatasetsModule());
framework.addModule(FILE, new FileSetModule());
framework.addModule(PFS, new PartitionedFileSetModule());
framework.addModule(TWICE, new SingleTypeModule(EmbedsTableTwiceDataset.class));
// Creating an instances
framework.addInstance(EmbedsTableTwiceDataset.class.getName(), MY_DS, PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addStringField("x").build()).build());
Assert.assertTrue(framework.hasInstance(MY_DS));
framework.getDataset(MY_DS, DatasetProperties.EMPTY.getProperties(), null);
} finally {
framework.deleteAllInstances(NAMESPACE_ID);
framework.deleteAllModules(NAMESPACE_ID);
}
}
Aggregations