Examples with SingleTypeModule io.cdap.cdap.data2.dataset2.SingleTypeModule used on opensource projects

Example 11 with SingleTypeModule

use of io.cdap.cdap.data2.dataset2.SingleTypeModule in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testMultipleTransitiveDependencies.

@Test
public void testMultipleTransitiveDependencies() throws DatasetManagementException, IOException {
    // Adding modules
    DatasetFramework framework = getFramework();
    try {
        framework.addModule(IN_MEMORY, new InMemoryTableModule());
        framework.addModule(CORE, new CoreDatasetsModule());
        framework.addModule(FILE, new FileSetModule());
        framework.addModule(PFS, new PartitionedFileSetModule());
        framework.addModule(TWICE, new SingleTypeModule(EmbedsTableTwiceDataset.class));
        // Creating an instances
        framework.addInstance(EmbedsTableTwiceDataset.class.getName(), MY_DS, PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addStringField("x").build()).build());
        Assert.assertTrue(framework.hasInstance(MY_DS));
        framework.getDataset(MY_DS, DatasetProperties.EMPTY.getProperties(), null);
    } finally {
        framework.deleteAllInstances(NAMESPACE_ID);
        framework.deleteAllModules(NAMESPACE_ID);
    }
}

Example 12 with SingleTypeModule

use of io.cdap.cdap.data2.dataset2.SingleTypeModule in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testDoubleCompositeDataset.

@Test
public void testDoubleCompositeDataset() throws Exception {
    // Configuring Dataset types
    DatasetFramework framework = getFramework();
    framework.addModule(IN_MEMORY, new InMemoryTableModule());
    framework.addModule(CORE, new CoreDatasetsModule());
    framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
    Assert.assertFalse(framework.hasSystemType(DoubleWrappedKVTable.class.getName()));
    Assert.assertFalse(framework.hasType(DOUBLE_KV_TYPE));
    framework.addModule(DOUBLE_KV, new SingleTypeModule(DoubleWrappedKVTable.class));
    Assert.assertTrue(framework.hasType(DOUBLE_KV_TYPE));
    Assert.assertFalse(framework.hasSystemType(DoubleWrappedKVTable.class.getName()));
    // Creating instance
    Assert.assertFalse(framework.hasInstance(MY_TABLE));
    framework.addInstance(DoubleWrappedKVTable.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    testCompositeDataset(framework);
    // cleanup
    framework.deleteInstance(MY_TABLE);
    framework.deleteModule(DOUBLE_KV);
    framework.deleteModule(KEY_VALUE);
    framework.deleteModule(CORE);
    framework.deleteModule(IN_MEMORY);
}

Example 13 with SingleTypeModule

use of io.cdap.cdap.data2.dataset2.SingleTypeModule in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testNamespaceModuleIsolation.

@Test
public void testNamespaceModuleIsolation() throws Exception {
    DatasetFramework framework = getFramework();
    // create 2 namespaces
    NamespaceId namespace1 = new NamespaceId("ns1");
    NamespaceId namespace2 = new NamespaceId("ns2");
    namespaceAdmin.create(new NamespaceMeta.Builder().setName(namespace1).build());
    namespaceAdmin.create(new NamespaceMeta.Builder().setName(namespace2).build());
    namespacePathLocator.get(namespace1).mkdirs();
    namespacePathLocator.get(namespace2).mkdirs();
    // add modules in each namespace, with one module that shares the same name
    DatasetModuleId simpleModuleNs1 = namespace1.datasetModule(SimpleKVTable.class.getName());
    DatasetModuleId simpleModuleNs2 = namespace2.datasetModule(SimpleKVTable.class.getName());
    DatasetModuleId doubleModuleNs2 = namespace2.datasetModule(DoubleWrappedKVTable.class.getName());
    DatasetModule module1 = new SingleTypeModule(SimpleKVTable.class);
    DatasetModule module2 = new SingleTypeModule(DoubleWrappedKVTable.class);
    framework.addModule(simpleModuleNs1, module1);
    framework.addModule(simpleModuleNs2, module1);
    framework.addModule(doubleModuleNs2, module2);
    // check that we can add instances of datasets in those modules
    framework.addInstance(SimpleKVTable.class.getName(), namespace1.dataset("kv1"), DatasetProperties.EMPTY);
    framework.addInstance(SimpleKVTable.class.getName(), namespace2.dataset("kv1"), DatasetProperties.EMPTY);
    // check that only namespace2 can add an instance of this type, since the module should only be in namespace2
    framework.addInstance(DoubleWrappedKVTable.class.getName(), namespace2.dataset("kv2"), DatasetProperties.EMPTY);
    try {
        framework.addInstance(DoubleWrappedKVTable.class.getName(), namespace2.dataset("kv2"), DatasetProperties.EMPTY);
        Assert.fail();
    } catch (Exception e) {
    // expected
    }
    // check that deleting all modules from namespace2 does not affect namespace1
    framework.deleteAllInstances(namespace2);
    framework.deleteAllModules(namespace2);
    // should still be able to add an instance in namespace1
    framework.addInstance(SimpleKVTable.class.getName(), namespace1.dataset("kv3"), DatasetProperties.EMPTY);
    // but not in namespace2
    try {
        framework.addInstance(SimpleKVTable.class.getName(), namespace2.dataset("kv3"), DatasetProperties.EMPTY);
        Assert.fail();
    } catch (Exception e) {
    // expected
    }
    // add back modules to namespace2
    framework.addModule(simpleModuleNs2, module1);
    framework.addModule(doubleModuleNs2, module2);
    // check that deleting a single module from namespace1 does not affect namespace2
    framework.deleteAllInstances(namespace1);
    framework.deleteModule(simpleModuleNs1);
    // should still be able to add an instance in namespace2
    framework.addInstance(DoubleWrappedKVTable.class.getName(), namespace2.dataset("kv1"), DatasetProperties.EMPTY);
    // but not in namespace1
    try {
        framework.addInstance(SimpleKVTable.class.getName(), namespace1.dataset("kv1"), DatasetProperties.EMPTY);
        Assert.fail();
    } catch (Exception e) {
    // expected
    }
}

Example 14 with SingleTypeModule

use of io.cdap.cdap.data2.dataset2.SingleTypeModule in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testSimpleDataset.

@Test
public void testSimpleDataset() throws Exception {
    // Configuring Dataset types
    DatasetFramework framework = getFramework();
    // system namespace has a module orderedTable-inMemory
    Assert.assertTrue(framework.hasSystemType("table"));
    // myspace namespace has no modules
    Assert.assertFalse(framework.hasType(IN_MEMORY_TYPE));
    Assert.assertFalse(framework.hasType(SIMPLE_KV_TYPE));
    // add module to namespace 'myspace'
    framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
    // make sure it got added to 'myspace'
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    // but not to 'system'
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    Assert.assertFalse(framework.hasInstance(MY_TABLE));
    // Creating instance using a type from own namespace
    framework.addInstance(SimpleKVTable.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    // verify it got added to the right namespace
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    // and not to the system namespace
    Assert.assertFalse(framework.hasInstance(NamespaceId.SYSTEM.dataset("my_table")));
    // Doing some admin and data ops
    DatasetAdmin admin = framework.getAdmin(MY_TABLE, null);
    Assert.assertNotNull(admin);
    final SimpleKVTable kvTable = framework.getDataset(MY_TABLE, DatasetDefinition.NO_ARGUMENTS, null);
    Assert.assertNotNull(kvTable);
    TransactionExecutor txnl = new DefaultTransactionExecutor(new MinimalTxSystemClient(), (TransactionAware) kvTable);
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            kvTable.put("key1", "value1");
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals("value1", kvTable.get("key1"));
        }
    });
    admin.truncate();
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertTrue(kvTable.get("key1") == null);
        }
    });
    // cleanup
    framework.deleteInstance(MY_TABLE);
    framework.deleteModule(KEY_VALUE);
    // recreate instance without adding a module in 'myspace'. This should use types from default namespace
    framework.addInstance("table", MY_TABLE, DatasetProperties.EMPTY);
    // verify it got added to the right namespace
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    admin = framework.getAdmin(MY_TABLE, null);
    Assert.assertNotNull(admin);
    final Table table = framework.getDataset(MY_TABLE, DatasetDefinition.NO_ARGUMENTS, null);
    Assert.assertNotNull(table);
    txnl = new DefaultTransactionExecutor(new MinimalTxSystemClient(), (TransactionAware) table);
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            table.put(Bytes.toBytes("key1"), Bytes.toBytes("column1"), Bytes.toBytes("value1"));
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals("value1", Bytes.toString(table.get(Bytes.toBytes("key1"), Bytes.toBytes("column1"))));
        }
    });
    // cleanup
    framework.deleteInstance(MY_TABLE);
}

Example 15 with SingleTypeModule

use of io.cdap.cdap.data2.dataset2.SingleTypeModule in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testBasicManagement.

@Test
public void testBasicManagement() throws Exception {
    DatasetTypeId tableType = NAMESPACE_ID.datasetType(Table.class.getName());
    // Adding modules
    DatasetFramework framework = getFramework();
    framework.addModule(IN_MEMORY, new InMemoryTableModule());
    framework.addModule(CORE, new CoreDatasetsModule());
    framework.addModule(FILE, new FileSetModule());
    framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
    // keyvalue has been added in the system namespace
    Assert.assertTrue(framework.hasSystemType(Table.class.getName()));
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    Assert.assertTrue(framework.hasType(tableType));
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    // Creating instances
    framework.addInstance(Table.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    DatasetSpecification spec = framework.getDatasetSpec(MY_TABLE);
    Assert.assertNotNull(spec);
    Assert.assertEquals(MY_TABLE.getEntityName(), spec.getName());
    Assert.assertEquals(Table.class.getName(), spec.getType());
    framework.addInstance(Table.class.getName(), MY_TABLE2, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE2));
    // Update instances
    File baseDir = TMP_FOLDER.newFolder();
    framework.addInstance(FileSet.class.getName(), MY_DS, FileSetProperties.builder().setBasePath(baseDir.getPath()).setDataExternal(true).build());
    // this should fail because it would "internalize" external data
    try {
        framework.updateInstance(MY_DS, DatasetProperties.EMPTY);
        Assert.fail("update should have thrown instance conflict");
    } catch (InstanceConflictException e) {
    // expected
    }
    baseDir = TMP_FOLDER.newFolder();
    // this should succeed because it simply changes the external path
    framework.updateInstance(MY_DS, FileSetProperties.builder().setBasePath(baseDir.getPath()).setDataExternal(true).build());
    spec = framework.getDatasetSpec(MY_DS);
    Assert.assertNotNull(spec);
    Assert.assertEquals(baseDir.getPath(), FileSetProperties.getBasePath(spec.getProperties()));
    // cleanup
    try {
        framework.deleteAllModules(NAMESPACE_ID);
        Assert.fail("should not delete modules: there are datasets using their types");
    } catch (DatasetManagementException e) {
    // expected
    }
    // types are still there
    Assert.assertTrue(framework.hasType(tableType));
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    framework.deleteAllInstances(NAMESPACE_ID);
    Assert.assertEquals(0, framework.getInstances(NAMESPACE_ID).size());
    Assert.assertFalse(framework.hasInstance(MY_TABLE));
    Assert.assertNull(framework.getDatasetSpec(MY_TABLE));
    Assert.assertFalse(framework.hasInstance(MY_TABLE2));
    Assert.assertNull(framework.getDatasetSpec(MY_TABLE2));
    // now it should succeed
    framework.deleteAllModules(NAMESPACE_ID);
    Assert.assertTrue(framework.hasSystemType(Table.class.getName()));
    Assert.assertFalse(framework.hasType(tableType));
    Assert.assertFalse(framework.hasType(SIMPLE_KV_TYPE));
}