Examples with SingleTypeModule co.cask.cdap.data2.dataset2.SingleTypeModule used on opensource projects

Example 1 with SingleTypeModule

use of co.cask.cdap.data2.dataset2.SingleTypeModule in project cdap by caskdata.

the class DatasetModulesDeployer method loadAndDeployModule.

private void loadAndDeployModule(ClassLoader artifactClassLoader, String className, Location jarLocation, String moduleName, NamespaceId namespaceId) throws ClassNotFoundException, IllegalAccessException, InstantiationException, DatasetManagementException {
    // note: using app class loader to load module class
    @SuppressWarnings("unchecked") Class<Dataset> clazz = (Class<Dataset>) artifactClassLoader.loadClass(className);
    try {
        // note: we can deploy module or create module from Dataset class
        // note: it seems dangerous to instantiate dataset module here, but this will be fine when we move deploy into
        //       isolated user's environment (e.g. separate yarn container)
        DatasetModuleId moduleId = namespaceId.datasetModule(moduleName);
        DatasetModule module;
        if (DatasetModule.class.isAssignableFrom(clazz)) {
            module = (DatasetModule) clazz.newInstance();
        } else if (Dataset.class.isAssignableFrom(clazz)) {
            if (systemDatasetFramework.hasSystemType(clazz.getName())) {
                return;
            }
            DatasetTypeId typeId = namespaceId.datasetType(clazz.getName());
            if (datasetFramework.hasType(typeId) && !allowDatasetUncheckedUpgrade) {
                return;
            }
            module = new SingleTypeModule(clazz);
        } else {
            throw new IllegalArgumentException(String.format("Cannot use class %s to add dataset module: it must be of type DatasetModule or Dataset", clazz.getName()));
        }
        LOG.info("Adding module: {}", clazz.getName());
        datasetFramework.addModule(moduleId, module, jarLocation);
    } catch (ModuleConflictException e) {
        LOG.info("Conflict while deploying module {}: {}", moduleName, e.getMessage());
        throw e;
    }
}

Example 2 with SingleTypeModule

use of co.cask.cdap.data2.dataset2.SingleTypeModule in project cdap by caskdata.

the class RemoteDatasetFrameworkTest method testSystemNamespace.

// Note: Cannot have these system namespace restrictions in system namespace since we use it internally in
// DatasetMetaTable util to add modules to system namespace. However, we should definitely impose these restrictions
// in RemoteDatasetFramework.
@Test
public void testSystemNamespace() throws DatasetManagementException {
    DatasetFramework framework = getFramework();
    // Adding module to system namespace should fail
    try {
        framework.addModule(NamespaceId.SYSTEM.datasetModule("keyValue"), new SingleTypeModule(SimpleKVTable.class));
        Assert.fail("Should not be able to add a module to system namespace");
    } catch (DatasetManagementException e) {
    // expected
    }
    try {
        framework.deleteModule(NamespaceId.SYSTEM.datasetModule("orderedTable-memory"));
        Assert.fail("Should not be able to delete a default module.");
    } catch (DatasetManagementException e) {
    // expected
    }
    try {
        framework.deleteAllModules(NamespaceId.SYSTEM);
        Assert.fail("Should not be able to delete modules from system namespace");
    } catch (DatasetManagementException e) {
    // expected
    }
}

Example 3 with SingleTypeModule

use of co.cask.cdap.data2.dataset2.SingleTypeModule in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testCompositeDataset.

@Test
public void testCompositeDataset() throws Exception {
    // Configuring Dataset types
    DatasetFramework framework = getFramework();
    framework.addModule(IN_MEMORY, new InMemoryTableModule());
    framework.addModule(CORE, new CoreDatasetsModule());
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    Assert.assertFalse(framework.hasType(SIMPLE_KV_TYPE));
    framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    // Creating instance
    Assert.assertFalse(framework.hasInstance(MY_TABLE));
    framework.addInstance(SimpleKVTable.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    testCompositeDataset(framework);
    // cleanup
    framework.deleteInstance(MY_TABLE);
    framework.deleteModule(KEY_VALUE);
    framework.deleteModule(CORE);
    framework.deleteModule(IN_MEMORY);
}

Example 4 with SingleTypeModule

use of co.cask.cdap.data2.dataset2.SingleTypeModule in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testBasicManagement.

@Test
public void testBasicManagement() throws Exception {
    DatasetTypeId tableType = NAMESPACE_ID.datasetType(Table.class.getName());
    // Adding modules
    DatasetFramework framework = getFramework();
    framework.addModule(IN_MEMORY, new InMemoryTableModule());
    framework.addModule(CORE, new CoreDatasetsModule());
    framework.addModule(FILE, new FileSetModule());
    framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
    // keyvalue has been added in the system namespace
    Assert.assertTrue(framework.hasSystemType(Table.class.getName()));
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    Assert.assertTrue(framework.hasType(tableType));
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    // Creating instances
    framework.addInstance(Table.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    DatasetSpecification spec = framework.getDatasetSpec(MY_TABLE);
    Assert.assertNotNull(spec);
    Assert.assertEquals(MY_TABLE.getEntityName(), spec.getName());
    Assert.assertEquals(Table.class.getName(), spec.getType());
    framework.addInstance(Table.class.getName(), MY_TABLE2, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE2));
    // Update instances
    File baseDir = TMP_FOLDER.newFolder();
    framework.addInstance(FileSet.class.getName(), MY_DS, FileSetProperties.builder().setBasePath(baseDir.getPath()).setDataExternal(true).build());
    // this should fail because it would "internalize" external data
    try {
        framework.updateInstance(MY_DS, DatasetProperties.EMPTY);
        Assert.fail("update should have thrown instance conflict");
    } catch (InstanceConflictException e) {
    // expected
    }
    baseDir = TMP_FOLDER.newFolder();
    // this should succeed because it simply changes the external path
    framework.updateInstance(MY_DS, FileSetProperties.builder().setBasePath(baseDir.getPath()).setDataExternal(true).build());
    spec = framework.getDatasetSpec(MY_DS);
    Assert.assertNotNull(spec);
    Assert.assertEquals(baseDir.getPath(), FileSetProperties.getBasePath(spec.getProperties()));
    // cleanup
    try {
        framework.deleteAllModules(NAMESPACE_ID);
        Assert.fail("should not delete modules: there are datasets using their types");
    } catch (DatasetManagementException e) {
    // expected
    }
    // types are still there
    Assert.assertTrue(framework.hasType(tableType));
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    framework.deleteAllInstances(NAMESPACE_ID);
    Assert.assertEquals(0, framework.getInstances(NAMESPACE_ID).size());
    Assert.assertFalse(framework.hasInstance(MY_TABLE));
    Assert.assertNull(framework.getDatasetSpec(MY_TABLE));
    Assert.assertFalse(framework.hasInstance(MY_TABLE2));
    Assert.assertNull(framework.getDatasetSpec(MY_TABLE2));
    // now it should succeed
    framework.deleteAllModules(NAMESPACE_ID);
    Assert.assertTrue(framework.hasSystemType(Table.class.getName()));
    Assert.assertFalse(framework.hasType(tableType));
    Assert.assertFalse(framework.hasType(SIMPLE_KV_TYPE));
}

Example 5 with SingleTypeModule

use of co.cask.cdap.data2.dataset2.SingleTypeModule in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testDoubleCompositeDataset.

@Test
public void testDoubleCompositeDataset() throws Exception {
    // Configuring Dataset types
    DatasetFramework framework = getFramework();
    framework.addModule(IN_MEMORY, new InMemoryTableModule());
    framework.addModule(CORE, new CoreDatasetsModule());
    framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
    Assert.assertFalse(framework.hasSystemType(DoubleWrappedKVTable.class.getName()));
    Assert.assertFalse(framework.hasType(DOUBLE_KV_TYPE));
    framework.addModule(DOUBLE_KV, new SingleTypeModule(DoubleWrappedKVTable.class));
    Assert.assertTrue(framework.hasType(DOUBLE_KV_TYPE));
    Assert.assertFalse(framework.hasSystemType(DoubleWrappedKVTable.class.getName()));
    // Creating instance
    Assert.assertFalse(framework.hasInstance(MY_TABLE));
    framework.addInstance(DoubleWrappedKVTable.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    testCompositeDataset(framework);
    // cleanup
    framework.deleteInstance(MY_TABLE);
    framework.deleteModule(DOUBLE_KV);
    framework.deleteModule(KEY_VALUE);
    framework.deleteModule(CORE);
    framework.deleteModule(IN_MEMORY);
}