Examples with SingleTypeModule io.cdap.cdap.data2.dataset2.SingleTypeModule used on opensource projects

Example 6 with SingleTypeModule

use of io.cdap.cdap.data2.dataset2.SingleTypeModule in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testMultipleTransitiveDependencies.

@Test
public void testMultipleTransitiveDependencies() throws DatasetManagementException, IOException, UnauthorizedException {
    // Adding modules
    DatasetFramework framework = getFramework();
    try {
        framework.addModule(IN_MEMORY, new InMemoryTableModule());
        framework.addModule(CORE, new CoreDatasetsModule());
        framework.addModule(FILE, new FileSetModule());
        framework.addModule(PFS, new PartitionedFileSetModule());
        framework.addModule(TWICE, new SingleTypeModule(EmbedsTableTwiceDataset.class));
        // Creating an instances
        framework.addInstance(EmbedsTableTwiceDataset.class.getName(), MY_DS, PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addStringField("x").build()).build());
        Assert.assertTrue(framework.hasInstance(MY_DS));
        framework.getDataset(MY_DS, DatasetProperties.EMPTY.getProperties(), null);
    } finally {
        framework.deleteAllInstances(NAMESPACE_ID);
        framework.deleteAllModules(NAMESPACE_ID);
    }
}

Example 7 with SingleTypeModule

use of io.cdap.cdap.data2.dataset2.SingleTypeModule in project cdap by cdapio.

the class DatasetModulesDeployer method loadAndDeployModule.

private void loadAndDeployModule(ClassLoader artifactClassLoader, String className, final Location jarLocation, String moduleName, NamespaceId namespaceId, String authorizingUser) throws Exception {
    // note: using app class loader to load module class
    @SuppressWarnings("unchecked") Class<Dataset> clazz = (Class<Dataset>) artifactClassLoader.loadClass(className);
    try {
        // note: we can deploy module or create module from Dataset class
        // note: it seems dangerous to instantiate dataset module here, but this will be fine when we move deploy into
        // isolated user's environment (e.g. separate yarn container)
        final DatasetModuleId moduleId = namespaceId.datasetModule(moduleName);
        final DatasetModule module;
        if (DatasetModule.class.isAssignableFrom(clazz)) {
            module = (DatasetModule) clazz.newInstance();
        } else if (Dataset.class.isAssignableFrom(clazz)) {
            if (systemDatasetFramework.hasSystemType(clazz.getName())) {
                return;
            }
            final DatasetTypeId typeId = namespaceId.datasetType(clazz.getName());
            boolean hasType = AuthorizationUtil.authorizeAs(authorizingUser, new Callable<Boolean>() {

                @Override
                public Boolean call() throws Exception {
                    return datasetFramework.hasType(typeId);
                }
            });
            if (hasType && !allowDatasetUncheckedUpgrade) {
                return;
            }
            module = new SingleTypeModule(clazz);
        } else {
            throw new IllegalArgumentException(String.format("Cannot use class %s to add dataset module: it must be of type DatasetModule or Dataset", clazz.getName()));
        }
        LOG.info("Adding module: {}", clazz.getName());
        AuthorizationUtil.authorizeAs(authorizingUser, new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                datasetFramework.addModule(moduleId, module, jarLocation);
                return null;
            }
        });
    } catch (ModuleConflictException e) {
        LOG.info("Conflict while deploying module {}: {}", moduleName, e.getMessage());
        throw e;
    }
}

Example 8 with SingleTypeModule

use of io.cdap.cdap.data2.dataset2.SingleTypeModule in project cdap by cdapio.

the class AbstractDatasetFrameworkTest method testCompositeDataset.

@Test
public void testCompositeDataset() throws Exception {
    // Configuring Dataset types
    DatasetFramework framework = getFramework();
    framework.addModule(IN_MEMORY, new InMemoryTableModule());
    framework.addModule(CORE, new CoreDatasetsModule());
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    Assert.assertFalse(framework.hasType(SIMPLE_KV_TYPE));
    framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    // Creating instance
    Assert.assertFalse(framework.hasInstance(MY_TABLE));
    framework.addInstance(SimpleKVTable.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    testCompositeDataset(framework);
    // cleanup
    framework.deleteInstance(MY_TABLE);
    framework.deleteModule(KEY_VALUE);
    framework.deleteModule(CORE);
    framework.deleteModule(IN_MEMORY);
}

Example 9 with SingleTypeModule

use of io.cdap.cdap.data2.dataset2.SingleTypeModule in project cdap by cdapio.

the class AbstractDatasetFrameworkTest method testBasicManagement.

@Test
public void testBasicManagement() throws Exception {
    DatasetTypeId tableType = NAMESPACE_ID.datasetType(Table.class.getName());
    // Adding modules
    DatasetFramework framework = getFramework();
    framework.addModule(IN_MEMORY, new InMemoryTableModule());
    framework.addModule(CORE, new CoreDatasetsModule());
    framework.addModule(FILE, new FileSetModule());
    framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
    // keyvalue has been added in the system namespace
    Assert.assertTrue(framework.hasSystemType(Table.class.getName()));
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    Assert.assertTrue(framework.hasType(tableType));
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    // Creating instances
    framework.addInstance(Table.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    DatasetSpecification spec = framework.getDatasetSpec(MY_TABLE);
    Assert.assertNotNull(spec);
    Assert.assertEquals(MY_TABLE.getEntityName(), spec.getName());
    Assert.assertEquals(Table.class.getName(), spec.getType());
    framework.addInstance(Table.class.getName(), MY_TABLE2, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE2));
    // Update instances
    File baseDir = TMP_FOLDER.newFolder();
    framework.addInstance(FileSet.class.getName(), MY_DS, FileSetProperties.builder().setBasePath(baseDir.getPath()).setDataExternal(true).build());
    // this should fail because it would "internalize" external data
    try {
        framework.updateInstance(MY_DS, DatasetProperties.EMPTY);
        Assert.fail("update should have thrown instance conflict");
    } catch (InstanceConflictException e) {
    // expected
    }
    baseDir = TMP_FOLDER.newFolder();
    // this should succeed because it simply changes the external path
    framework.updateInstance(MY_DS, FileSetProperties.builder().setBasePath(baseDir.getPath()).setDataExternal(true).build());
    spec = framework.getDatasetSpec(MY_DS);
    Assert.assertNotNull(spec);
    Assert.assertEquals(baseDir.getPath(), FileSetProperties.getBasePath(spec.getProperties()));
    // cleanup
    try {
        framework.deleteAllModules(NAMESPACE_ID);
        Assert.fail("should not delete modules: there are datasets using their types");
    } catch (DatasetManagementException e) {
    // expected
    }
    // types are still there
    Assert.assertTrue(framework.hasType(tableType));
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    framework.deleteAllInstances(NAMESPACE_ID);
    Assert.assertEquals(0, framework.getInstances(NAMESPACE_ID).size());
    Assert.assertFalse(framework.hasInstance(MY_TABLE));
    Assert.assertNull(framework.getDatasetSpec(MY_TABLE));
    Assert.assertFalse(framework.hasInstance(MY_TABLE2));
    Assert.assertNull(framework.getDatasetSpec(MY_TABLE2));
    // now it should succeed
    framework.deleteAllModules(NAMESPACE_ID);
    Assert.assertTrue(framework.hasSystemType(Table.class.getName()));
    Assert.assertFalse(framework.hasType(tableType));
    Assert.assertFalse(framework.hasType(SIMPLE_KV_TYPE));
}

Example 10 with SingleTypeModule

use of io.cdap.cdap.data2.dataset2.SingleTypeModule in project cdap by cdapio.

the class AbstractDatasetFrameworkTest method testMultipleTransitiveDependencies.

@Test
public void testMultipleTransitiveDependencies() throws DatasetManagementException, IOException, UnauthorizedException {
    // Adding modules
    DatasetFramework framework = getFramework();
    try {
        framework.addModule(IN_MEMORY, new InMemoryTableModule());
        framework.addModule(CORE, new CoreDatasetsModule());
        framework.addModule(FILE, new FileSetModule());
        framework.addModule(PFS, new PartitionedFileSetModule());
        framework.addModule(TWICE, new SingleTypeModule(EmbedsTableTwiceDataset.class));
        // Creating an instances
        framework.addInstance(EmbedsTableTwiceDataset.class.getName(), MY_DS, PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addStringField("x").build()).build());
        Assert.assertTrue(framework.hasInstance(MY_DS));
        framework.getDataset(MY_DS, DatasetProperties.EMPTY.getProperties(), null);
    } finally {
        framework.deleteAllInstances(NAMESPACE_ID);
        framework.deleteAllModules(NAMESPACE_ID);
    }
}