Examples with DatasetModuleId io.cdap.cdap.proto.id.DatasetModuleId used on opensource projects

Example 41 with DatasetModuleId

use of io.cdap.cdap.proto.id.DatasetModuleId in project cdap by cdapio.

the class DefaultDatasetTypeService method deployExtensionModules.

private void deployExtensionModules() {
    // adding any defined extension modules to be available in dataset manager service
    for (Map.Entry<String, DatasetModule> module : extensionModules.entrySet()) {
        try {
            // NOTE: we assume extension modules are always in classpath, hence passing null for jar location
            // NOTE: we add extension modules in the system namespace
            DatasetModuleId theModule = NamespaceId.SYSTEM.datasetModule(module.getKey());
            typeManager.addModule(theModule, module.getValue().getClass().getName(), null, false);
        } catch (DatasetModuleConflictException e) {
            // perfectly fine: we need to add the modules only the very first time service is started
            LOG.debug("Not adding {} extension module: it already exists", module.getKey());
        } catch (Throwable th) {
            LOG.error("Failed to add {} extension module. Aborting.", module.getKey(), th);
            throw Throwables.propagate(th);
        }
    }
}

Example 42 with DatasetModuleId

use of io.cdap.cdap.proto.id.DatasetModuleId in project cdap by cdapio.

the class DefaultDatasetTypeService method deleteSystemModules.

private void deleteSystemModules() {
    TransactionRunners.run(transactionRunner, context -> {
        DatasetTypeTable datasetTypeTable = DatasetTypeTable.create(context);
        Collection<DatasetModuleMeta> allDatasets = datasetTypeTable.getModules(NamespaceId.SYSTEM);
        for (DatasetModuleMeta ds : allDatasets) {
            if (ds.getJarLocationPath() == null) {
                LOG.debug("Deleting system dataset module: {}", ds.toString());
                DatasetModuleId moduleId = NamespaceId.SYSTEM.datasetModule(ds.getName());
                datasetTypeTable.deleteModule(moduleId);
            }
        }
    });
}

Example 43 with DatasetModuleId

use of io.cdap.cdap.proto.id.DatasetModuleId in project cdap by cdapio.

the class AbstractDatasetFrameworkTest method testNamespaceModuleIsolation.

@Test
public void testNamespaceModuleIsolation() throws Exception {
    DatasetFramework framework = getFramework();
    // create 2 namespaces
    NamespaceId namespace1 = new NamespaceId("ns1");
    NamespaceId namespace2 = new NamespaceId("ns2");
    namespaceAdmin.create(new NamespaceMeta.Builder().setName(namespace1).build());
    namespaceAdmin.create(new NamespaceMeta.Builder().setName(namespace2).build());
    namespacePathLocator.get(namespace1).mkdirs();
    namespacePathLocator.get(namespace2).mkdirs();
    // add modules in each namespace, with one module that shares the same name
    DatasetModuleId simpleModuleNs1 = namespace1.datasetModule(SimpleKVTable.class.getName());
    DatasetModuleId simpleModuleNs2 = namespace2.datasetModule(SimpleKVTable.class.getName());
    DatasetModuleId doubleModuleNs2 = namespace2.datasetModule(DoubleWrappedKVTable.class.getName());
    DatasetModule module1 = new SingleTypeModule(SimpleKVTable.class);
    DatasetModule module2 = new SingleTypeModule(DoubleWrappedKVTable.class);
    framework.addModule(simpleModuleNs1, module1);
    framework.addModule(simpleModuleNs2, module1);
    framework.addModule(doubleModuleNs2, module2);
    // check that we can add instances of datasets in those modules
    framework.addInstance(SimpleKVTable.class.getName(), namespace1.dataset("kv1"), DatasetProperties.EMPTY);
    framework.addInstance(SimpleKVTable.class.getName(), namespace2.dataset("kv1"), DatasetProperties.EMPTY);
    // check that only namespace2 can add an instance of this type, since the module should only be in namespace2
    framework.addInstance(DoubleWrappedKVTable.class.getName(), namespace2.dataset("kv2"), DatasetProperties.EMPTY);
    try {
        framework.addInstance(DoubleWrappedKVTable.class.getName(), namespace2.dataset("kv2"), DatasetProperties.EMPTY);
        Assert.fail();
    } catch (Exception e) {
    // expected
    }
    // check that deleting all modules from namespace2 does not affect namespace1
    framework.deleteAllInstances(namespace2);
    framework.deleteAllModules(namespace2);
    // should still be able to add an instance in namespace1
    framework.addInstance(SimpleKVTable.class.getName(), namespace1.dataset("kv3"), DatasetProperties.EMPTY);
    // but not in namespace2
    try {
        framework.addInstance(SimpleKVTable.class.getName(), namespace2.dataset("kv3"), DatasetProperties.EMPTY);
        Assert.fail();
    } catch (Exception e) {
    // expected
    }
    // add back modules to namespace2
    framework.addModule(simpleModuleNs2, module1);
    framework.addModule(doubleModuleNs2, module2);
    // check that deleting a single module from namespace1 does not affect namespace2
    framework.deleteAllInstances(namespace1);
    framework.deleteModule(simpleModuleNs1);
    // should still be able to add an instance in namespace2
    framework.addInstance(DoubleWrappedKVTable.class.getName(), namespace2.dataset("kv1"), DatasetProperties.EMPTY);
    // but not in namespace1
    try {
        framework.addInstance(SimpleKVTable.class.getName(), namespace1.dataset("kv1"), DatasetProperties.EMPTY);
        Assert.fail();
    } catch (Exception e) {
    // expected
    }
}

Example 44 with DatasetModuleId

use of io.cdap.cdap.proto.id.DatasetModuleId in project cdap by cdapio.

the class ExploreDisabledTest method testDeployRecordScannable.

@Test
public void testDeployRecordScannable() throws Exception {
    // Try to deploy a dataset that is not record scannable, when explore is enabled.
    // This should be processed with no exception being thrown
    DatasetModuleId module1 = new DatasetModuleId(namespaceId.getNamespace(), "module1");
    DatasetId instance1 = namespaceId.dataset("table1");
    datasetFramework.addModule(module1, new KeyStructValueTableDefinition.KeyStructValueTableModule());
    // Performing admin operations to create dataset instance
    datasetFramework.addInstance("keyStructValueTable", instance1, DatasetProperties.EMPTY);
    Transaction tx1 = transactionManager.startShort(100);
    // Accessing dataset instance to perform data operations
    KeyStructValueTableDefinition.KeyStructValueTable table = datasetFramework.getDataset(instance1, DatasetDefinition.NO_ARGUMENTS, null);
    Assert.assertNotNull(table);
    table.startTx(tx1);
    KeyStructValueTableDefinition.KeyValue.Value value1 = new KeyStructValueTableDefinition.KeyValue.Value("first", Lists.newArrayList(1, 2, 3, 4, 5));
    KeyStructValueTableDefinition.KeyValue.Value value2 = new KeyStructValueTableDefinition.KeyValue.Value("two", Lists.newArrayList(10, 11, 12, 13, 14));
    table.put("1", value1);
    table.put("2", value2);
    Assert.assertEquals(value1, table.get("1"));
    Assert.assertTrue(table.commitTx());
    transactionManager.canCommit(tx1.getTransactionId(), table.getTxChanges());
    transactionManager.commit(tx1.getTransactionId(), tx1.getWritePointer());
    table.postTxCommit();
    Transaction tx2 = transactionManager.startShort(100);
    table.startTx(tx2);
    Assert.assertEquals(value1, table.get("1"));
    datasetFramework.deleteInstance(instance1);
    datasetFramework.deleteModule(module1);
}

Example 45 with DatasetModuleId

use of io.cdap.cdap.proto.id.DatasetModuleId in project cdap by cdapio.

the class ExploreDisabledTest method testDeployNotRecordScannable.

@Test
public void testDeployNotRecordScannable() throws Exception {
    // Try to deploy a dataset that is not record scannable, when explore is enabled.
    // This should be processed with no exceptionbeing thrown
    DatasetModuleId module2 = namespaceId.datasetModule("module2");
    DatasetId instance2 = namespaceId.dataset("table1");
    datasetFramework.addModule(module2, new NotRecordScannableTableDefinition.NotRecordScannableTableModule());
    // Performing admin operations to create dataset instance
    datasetFramework.addInstance("NotRecordScannableTableDef", instance2, DatasetProperties.EMPTY);
    Transaction tx1 = transactionManager.startShort(100);
    // Accessing dataset instance to perform data operations
    NotRecordScannableTableDefinition.KeyValueTable table = datasetFramework.getDataset(instance2, DatasetDefinition.NO_ARGUMENTS, null);
    Assert.assertNotNull(table);
    table.startTx(tx1);
    table.write("key1", "value1");
    table.write("key2", "value2");
    byte[] value = table.read("key1");
    Assert.assertEquals("value1", Bytes.toString(value));
    Assert.assertTrue(table.commitTx());
    transactionManager.canCommit(tx1.getTransactionId(), table.getTxChanges());
    transactionManager.commit(tx1.getTransactionId(), tx1.getWritePointer());
    table.postTxCommit();
    Transaction tx2 = transactionManager.startShort(100);
    table.startTx(tx2);
    value = table.read("key1");
    Assert.assertNotNull(value);
    Assert.assertEquals("value1", Bytes.toString(value));
    datasetFramework.deleteInstance(instance2);
    datasetFramework.deleteModule(module2);
}