Search in sources :

Example 1 with InMemoryTableModule

use of io.cdap.cdap.data2.dataset2.module.lib.inmemory.InMemoryTableModule in project cdap by caskdata.

the class SystemDatasetRuntimeModule method getInMemoryModules.

@Override
public Module getInMemoryModules() {
    return new AbstractModule() {

        @Override
        protected void configure() {
            MapBinder<String, DatasetModule> mapBinder = MapBinder.newMapBinder(binder(), String.class, DatasetModule.class, Constants.Dataset.Manager.DefaultDatasetModules.class);
            // NOTE: order is important due to dependencies between modules
            mapBinder.addBinding("orderedTable-memory").toInstance(new InMemoryTableModule());
            mapBinder.addBinding("metricsTable-memory").toInstance(new InMemoryMetricsTableModule());
            bindDefaultModules(mapBinder);
        }
    };
}
Also used : InMemoryTableModule(co.cask.cdap.data2.dataset2.module.lib.inmemory.InMemoryTableModule) InMemoryMetricsTableModule(co.cask.cdap.data2.dataset2.module.lib.inmemory.InMemoryMetricsTableModule) Constants(co.cask.cdap.common.conf.Constants) HBaseQueueDatasetModule(co.cask.cdap.data2.transaction.queue.hbase.HBaseQueueDatasetModule) LineageDatasetModule(co.cask.cdap.data2.metadata.lineage.LineageDatasetModule) ExternalDatasetModule(co.cask.cdap.data2.dataset2.lib.external.ExternalDatasetModule) UsageDatasetModule(co.cask.cdap.data2.registry.UsageDatasetModule) MetadataDatasetModule(co.cask.cdap.data2.metadata.dataset.MetadataDatasetModule) DatasetModule(co.cask.cdap.api.dataset.module.DatasetModule) AbstractModule(com.google.inject.AbstractModule)

Example 2 with InMemoryTableModule

use of io.cdap.cdap.data2.dataset2.module.lib.inmemory.InMemoryTableModule in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testDoubleCompositeDataset.

@Test
public void testDoubleCompositeDataset() throws Exception {
    // Configuring Dataset types
    DatasetFramework framework = getFramework();
    framework.addModule(IN_MEMORY, new InMemoryTableModule());
    framework.addModule(CORE, new CoreDatasetsModule());
    framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
    Assert.assertFalse(framework.hasSystemType(DoubleWrappedKVTable.class.getName()));
    Assert.assertFalse(framework.hasType(DOUBLE_KV_TYPE));
    framework.addModule(DOUBLE_KV, new SingleTypeModule(DoubleWrappedKVTable.class));
    Assert.assertTrue(framework.hasType(DOUBLE_KV_TYPE));
    Assert.assertFalse(framework.hasSystemType(DoubleWrappedKVTable.class.getName()));
    // Creating instance
    Assert.assertFalse(framework.hasInstance(MY_TABLE));
    framework.addInstance(DoubleWrappedKVTable.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    testCompositeDataset(framework);
    // cleanup
    framework.deleteInstance(MY_TABLE);
    framework.deleteModule(DOUBLE_KV);
    framework.deleteModule(KEY_VALUE);
    framework.deleteModule(CORE);
    framework.deleteModule(IN_MEMORY);
}
Also used : LineageWriterDatasetFramework(co.cask.cdap.data2.metadata.writer.LineageWriterDatasetFramework) InMemoryTableModule(co.cask.cdap.data2.dataset2.module.lib.inmemory.InMemoryTableModule) CoreDatasetsModule(co.cask.cdap.data2.dataset2.lib.table.CoreDatasetsModule) Test(org.junit.Test)

Example 3 with InMemoryTableModule

use of io.cdap.cdap.data2.dataset2.module.lib.inmemory.InMemoryTableModule in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testCompositeDataset.

@Test
public void testCompositeDataset() throws Exception {
    // Configuring Dataset types
    DatasetFramework framework = getFramework();
    framework.addModule(IN_MEMORY, new InMemoryTableModule());
    framework.addModule(CORE, new CoreDatasetsModule());
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    Assert.assertFalse(framework.hasType(SIMPLE_KV_TYPE));
    framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    // Creating instance
    Assert.assertFalse(framework.hasInstance(MY_TABLE));
    framework.addInstance(SimpleKVTable.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    testCompositeDataset(framework);
    // cleanup
    framework.deleteInstance(MY_TABLE);
    framework.deleteModule(KEY_VALUE);
    framework.deleteModule(CORE);
    framework.deleteModule(IN_MEMORY);
}
Also used : LineageWriterDatasetFramework(co.cask.cdap.data2.metadata.writer.LineageWriterDatasetFramework) InMemoryTableModule(co.cask.cdap.data2.dataset2.module.lib.inmemory.InMemoryTableModule) CoreDatasetsModule(co.cask.cdap.data2.dataset2.lib.table.CoreDatasetsModule) Test(org.junit.Test)

Example 4 with InMemoryTableModule

use of io.cdap.cdap.data2.dataset2.module.lib.inmemory.InMemoryTableModule in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testBasicManagement.

@Test
public void testBasicManagement() throws Exception {
    DatasetTypeId tableType = NAMESPACE_ID.datasetType(Table.class.getName());
    // Adding modules
    DatasetFramework framework = getFramework();
    framework.addModule(IN_MEMORY, new InMemoryTableModule());
    framework.addModule(CORE, new CoreDatasetsModule());
    framework.addModule(FILE, new FileSetModule());
    framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
    // keyvalue has been added in the system namespace
    Assert.assertTrue(framework.hasSystemType(Table.class.getName()));
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    Assert.assertTrue(framework.hasType(tableType));
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    // Creating instances
    framework.addInstance(Table.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    DatasetSpecification spec = framework.getDatasetSpec(MY_TABLE);
    Assert.assertNotNull(spec);
    Assert.assertEquals(MY_TABLE.getEntityName(), spec.getName());
    Assert.assertEquals(Table.class.getName(), spec.getType());
    framework.addInstance(Table.class.getName(), MY_TABLE2, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE2));
    // Update instances
    File baseDir = TMP_FOLDER.newFolder();
    framework.addInstance(FileSet.class.getName(), MY_DS, FileSetProperties.builder().setBasePath(baseDir.getPath()).setDataExternal(true).build());
    // this should fail because it would "internalize" external data
    try {
        framework.updateInstance(MY_DS, DatasetProperties.EMPTY);
        Assert.fail("update should have thrown instance conflict");
    } catch (InstanceConflictException e) {
    // expected
    }
    baseDir = TMP_FOLDER.newFolder();
    // this should succeed because it simply changes the external path
    framework.updateInstance(MY_DS, FileSetProperties.builder().setBasePath(baseDir.getPath()).setDataExternal(true).build());
    spec = framework.getDatasetSpec(MY_DS);
    Assert.assertNotNull(spec);
    Assert.assertEquals(baseDir.getPath(), FileSetProperties.getBasePath(spec.getProperties()));
    // cleanup
    try {
        framework.deleteAllModules(NAMESPACE_ID);
        Assert.fail("should not delete modules: there are datasets using their types");
    } catch (DatasetManagementException e) {
    // expected
    }
    // types are still there
    Assert.assertTrue(framework.hasType(tableType));
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    framework.deleteAllInstances(NAMESPACE_ID);
    Assert.assertEquals(0, framework.getInstances(NAMESPACE_ID).size());
    Assert.assertFalse(framework.hasInstance(MY_TABLE));
    Assert.assertNull(framework.getDatasetSpec(MY_TABLE));
    Assert.assertFalse(framework.hasInstance(MY_TABLE2));
    Assert.assertNull(framework.getDatasetSpec(MY_TABLE2));
    // now it should succeed
    framework.deleteAllModules(NAMESPACE_ID);
    Assert.assertTrue(framework.hasSystemType(Table.class.getName()));
    Assert.assertFalse(framework.hasType(tableType));
    Assert.assertFalse(framework.hasType(SIMPLE_KV_TYPE));
}
Also used : DatasetTypeId(co.cask.cdap.proto.id.DatasetTypeId) Table(co.cask.cdap.api.dataset.table.Table) FileSet(co.cask.cdap.api.dataset.lib.FileSet) DatasetSpecification(co.cask.cdap.api.dataset.DatasetSpecification) LineageWriterDatasetFramework(co.cask.cdap.data2.metadata.writer.LineageWriterDatasetFramework) InMemoryTableModule(co.cask.cdap.data2.dataset2.module.lib.inmemory.InMemoryTableModule) DatasetManagementException(co.cask.cdap.api.dataset.DatasetManagementException) InstanceConflictException(co.cask.cdap.api.dataset.InstanceConflictException) CoreDatasetsModule(co.cask.cdap.data2.dataset2.lib.table.CoreDatasetsModule) PartitionedFileSetModule(co.cask.cdap.data2.dataset2.lib.partitioned.PartitionedFileSetModule) FileSetModule(co.cask.cdap.data2.dataset2.lib.file.FileSetModule) File(java.io.File) Test(org.junit.Test)

Example 5 with InMemoryTableModule

use of io.cdap.cdap.data2.dataset2.module.lib.inmemory.InMemoryTableModule in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testAuditPublish.

@Test
public void testAuditPublish() throws Exception {
    // Clear all audit messages
    inMemoryAuditPublisher.popMessages();
    List<AuditMessage> expectedMessages = new ArrayList<>();
    // Adding modules
    DatasetFramework framework = getFramework();
    framework.addModule(IN_MEMORY, new InMemoryTableModule());
    // Creating instances
    framework.addInstance(Table.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    expectedMessages.add(new AuditMessage(0, MY_TABLE, "", AuditType.CREATE, AuditPayload.EMPTY_PAYLOAD));
    framework.addInstance(Table.class.getName(), MY_TABLE2, DatasetProperties.EMPTY);
    expectedMessages.add(new AuditMessage(0, MY_TABLE2, "", AuditType.CREATE, AuditPayload.EMPTY_PAYLOAD));
    // Update instance
    framework.updateInstance(MY_TABLE, DatasetProperties.EMPTY);
    expectedMessages.add(new AuditMessage(0, MY_TABLE, "", AuditType.UPDATE, AuditPayload.EMPTY_PAYLOAD));
    // Access instance
    ProgramRunId runId = new ProgramId("ns", "app", ProgramType.FLOW, "flow").run(RunIds.generate().getId());
    LineageWriterDatasetFramework lineageFramework = new LineageWriterDatasetFramework(framework, new NoOpLineageWriter(), new NoOpUsageRegistry(), new AuthenticationTestContext(), new NoOpAuthorizer());
    lineageFramework.setContext(new TestProgramContext(runId));
    lineageFramework.setAuditPublisher(inMemoryAuditPublisher);
    lineageFramework.getDataset(MY_TABLE, ImmutableMap.<String, String>of(), getClass().getClassLoader());
    expectedMessages.add(new AuditMessage(0, MY_TABLE, "", AuditType.ACCESS, new AccessPayload(AccessType.UNKNOWN, runId)));
    // Truncate instance
    framework.truncateInstance(MY_TABLE);
    expectedMessages.add(new AuditMessage(0, MY_TABLE, "", AuditType.TRUNCATE, AuditPayload.EMPTY_PAYLOAD));
    // Delete instance
    framework.deleteInstance(MY_TABLE);
    expectedMessages.add(new AuditMessage(0, MY_TABLE, "", AuditType.DELETE, AuditPayload.EMPTY_PAYLOAD));
    // Delete all instances in a namespace
    framework.deleteAllInstances(MY_TABLE2.getParent());
    expectedMessages.add(new AuditMessage(0, MY_TABLE2, "", AuditType.DELETE, AuditPayload.EMPTY_PAYLOAD));
    Assert.assertEquals(expectedMessages, inMemoryAuditPublisher.popMessages());
    // cleanup
    framework.deleteModule(IN_MEMORY);
}
Also used : AuditMessage(co.cask.cdap.proto.audit.AuditMessage) Table(co.cask.cdap.api.dataset.table.Table) ArrayList(java.util.ArrayList) AuthenticationTestContext(co.cask.cdap.security.auth.context.AuthenticationTestContext) NoOpAuthorizer(co.cask.cdap.security.spi.authorization.NoOpAuthorizer) NoOpUsageRegistry(co.cask.cdap.data2.registry.NoOpUsageRegistry) ProgramId(co.cask.cdap.proto.id.ProgramId) LineageWriterDatasetFramework(co.cask.cdap.data2.metadata.writer.LineageWriterDatasetFramework) LineageWriterDatasetFramework(co.cask.cdap.data2.metadata.writer.LineageWriterDatasetFramework) InMemoryTableModule(co.cask.cdap.data2.dataset2.module.lib.inmemory.InMemoryTableModule) AccessPayload(co.cask.cdap.proto.audit.payload.access.AccessPayload) NoOpLineageWriter(co.cask.cdap.data2.metadata.writer.NoOpLineageWriter) ProgramRunId(co.cask.cdap.proto.id.ProgramRunId) Test(org.junit.Test)

Aggregations

Test (org.junit.Test)10 InMemoryTableModule (co.cask.cdap.data2.dataset2.module.lib.inmemory.InMemoryTableModule)7 InMemoryTableModule (io.cdap.cdap.data2.dataset2.module.lib.inmemory.InMemoryTableModule)6 CoreDatasetsModule (co.cask.cdap.data2.dataset2.lib.table.CoreDatasetsModule)5 LineageWriterDatasetFramework (co.cask.cdap.data2.metadata.writer.LineageWriterDatasetFramework)5 LineageWriterDatasetFramework (io.cdap.cdap.data2.metadata.writer.LineageWriterDatasetFramework)5 CoreDatasetsModule (io.cdap.cdap.data2.dataset2.lib.table.CoreDatasetsModule)4 AbstractModule (com.google.inject.AbstractModule)3 DatasetModule (co.cask.cdap.api.dataset.module.DatasetModule)2 Table (co.cask.cdap.api.dataset.table.Table)2 FileSetModule (co.cask.cdap.data2.dataset2.lib.file.FileSetModule)2 PartitionedFileSetModule (co.cask.cdap.data2.dataset2.lib.partitioned.PartitionedFileSetModule)2 ArrayList (java.util.ArrayList)2 DatasetManagementException (co.cask.cdap.api.dataset.DatasetManagementException)1 DatasetSpecification (co.cask.cdap.api.dataset.DatasetSpecification)1 InstanceConflictException (co.cask.cdap.api.dataset.InstanceConflictException)1 FileSet (co.cask.cdap.api.dataset.lib.FileSet)1 DatasetDefinitionRegistry (co.cask.cdap.api.dataset.module.DatasetDefinitionRegistry)1 MetricsCollectionService (co.cask.cdap.api.metrics.MetricsCollectionService)1 Constants (co.cask.cdap.common.conf.Constants)1