Examples with KeyValueTable co.cask.cdap.api.dataset.lib.KeyValueTable used on opensource projects

Example 16 with KeyValueTable

use of co.cask.cdap.api.dataset.lib.KeyValueTable in project cdap by caskdata.

the class BatchStreamIntegrationTestRun method verifyStreamBatchJob.

private void verifyStreamBatchJob(StreamManager streamManager, ApplicationManager applicationManager, String mapReduceName, int timeout) throws Exception {
    for (int i = 0; i < 50; i++) {
        streamManager.send(String.valueOf(i));
    }
    MapReduceManager mapReduceManager = applicationManager.getMapReduceManager(mapReduceName).start();
    mapReduceManager.waitForRun(ProgramRunStatus.COMPLETED, timeout, TimeUnit.SECONDS);
    // The MR job simply turns every stream event body into key/value pairs, with key==value.
    DataSetManager<KeyValueTable> datasetManager = getDataset("results");
    KeyValueTable results = datasetManager.get();
    for (int i = 0; i < 50; i++) {
        byte[] key = String.valueOf(i).getBytes(Charsets.UTF_8);
        Assert.assertArrayEquals(key, results.read(key));
    }
}

Example 17 with KeyValueTable

use of co.cask.cdap.api.dataset.lib.KeyValueTable in project cdap by caskdata.

the class SparkStreamIntegrationTestRun method testSparkWithStream.

@Test
public void testSparkWithStream() throws Exception {
    ApplicationManager applicationManager = deployApplication(TestSparkStreamIntegrationApp.class);
    StreamManager streamManager = getStreamManager("testStream");
    for (int i = 0; i < 50; i++) {
        streamManager.send(String.valueOf(i));
    }
    SparkManager sparkManager = applicationManager.getSparkManager("SparkStreamProgram").start();
    sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 120, TimeUnit.SECONDS);
    // The Spark job simply turns every stream event body into key/value pairs, with key==value.
    DataSetManager<KeyValueTable> datasetManager = getDataset("result");
    verifyDatasetResult(datasetManager);
}

Example 18 with KeyValueTable

use of co.cask.cdap.api.dataset.lib.KeyValueTable in project cdap by caskdata.

the class SparkStreamIntegrationTestRun method verifyDatasetResult.

private void verifyDatasetResult(DataSetManager<KeyValueTable> datasetManager) {
    KeyValueTable results = datasetManager.get();
    for (int i = 0; i < 50; i++) {
        byte[] key = String.valueOf(i).getBytes(Charsets.UTF_8);
        Assert.assertArrayEquals(key, results.read(key));
    }
}

Example 19 with KeyValueTable

use of co.cask.cdap.api.dataset.lib.KeyValueTable in project cdap by caskdata.

the class Spark2Test method testScalaSparkWithObjectStore.

@Test
public void testScalaSparkWithObjectStore() throws Exception {
    ApplicationManager applicationManager = deploy(NamespaceId.DEFAULT, SparkAppUsingObjectStore.class);
    DataSetManager<ObjectStore<String>> keysManager = getDataset("keys");
    prepareInputData(keysManager);
    SparkManager sparkManager = applicationManager.getSparkManager(ScalaCharCountProgram.class.getSimpleName()).start();
    sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 1, TimeUnit.MINUTES);
    DataSetManager<KeyValueTable> countManager = getDataset("count");
    checkOutputData(countManager);
}

Example 20 with KeyValueTable

use of co.cask.cdap.api.dataset.lib.KeyValueTable in project cdap by caskdata.

the class CoreDatasetsModule method register.

@Override
public void register(DatasetDefinitionRegistry registry) {
    DatasetDefinition<Table, DatasetAdmin> tableDef = registry.get("table");
    DatasetDefinition<KeyValueTable, DatasetAdmin> kvTableDef = new KeyValueTableDefinition("keyValueTable", tableDef);
    registry.add(kvTableDef);
    registry.add(new KeyValueTableDefinition(KeyValueTable.class.getName(), tableDef));
    DatasetDefinition<ObjectStore, DatasetAdmin> objectStoreDef = new ObjectStoreDefinition("objectStore", kvTableDef);
    registry.add(new ObjectStoreDefinition("objectStore", kvTableDef));
    registry.add(new ObjectStoreDefinition(ObjectStore.class.getName(), kvTableDef));
    registry.add(new IndexedObjectStoreDefinition("indexedObjectStore", tableDef, objectStoreDef));
    registry.add(new IndexedObjectStoreDefinition(IndexedObjectStore.class.getName(), tableDef, objectStoreDef));
    registry.add(new IndexedTableDefinition("indexedTable", tableDef));
    registry.add(new IndexedTableDefinition(IndexedTable.class.getName(), tableDef));
    registry.add(new TimeseriesTableDefinition("timeseriesTable", tableDef));
    registry.add(new TimeseriesTableDefinition(TimeseriesTable.class.getName(), tableDef));
    registry.add(new CounterTimeseriesTableDefinition("counterTimeseriesTable", tableDef));
    registry.add(new CounterTimeseriesTableDefinition(CounterTimeseriesTable.class.getName(), tableDef));
    // in-memory table
    registry.add(new InMemoryTableDefinition("memoryTable"));
}