Examples with KeyValue co.cask.cdap.api.dataset.lib.KeyValue used on opensource projects

Example 6 with KeyValue

use of co.cask.cdap.api.dataset.lib.KeyValue in project cdap by caskdata.

the class ObjectMappedTableDatasetTest method testScan.

@Test
public void testScan() throws Exception {
    dsFrameworkUtil.createInstance(ObjectMappedTable.class.getName(), RECORDS_ID, ObjectMappedTableProperties.builder().setType(Record.class).build());
    try {
        final ObjectMappedTableDataset<Record> records = dsFrameworkUtil.getInstance(RECORDS_ID);
        TransactionExecutor txnl = dsFrameworkUtil.newInMemoryTransactionExecutor((TransactionAware) records);
        Record record1 = new Record(Integer.MAX_VALUE, Long.MAX_VALUE, Float.MAX_VALUE, Double.MAX_VALUE, "foobar", Bytes.toBytes("foobar"), ByteBuffer.wrap(Bytes.toBytes("foobar")), UUID.randomUUID());
        Record record2 = new Record(Integer.MIN_VALUE, Long.MIN_VALUE, Float.MIN_VALUE, Double.MIN_VALUE, "baz", Bytes.toBytes("baz"), ByteBuffer.wrap(Bytes.toBytes("baz")), UUID.randomUUID());
        Record record3 = new Record(1, 0L, 3.14f, 3.14159265358979323846, "hello", Bytes.toBytes("world"), ByteBuffer.wrap(Bytes.toBytes("yo")), UUID.randomUUID());
        final List<KeyValue<byte[], Record>> recordList = Lists.newArrayList();
        recordList.add(new KeyValue<>(Bytes.toBytes("123"), record1));
        recordList.add(new KeyValue<>(Bytes.toBytes("456"), record2));
        recordList.add(new KeyValue<>(Bytes.toBytes("789"), record3));
        for (final KeyValue<byte[], Record> record : recordList) {
            txnl.execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    records.write(record.getKey(), record.getValue());
                }
            });
        }
        final List<KeyValue<byte[], Record>> actualList = Lists.newArrayList();
        txnl.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                CloseableIterator<KeyValue<byte[], Record>> results = records.scan((String) null, null);
                while (results.hasNext()) {
                    actualList.add(results.next());
                }
                results.close();
            }
        });
        Assert.assertEquals(recordList.size(), actualList.size());
        for (int i = 0; i < actualList.size(); i++) {
            KeyValue<byte[], Record> expected = recordList.get(i);
            KeyValue<byte[], Record> actual = actualList.get(i);
            Assert.assertArrayEquals(expected.getKey(), actual.getKey());
            Assert.assertEquals(expected.getValue(), actual.getValue());
        }
        txnl.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                CloseableIterator<KeyValue<byte[], Record>> results = records.scan("789", null);
                KeyValue<byte[], Record> actualRecord = results.next();
                Assert.assertFalse(results.hasNext());
                Assert.assertArrayEquals(actualRecord.getKey(), recordList.get(2).getKey());
                Assert.assertEquals(actualRecord.getValue(), recordList.get(2).getValue());
                results.close();
                results = records.scan(null, "124");
                actualRecord = results.next();
                Assert.assertFalse(results.hasNext());
                Assert.assertArrayEquals(actualRecord.getKey(), recordList.get(0).getKey());
                Assert.assertEquals(actualRecord.getValue(), recordList.get(0).getValue());
                results.close();
                results = records.scan(null, "123");
                Assert.assertFalse(results.hasNext());
                results.close();
            }
        });
        actualList.clear();
        txnl.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                Scan scan = new Scan(null, null);
                CloseableIterator<KeyValue<byte[], Record>> results = records.scan(scan);
                while (results.hasNext()) {
                    actualList.add(results.next());
                }
            }
        });
        Assert.assertEquals(recordList.size(), actualList.size());
    } finally {
        dsFrameworkUtil.deleteInstance(RECORDS_ID);
    }
}

Example 7 with KeyValue

use of co.cask.cdap.api.dataset.lib.KeyValue in project cdap by caskdata.

the class SparkTestRun method testTransaction.

@Test
public void testTransaction() throws Exception {
    ApplicationManager applicationManager = deploy(TestSparkApp.class);
    StreamManager streamManager = getStreamManager("SparkStream");
    // Write some sentences to the stream
    streamManager.send("red fox");
    streamManager.send("brown fox");
    streamManager.send("grey fox");
    streamManager.send("brown bear");
    streamManager.send("black bear");
    // Run the spark program
    SparkManager sparkManager = applicationManager.getSparkManager(TransactionSpark.class.getSimpleName());
    sparkManager.start(ImmutableMap.of("source.stream", "SparkStream", "keyvalue.table", "KeyValueTable", "result.all.dataset", "SparkResult", "result.threshold", "2", "result.threshold.dataset", "SparkThresholdResult"));
    // Verify result from dataset before the Spark program terminates
    final DataSetManager<KeyValueTable> resultManager = getDataset("SparkThresholdResult");
    final KeyValueTable resultTable = resultManager.get();
    // Expect the threshold result dataset, with threshold >=2, contains [brown, fox, bear]
    Tasks.waitFor(ImmutableSet.of("brown", "fox", "bear"), new Callable<Set<String>>() {

        @Override
        public Set<String> call() throws Exception {
            // This is to start a new TX
            resultManager.flush();
            LOG.info("Reading from threshold result");
            try (CloseableIterator<KeyValue<byte[], byte[]>> itor = resultTable.scan(null, null)) {
                return ImmutableSet.copyOf(Iterators.transform(itor, new Function<KeyValue<byte[], byte[]>, String>() {

                    @Override
                    public String apply(KeyValue<byte[], byte[]> input) {
                        String word = Bytes.toString(input.getKey());
                        LOG.info("{}, {}", word, Bytes.toInt(input.getValue()));
                        return word;
                    }
                }));
            }
        }
    }, 3, TimeUnit.MINUTES, 1, TimeUnit.SECONDS);
    sparkManager.stop();
    sparkManager.waitForRun(ProgramRunStatus.KILLED, 60, TimeUnit.SECONDS);
}

Example 8 with KeyValue

use of co.cask.cdap.api.dataset.lib.KeyValue in project cdap by caskdata.

the class SparkTestRun method testSparkWithLocalFiles.

private void testSparkWithLocalFiles(Class<? extends Application> appClass, String sparkProgram, String prefix) throws Exception {
    ApplicationManager applicationManager = deploy(appClass);
    URI localFile = createLocalPropertiesFile(prefix);
    SparkManager sparkManager = applicationManager.getSparkManager(sparkProgram).start(Collections.singletonMap(SparkAppUsingLocalFiles.LOCAL_FILE_RUNTIME_ARG, localFile.toString()));
    sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 2, TimeUnit.MINUTES);
    DataSetManager<KeyValueTable> kvTableManager = getDataset(SparkAppUsingLocalFiles.OUTPUT_DATASET_NAME);
    KeyValueTable kvTable = kvTableManager.get();
    Map<String, String> expected = ImmutableMap.of("a", "1", "b", "2", "c", "3");
    List<byte[]> deleteKeys = new ArrayList<>();
    try (CloseableIterator<KeyValue<byte[], byte[]>> scan = kvTable.scan(null, null)) {
        for (int i = 0; i < 3; i++) {
            KeyValue<byte[], byte[]> next = scan.next();
            Assert.assertEquals(expected.get(Bytes.toString(next.getKey())), Bytes.toString(next.getValue()));
            deleteKeys.add(next.getKey());
        }
        Assert.assertFalse(scan.hasNext());
    }
    // Cleanup after run
    kvTableManager.flush();
    for (byte[] key : deleteKeys) {
        kvTable.delete(key);
    }
    kvTableManager.flush();
}

Example 9 with KeyValue

use of co.cask.cdap.api.dataset.lib.KeyValue in project cdap by caskdata.

the class TestFrameworkTestRun method assertWorkerDatasetWrites.

private void assertWorkerDatasetWrites(byte[] startRow, byte[] endRow, int expectedCount, int expectedTotalCount) throws Exception {
    DataSetManager<KeyValueTable> datasetManager = getDataset(testSpace.dataset(AppUsingGetServiceURL.WORKER_INSTANCES_DATASET));
    KeyValueTable instancesTable = datasetManager.get();
    try (CloseableIterator<KeyValue<byte[], byte[]>> instancesIterator = instancesTable.scan(startRow, endRow)) {
        List<KeyValue<byte[], byte[]>> workerInstances = Lists.newArrayList(instancesIterator);
        // Assert that the worker starts with expectedCount instances
        Assert.assertEquals(expectedCount, workerInstances.size());
        // Assert that each instance of the worker knows the total number of instances
        for (KeyValue<byte[], byte[]> keyValue : workerInstances) {
            Assert.assertEquals(expectedTotalCount, Bytes.toInt(keyValue.getValue()));
        }
    }
}

Example 10 with KeyValue

use of co.cask.cdap.api.dataset.lib.KeyValue in project cdap by caskdata.

the class TestFrameworkTestRun method testDynamicBatchSize.

@Category(SlowTests.class)
@Test
public void testDynamicBatchSize() throws Exception {
    ApplicationManager applicationManager = deployApplication(testSpace, GenSinkApp2.class);
    DataSetManager<KeyValueTable> table = getDataset(testSpace.dataset("table"));
    // Start the flow with runtime argument. It should set the batch size to 1.
    FlowManager flowManager = applicationManager.getFlowManager("GenSinkFlow").start(Collections.singletonMap("flowlet.BatchSinkFlowlet.batch.size", "1"));
    RuntimeMetrics batchSinkMetrics = flowManager.getFlowletMetrics("BatchSinkFlowlet");
    // Batch sink only get the 99 batch events
    batchSinkMetrics.waitForProcessed(99, 5, TimeUnit.SECONDS);
    flowManager.stop();
    flowManager.waitForRun(ProgramRunStatus.KILLED, 10, TimeUnit.SECONDS);
    try (CloseableIterator<KeyValue<byte[], byte[]>> itor = table.get().scan(null, null)) {
        // Should only see batch size of 1.
        while (itor.hasNext()) {
            Assert.assertEquals(1, Bytes.toInt(itor.next().getKey()));
        }
    }
}