Examples with KeyValueTable co.cask.cdap.api.dataset.lib.KeyValueTable used on opensource projects

Example 61 with KeyValueTable

use of co.cask.cdap.api.dataset.lib.KeyValueTable in project cdap by caskdata.

the class TestFrameworkTestRun method testDynamicBatchSize.

@Category(SlowTests.class)
@Test
public void testDynamicBatchSize() throws Exception {
    ApplicationManager applicationManager = deployApplication(testSpace, GenSinkApp2.class);
    DataSetManager<KeyValueTable> table = getDataset(testSpace.dataset("table"));
    // Start the flow with runtime argument. It should set the batch size to 1.
    FlowManager flowManager = applicationManager.getFlowManager("GenSinkFlow").start(Collections.singletonMap("flowlet.BatchSinkFlowlet.batch.size", "1"));
    RuntimeMetrics batchSinkMetrics = flowManager.getFlowletMetrics("BatchSinkFlowlet");
    // Batch sink only get the 99 batch events
    batchSinkMetrics.waitForProcessed(99, 5, TimeUnit.SECONDS);
    flowManager.stop();
    flowManager.waitForRun(ProgramRunStatus.KILLED, 10, TimeUnit.SECONDS);
    try (CloseableIterator<KeyValue<byte[], byte[]>> itor = table.get().scan(null, null)) {
        // Should only see batch size of 1.
        while (itor.hasNext()) {
            Assert.assertEquals(1, Bytes.toInt(itor.next().getKey()));
        }
    }
}

Example 62 with KeyValueTable

use of co.cask.cdap.api.dataset.lib.KeyValueTable in project cdap by caskdata.

the class NotificationTest method useTransactionTest.

@Test
public void useTransactionTest() throws Exception {
    // Performing admin operations to create dataset instance
    // keyValueTable is a system dataset module
    namespaceAdmin.create(new NamespaceMeta.Builder().setName(namespace).build());
    DatasetId myTableInstance = namespace.dataset("myTable");
    dsFramework.addInstance("keyValueTable", myTableInstance, DatasetProperties.EMPTY);
    final CountDownLatch receivedLatch = new CountDownLatch(1);
    Assert.assertTrue(feedManager.createFeed(FEED1_INFO));
    try {
        Cancellable cancellable = notificationService.subscribe(FEED1, new NotificationHandler<String>() {

            private int received = 0;

            @Override
            public Type getNotificationType() {
                return String.class;
            }

            @Override
            public void received(final String notification, NotificationContext notificationContext) {
                notificationContext.execute(new TxRunnable() {

                    @Override
                    public void run(DatasetContext context) throws Exception {
                        KeyValueTable table = context.getDataset("myTable");
                        table.write("foo", String.format("%s-%d", notification, received++));
                        receivedLatch.countDown();
                    }
                }, TxRetryPolicy.maxRetries(5));
            }
        });
        // Short delay for the subscriber to setup the subscription.
        TimeUnit.MILLISECONDS.sleep(500);
        try {
            notificationService.publish(FEED1, "foobar");
            // Waiting for the subscriber to receive that notification
            Assert.assertTrue(receivedLatch.await(5, TimeUnit.SECONDS));
            // Read the KeyValueTable for the value updated from the subscriber.
            // Need to poll it couple times since after the received method returned,
            // the tx may not yet committed when we try to read it here.
            final KeyValueTable table = dsFramework.getDataset(myTableInstance, DatasetDefinition.NO_ARGUMENTS, null);
            Assert.assertNotNull(table);
            final TransactionContext txContext = new TransactionContext(txClient, table);
            Tasks.waitFor(true, new Callable<Boolean>() {

                @Override
                public Boolean call() throws Exception {
                    txContext.start();
                    try {
                        return "foobar-0".equals(Bytes.toString(table.read("foo")));
                    } finally {
                        txContext.finish();
                    }
                }
            }, 5, TimeUnit.SECONDS);
        } finally {
            cancellable.cancel();
        }
    } finally {
        dsFramework.deleteInstance(myTableInstance);
        feedManager.deleteFeed(FEED1);
        namespaceAdmin.delete(namespace);
    }
}

Example 63 with KeyValueTable

use of co.cask.cdap.api.dataset.lib.KeyValueTable in project cdap by caskdata.

the class AuthorizationTest method assertDatasetIsEmpty.

private void assertDatasetIsEmpty(NamespaceId namespaceId, String datasetName) throws Exception {
    DataSetManager<KeyValueTable> outTableManager = getDataset(namespaceId.dataset(datasetName));
    KeyValueTable outputTable = outTableManager.get();
    try (CloseableIterator<KeyValue<byte[], byte[]>> scanner = outputTable.scan(null, null)) {
        Assert.assertFalse(scanner.hasNext());
    }
}

Example 64 with KeyValueTable

use of co.cask.cdap.api.dataset.lib.KeyValueTable in project cdap by caskdata.

the class AuthorizationTest method testMRStreamAuth.

@Test
@Category(SlowTests.class)
public void testMRStreamAuth() throws Exception {
    createAuthNamespace();
    Authorizer authorizer = getAuthorizer();
    ApplicationManager appManager = deployApplication(AUTH_NAMESPACE, StreamAuthApp.class);
    // After deploy, change Alice from ALL to ADMIN on the namespace
    authorizer.revoke(AUTH_NAMESPACE, ALICE, EnumSet.allOf(Action.class));
    authorizer.grant(AUTH_NAMESPACE, ALICE, EnumSet.of(Action.ADMIN));
    StreamManager streamManager = getStreamManager(AUTH_NAMESPACE.stream(StreamAuthApp.STREAM));
    streamManager.send("Hello");
    final MapReduceManager mrManager = appManager.getMapReduceManager(StreamAuthApp.MAPREDUCE);
    mrManager.start();
    // Since Alice had full permissions, she should be able to execute the MR job successfully
    mrManager.waitForRun(ProgramRunStatus.COMPLETED, 1, TimeUnit.MINUTES);
    DataSetManager<KeyValueTable> kvManager = getDataset(AUTH_NAMESPACE.dataset(StreamAuthApp.KVTABLE));
    try (KeyValueTable kvTable = kvManager.get()) {
        byte[] value = kvTable.read("Hello");
        Assert.assertArrayEquals(Bytes.toBytes("Hello"), value);
    }
    ProgramId mrId = AUTH_NAMESPACE.app(StreamAuthApp.APP).mr(StreamAuthApp.MAPREDUCE);
    authorizer.grant(mrId.getNamespaceId(), BOB, ImmutableSet.of(Action.ADMIN));
    ArtifactSummary artifactSummary = appManager.getInfo().getArtifact();
    ArtifactId artifactId = AUTH_NAMESPACE.artifact(artifactSummary.getName(), artifactSummary.getVersion());
    authorizer.grant(artifactId, BOB, EnumSet.allOf(Action.class));
    authorizer.grant(mrId.getParent(), BOB, EnumSet.allOf(Action.class));
    authorizer.grant(mrId, BOB, EnumSet.allOf(Action.class));
    authorizer.grant(AUTH_NAMESPACE.stream(StreamAuthApp.STREAM), BOB, EnumSet.of(Action.ADMIN));
    authorizer.grant(AUTH_NAMESPACE.dataset(StreamAuthApp.KVTABLE), BOB, EnumSet.allOf(Action.class));
    streamManager.send("World");
    // Switch user to Bob. Note that he doesn't have READ access on the stream.
    SecurityRequestContext.setUserId(BOB.getName());
    mrManager.start();
    mrManager.waitForRun(ProgramRunStatus.FAILED, 1, TimeUnit.MINUTES);
    kvManager = getDataset(AUTH_NAMESPACE.dataset(StreamAuthApp.KVTABLE));
    try (KeyValueTable kvTable = kvManager.get()) {
        byte[] value = kvTable.read("World");
        Assert.assertNull(value);
    }
    // Now grant Bob, READ access on the stream. MR job should execute successfully now.
    authorizer.grant(AUTH_NAMESPACE.stream(StreamAuthApp.STREAM), BOB, ImmutableSet.of(Action.READ));
    mrManager.start();
    mrManager.waitForRuns(ProgramRunStatus.COMPLETED, 2, 1, TimeUnit.MINUTES);
    kvManager = getDataset(AUTH_NAMESPACE.dataset(StreamAuthApp.KVTABLE));
    try (KeyValueTable kvTable = kvManager.get()) {
        byte[] value = kvTable.read("World");
        Assert.assertEquals("World", Bytes.toString(value));
    }
    SecurityRequestContext.setUserId(ALICE.getName());
    appManager.delete();
    assertNoAccess(AUTH_NAMESPACE.app(StreamAuthApp.APP));
}

Example 65 with KeyValueTable

use of co.cask.cdap.api.dataset.lib.KeyValueTable in project cdap by caskdata.

the class AuthorizationTest method addDummyData.

private void addDummyData(NamespaceId namespaceId, String datasetName) throws Exception {
    DataSetManager<KeyValueTable> tableManager = getDataset(namespaceId.dataset(datasetName));
    KeyValueTable inputTable = tableManager.get();
    inputTable.write("hello", "world");
    tableManager.flush();
}