Examples with KeyValueTable io.cdap.cdap.api.dataset.lib.KeyValueTable used on opensource projects

Example 21 with KeyValueTable

use of io.cdap.cdap.api.dataset.lib.KeyValueTable in project cdap by caskdata.

the class AuthorizationTest method verifyDummyData.

private void verifyDummyData(NamespaceId namespaceId, String datasetName) throws Exception {
    DataSetManager<KeyValueTable> outTableManager = getDataset(namespaceId.dataset(datasetName));
    KeyValueTable outputTable = outTableManager.get();
    Assert.assertEquals("world", Bytes.toString(outputTable.read("hello")));
}

Example 22 with KeyValueTable

use of io.cdap.cdap.api.dataset.lib.KeyValueTable in project cdap by caskdata.

the class AuthorizationTest method testCrossNSDatasetAccessFromService.

private void testCrossNSDatasetAccessFromService(ServiceManager serviceManager) throws Exception {
    NamespaceMeta outputDatasetNS = new NamespaceMeta.Builder().setName("outputNS").build();
    NamespaceId outputDatasetNSId = outputDatasetNS.getNamespaceId();
    DatasetId datasetId = outputDatasetNSId.dataset("store");
    Map<EntityId, Set<? extends Permission>> neededPrivileges = ImmutableMap.<EntityId, Set<? extends Permission>>builder().put(outputDatasetNSId, EnumSet.of(StandardPermission.GET, StandardPermission.CREATE, StandardPermission.DELETE)).put(datasetId, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET, StandardPermission.DELETE)).put(outputDatasetNSId.datasetType("keyValueTable"), EnumSet.of(StandardPermission.UPDATE)).build();
    setUpPrivilegeAndRegisterForDeletion(ALICE, neededPrivileges);
    getNamespaceAdmin().create(outputDatasetNS);
    addDatasetInstance(datasetId, "keyValueTable");
    // switch to BOB
    SecurityRequestContext.setUserId(BOB.getName());
    Map<String, String> args = ImmutableMap.of(CrossNsDatasetAccessApp.OUTPUT_DATASET_NS, outputDatasetNS.getNamespaceId().getNamespace(), CrossNsDatasetAccessApp.OUTPUT_DATASET_NAME, "store");
    // Start the service as BOB
    serviceManager.start(args);
    // Call to the service would result in failure due to BOB doesn't have permission on the namespace as set in args
    URL url = new URL(serviceManager.getServiceURL(5, TimeUnit.SECONDS), "write/data");
    HttpResponse response = executeAuthenticated(HttpRequest.put(url));
    Assert.assertEquals(500, response.getResponseCode());
    // This is a hack that works around the fact that we cannot properly catch exceptions in the service handler.
    // TODO: Figure out a way to stop checking error messages.
    Assert.assertTrue("Wrong message " + response.getResponseBodyAsString(), response.getResponseBodyAsString().contains("'" + BOB + "' has insufficient privileges"));
    serviceManager.stop();
    serviceManager.waitForStopped(10, TimeUnit.SECONDS);
    SecurityRequestContext.setUserId(ALICE.getName());
    assertDatasetIsEmpty(outputDatasetNS.getNamespaceId(), "store");
    // Give BOB permission to write to the dataset in another namespace
    grantAndAssertSuccess(datasetId, BOB, EnumSet.of(StandardPermission.GET, StandardPermission.UPDATE));
    // switch back to BOB to run service again
    SecurityRequestContext.setUserId(BOB.getName());
    // Write data in another namespace should be successful now
    serviceManager.start(args);
    for (int i = 0; i < 10; i++) {
        url = new URL(serviceManager.getServiceURL(5, TimeUnit.SECONDS), "write/" + i);
        response = executeAuthenticated(HttpRequest.put(url));
        Assert.assertEquals(200, response.getResponseCode());
    }
    serviceManager.stop();
    serviceManager.waitForStopped(10, TimeUnit.SECONDS);
    // switch back to alice and verify the data its fine now to verify.
    SecurityRequestContext.setUserId(ALICE.getName());
    DataSetManager<KeyValueTable> dataSetManager = getDataset(outputDatasetNS.getNamespaceId().dataset("store"));
    KeyValueTable results = dataSetManager.get();
    for (int i = 0; i < 10; i++) {
        byte[] key = String.valueOf(i).getBytes(Charsets.UTF_8);
        Assert.assertArrayEquals(key, results.read(key));
    }
    getNamespaceAdmin().delete(outputDatasetNS.getNamespaceId());
}

Example 23 with KeyValueTable

use of io.cdap.cdap.api.dataset.lib.KeyValueTable in project cdap by caskdata.

the class AuthorizationTest method addDummyData.

private void addDummyData(NamespaceId namespaceId, String datasetName) throws Exception {
    DataSetManager<KeyValueTable> tableManager = getDataset(namespaceId.dataset(datasetName));
    KeyValueTable inputTable = tableManager.get();
    inputTable.write("hello", "world");
    tableManager.flush();
}

Example 24 with KeyValueTable

use of io.cdap.cdap.api.dataset.lib.KeyValueTable in project cdap by caskdata.

the class SparkTest method testClassicSpark.

@Test
public void testClassicSpark() throws Exception {
    ApplicationManager appManager = deploy(TestSparkApp.class);
    for (Class<?> sparkClass : Arrays.asList(TestSparkApp.ClassicSpark.class, TestSparkApp.ScalaClassicSpark.class)) {
        final SparkManager sparkManager = appManager.getSparkManager(sparkClass.getSimpleName());
        sparkManager.startAndWaitForGoodRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES);
    }
    KeyValueTable resultTable = this.<KeyValueTable>getDataset("ResultTable").get();
    Assert.assertEquals(1L, Bytes.toLong(resultTable.read(ClassicSparkProgram.class.getName())));
    Assert.assertEquals(1L, Bytes.toLong(resultTable.read(ScalaClassicSparkProgram.class.getName())));
}

Example 25 with KeyValueTable

use of io.cdap.cdap.api.dataset.lib.KeyValueTable in project cdap by caskdata.

the class SparkTest method testTransaction.

@Test
public void testTransaction() throws Exception {
    ApplicationManager applicationManager = deploy(TestSparkApp.class);
    // Write some data to a local file
    File inputFile = TEMP_FOLDER.newFile();
    try (PrintWriter writer = new PrintWriter(Files.newBufferedWriter(inputFile.toPath(), StandardCharsets.UTF_8))) {
        writer.println("red fox");
        writer.println("brown fox");
        writer.println("grey fox");
        writer.println("brown bear");
        writer.println("black bear");
    }
    // Run the spark program
    SparkManager sparkManager = applicationManager.getSparkManager(TransactionSpark.class.getSimpleName());
    sparkManager.start(ImmutableMap.of("input.file", inputFile.getAbsolutePath(), "keyvalue.table", "KeyValueTable", "result.all.dataset", "SparkResult", "result.threshold", "2", "result.threshold.dataset", "SparkThresholdResult"));
    // Verify result from dataset before the Spark program terminates
    final DataSetManager<KeyValueTable> resultManager = getDataset("SparkThresholdResult");
    final KeyValueTable resultTable = resultManager.get();
    // Expect the threshold result dataset, with threshold >=2, contains [brown, fox, bear]
    Tasks.waitFor(ImmutableSet.of("brown", "fox", "bear"), () -> {
        // This is to start a new TX
        resultManager.flush();
        LOG.info("Reading from threshold result");
        try (CloseableIterator<KeyValue<byte[], byte[]>> itor = resultTable.scan(null, null)) {
            return ImmutableSet.copyOf(Iterators.transform(itor, input -> {
                String word = Bytes.toString(input.getKey());
                LOG.info("{}, {}", word, Bytes.toInt(input.getValue()));
                return word;
            }));
        }
    }, 3, TimeUnit.MINUTES, 1, TimeUnit.SECONDS);
    sparkManager.stop();
    sparkManager.waitForRun(ProgramRunStatus.KILLED, 60, TimeUnit.SECONDS);
}