Examples with Row io.cdap.cdap.api.dataset.table.Row used on opensource projects

Example 51 with Row

use of io.cdap.cdap.api.dataset.table.Row in project cdap by cdapio.

the class IndexedTableTest method assertEmpty.

/**
 * Asserts that the given scanner contains no more rows.
 */
private void assertEmpty(Scanner scanner) {
    Row row = scanner.next();
    Assert.assertNull(row);
}

Example 52 with Row

use of io.cdap.cdap.api.dataset.table.Row in project cdap by cdapio.

the class IndexedTableTest method testIndexKeyDelimiterAmbiguity.

@Test
public void testIndexKeyDelimiterAmbiguity() throws Exception {
    final byte[] a = { 'a' };
    final byte[] ab = { 'a', 0, 'b' };
    final byte[] abc = { 'a', 0, 'b', 0, 'c' };
    final byte[] bc = { 'b', 0, 'c' };
    final byte[] bcd = { 'b', 0, 'c', 'd' };
    final byte[] c = { 'c' };
    final byte[] d = { 'd' };
    final byte[] w = { 'w' };
    final byte[] x = { 'x' };
    final byte[] y = { 'y' };
    final byte[] z = { 'z' };
    DatasetId delimTabInstance = DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("delimtab");
    dsFrameworkUtil.createInstance("indexedTable", delimTabInstance, DatasetProperties.builder().add(IndexedTable.INDEX_COLUMNS_CONF_KEY, Bytes.toString(a) + "," + Bytes.toString(ab)).build());
    final IndexedTable iTable = dsFrameworkUtil.getInstance(delimTabInstance);
    try {
        TransactionExecutor tx = dsFrameworkUtil.newTransactionExecutor(iTable);
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                iTable.put(x, a, bc);
                iTable.put(y, ab, c);
                iTable.put(w, a, bcd);
                iTable.put(z, abc, d);
            }
        });
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // ensure that readByIndex filters teh false positive rows in index
                Scanner scanner = iTable.readByIndex(a, bc);
                try {
                    Row row = scanner.next();
                    Assert.assertNotNull(row);
                    Assert.assertArrayEquals(x, row.getRow());
                    Assert.assertArrayEquals(bc, row.get(a));
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
                scanner = iTable.readByIndex(ab, c);
                try {
                    Row row = scanner.next();
                    Assert.assertNotNull(row);
                    Assert.assertArrayEquals(y, row.getRow());
                    Assert.assertArrayEquals(c, row.get(ab));
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
                // ensure that scanByIndex filters the false positive rows in index
                scanner = iTable.scanByIndex(a, bcd, null);
                try {
                    Row row = scanner.next();
                    Assert.assertNotNull(row);
                    Assert.assertArrayEquals(w, row.getRow());
                    Assert.assertArrayEquals(bcd, row.get(a));
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
                scanner = iTable.scanByIndex(a, null, bcd);
                try {
                    Row row = scanner.next();
                    Assert.assertNotNull(row);
                    Assert.assertArrayEquals(x, row.getRow());
                    Assert.assertArrayEquals(bc, row.get(a));
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
            }
        });
    } finally {
        dsFrameworkUtil.deleteInstance(delimTabInstance);
    }
}

Example 53 with Row

use of io.cdap.cdap.api.dataset.table.Row in project cdap by cdapio.

the class IndexedTableTest method testIndexedOperations.

@Test
public void testIndexedOperations() throws Exception {
    TransactionExecutor txnl = dsFrameworkUtil.newTransactionExecutor(table);
    // start a new transaction
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // add a value c with idx = 1, and b with idx = 2
            table.put(new Put(keyC).add(idxCol, idx1).add(valCol, valC));
            table.put(new Put(keyB).add(idxCol, idx2).add(valCol, valB));
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // read by key c
            Row row = table.get(new Get(keyC, colIdxVal));
            TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx1, valC });
            // read by key b
            row = table.get(new Get(keyB, colIdxVal));
            TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx2, valB });
            // read by idx 1 -> c
            row = readFirst(table.readByIndex(idxCol, idx1));
            TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx1, valC });
            // read by idx 2 -> b
            row = readFirst(table.readByIndex(idxCol, idx2));
            TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx2, valB });
            // test read over empty index (idx 3)
            assertEmpty(table.readByIndex(idxCol, idx3));
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // add a value a with idx = 1
            table.put(new Put(keyA).add(idxCol, idx1).add(valCol, valA));
        }
    });
    // read by idx 1 -> a
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Row row = readFirst(table.readByIndex(idxCol, idx1));
            TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx1, valA });
        }
    });
    // start a new transaction
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // delete value a
            table.delete(new Delete(keyA, colIdxVal));
        }
    });
    // read by idx 1 -> c
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Row row = readFirst(table.readByIndex(idxCol, idx1));
            TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx1, valC });
        }
    });
    // start a new transaction
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // add a value aa with idx 2
            table.put(new Put(keyAA).add(idxCol, idx2).add(valCol, valAA));
        }
    });
    // read by idx 2 -> aa
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Row row = readFirst(table.readByIndex(idxCol, idx2));
            TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx2, valAA });
        }
    });
    // start a new transaction
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // swap value for aa to ab
            Assert.assertTrue(table.compareAndSwap(keyAA, valCol, valAA, valAB));
        }
    });
    // read by idx 2 -> ab
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Row row = readFirst(table.readByIndex(idxCol, idx2));
            TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx2, valAB });
        }
    });
    // start a new transaction
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // swap value for aa to bb
            Assert.assertTrue(table.compareAndSwap(keyAA, valCol, valAB, valBB));
        }
    });
    // read by idx 2 -> bb (value of key aa)
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Row row = readFirst(table.readByIndex(idxCol, idx2));
            TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx2, valBB });
        }
    });
    // start a new transaction
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // swap value for aa to null
            Assert.assertTrue(table.compareAndSwap(keyAA, valCol, valBB, null));
        }
    });
    // read by idx 2 -> null (value of b)
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Row row = readFirst(table.readByIndex(idxCol, idx2));
            TableAssert.assertColumn(row, idxCol, idx2);
        }
    });
    // start a new transaction
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // swap idx for c to 3
            Assert.assertTrue(table.compareAndSwap(keyC, idxCol, idx1, idx3));
        }
    });
    // read by idx 1 -> null (no row has that any more)
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            assertEmpty(table.readByIndex(idxCol, idx1));
            // read by idx 3 > c
            Row row = readFirst(table.readByIndex(idxCol, idx3));
            TableAssert.assertColumns(row, new byte[][] { idxCol, valCol }, new byte[][] { idx3, valC });
        }
    });
}

Example 54 with Row

use of io.cdap.cdap.api.dataset.table.Row in project cdap by cdapio.

the class ReflectionTableTest method testStructuredRecordRepresentation.

@Test
public void testStructuredRecordRepresentation() throws Exception {
    dsFrameworkUtil.createInstance("table", users, DatasetProperties.builder().build());
    try {
        final Table usersTable = dsFrameworkUtil.getInstance(users);
        final byte[] rowKey = Bytes.toBytes(123);
        final Schema schema = new ReflectionSchemaGenerator().generate(User.class);
        // TableDataset is not accessible here, but we know that's the underlying implementation...
        TransactionExecutor tx = dsFrameworkUtil.newTransactionExecutor((TransactionAware) usersTable);
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                Put put = new Put(rowKey);
                ReflectionPutWriter<User> putWriter = new ReflectionPutWriter<>(schema);
                putWriter.write(SAMUEL, put);
                usersTable.put(put);
                Row row = usersTable.get(rowKey);
                ReflectionRowRecordReader rowReader = new ReflectionRowRecordReader(schema, null);
                StructuredRecord actual = rowReader.read(row, schema);
                assertRecordEqualsUser(SAMUEL, actual);
            }
        });
    } finally {
        dsFrameworkUtil.deleteInstance(users);
    }
}

Example 55 with Row

use of io.cdap.cdap.api.dataset.table.Row in project cdap by cdapio.

the class TestFrameworkTestRun method testAppWithPlugin.

@Test
public void testAppWithPlugin() throws Exception {
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("app-with-plugin", "1.0.0-SNAPSHOT");
    addAppArtifact(artifactId, AppWithPlugin.class);
    ArtifactId pluginArtifactId = NamespaceId.DEFAULT.artifact("test-plugin", "1.0.0-SNAPSHOT");
    addPluginArtifact(pluginArtifactId, artifactId, ToStringPlugin.class);
    ApplicationId appId = NamespaceId.DEFAULT.app("AppWithPlugin");
    AppRequest createRequest = new AppRequest(new ArtifactSummary(artifactId.getArtifact(), artifactId.getVersion()));
    ApplicationManager appManager = deployApplication(appId, createRequest);
    final WorkerManager workerManager = appManager.getWorkerManager(AppWithPlugin.WORKER);
    workerManager.start();
    workerManager.waitForRun(ProgramRunStatus.COMPLETED, 10, TimeUnit.SECONDS);
    final ServiceManager serviceManager = appManager.getServiceManager(AppWithPlugin.SERVICE);
    serviceManager.start();
    serviceManager.waitForRun(ProgramRunStatus.RUNNING, 10, TimeUnit.SECONDS);
    URL serviceURL = serviceManager.getServiceURL(5, TimeUnit.SECONDS);
    callServiceGet(serviceURL, "dummy");
    serviceManager.stop();
    serviceManager.waitForStopped(10, TimeUnit.SECONDS);
    WorkflowManager workflowManager = appManager.getWorkflowManager(AppWithPlugin.WORKFLOW);
    workflowManager.start();
    workflowManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES);
    List<RunRecord> runRecords = workflowManager.getHistory();
    Assert.assertNotEquals(ProgramRunStatus.FAILED, runRecords.get(0).getStatus());
    DataSetManager<KeyValueTable> workflowTableManager = getDataset(AppWithPlugin.WORKFLOW_TABLE);
    String value = Bytes.toString(workflowTableManager.get().read("val"));
    Assert.assertEquals(AppWithPlugin.TEST, value);
    Map<String, String> workflowTags = ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, NamespaceId.DEFAULT.getNamespace(), Constants.Metrics.Tag.APP, "AppWithPlugin", Constants.Metrics.Tag.WORKFLOW, AppWithPlugin.WORKFLOW, Constants.Metrics.Tag.RUN_ID, runRecords.get(0).getPid());
    getMetricsManager().waitForTotalMetricCount(workflowTags, String.format("user.destroy.%s", AppWithPlugin.WORKFLOW), 1, 60, TimeUnit.SECONDS);
    // Testing Spark Plugins. First send some data to fileset for the Spark program to process
    DataSetManager<FileSet> fileSetManager = getDataset(AppWithPlugin.SPARK_INPUT);
    FileSet fileSet = fileSetManager.get();
    try (PrintStream out = new PrintStream(fileSet.getLocation("input").append("file.txt").getOutputStream(), true, "UTF-8")) {
        for (int i = 0; i < 5; i++) {
            out.println("Message " + i);
        }
    }
    Map<String, String> sparkArgs = new HashMap<>();
    FileSetArguments.setInputPath(sparkArgs, "input");
    SparkManager sparkManager = appManager.getSparkManager(AppWithPlugin.SPARK).start(sparkArgs);
    sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 2, TimeUnit.MINUTES);
    // Verify the Spark result.
    DataSetManager<Table> dataSetManager = getDataset(AppWithPlugin.SPARK_TABLE);
    Table table = dataSetManager.get();
    try (Scanner scanner = table.scan(null, null)) {
        for (int i = 0; i < 5; i++) {
            Row row = scanner.next();
            Assert.assertNotNull(row);
            String expected = "Message " + i + " " + AppWithPlugin.TEST;
            Assert.assertEquals(expected, Bytes.toString(row.getRow()));
            Assert.assertEquals(expected, Bytes.toString(row.get(expected)));
        }
        // There shouldn't be any more rows in the table.
        Assert.assertNull(scanner.next());
    }
}