Examples with Row io.cdap.cdap.api.dataset.table.Row used on opensource projects

Example 36 with Row

use of io.cdap.cdap.api.dataset.table.Row in project cdap by caskdata.

the class MockRuntimeDatasetSink method readOutput.

/**
 * Used to read the records written by this sink.
 *
 * @param tableManager dataset manager used to get the sink dataset to read from
 */
public static List<StructuredRecord> readOutput(DataSetManager<Table> tableManager) throws Exception {
    Table table = tableManager.get();
    try (Scanner scanner = table.scan(null, null)) {
        List<StructuredRecord> records = new ArrayList<>();
        Row row;
        while ((row = scanner.next()) != null) {
            Schema schema = Schema.parseJson(row.getString(SCHEMA_COL));
            String recordStr = row.getString(RECORD_COL);
            records.add(StructuredRecordStringConverter.fromJsonString(recordStr, schema));
        }
        return records;
    }
}

Example 37 with Row

use of io.cdap.cdap.api.dataset.table.Row in project cdap by caskdata.

the class AbstractMockSink method readOutput.

/**
 * Used to read the records written by this sink.
 *
 * @param tableManager dataset manager used to get the sink dataset to read from
 */
public static List<StructuredRecord> readOutput(DataSetManager<Table> tableManager) throws Exception {
    tableManager.flush();
    Table table = tableManager.get();
    try (Scanner scanner = table.scan(null, null)) {
        List<StructuredRecord> records = new ArrayList<>();
        Row row;
        while ((row = scanner.next()) != null) {
            Schema schema = Schema.parseJson(row.getString(SCHEMA_COL));
            String recordStr = row.getString(RECORD_COL);
            records.add(StructuredRecordStringConverter.fromJsonString(recordStr, schema));
        }
        return records;
    }
}

Example 38 with Row

use of io.cdap.cdap.api.dataset.table.Row in project cdap by caskdata.

the class LevelDBTableCoreTest method testDeleteRows.

@Test
public void testDeleteRows() throws Exception {
    String tableName = "testDeleteRowsTable";
    TableId tableId = TableId.from("default", tableName);
    // Single value table
    {
        Assert.assertNull(service.getTableStats().get(tableId));
        service.ensureTableExists(tableName);
        LevelDBTableCore table = new LevelDBTableCore(tableName, service);
        writeData(table, rowNamePrefix, 32, colName, 1024, 1);
        List<byte[]> rowsToDelete = new ArrayList<byte[]>();
        rowsToDelete.add(getRowName(rowNamePrefix, 1).getBytes(StandardCharsets.UTF_8));
        rowsToDelete.add(getRowName(rowNamePrefix, 5).getBytes(StandardCharsets.UTF_8));
        rowsToDelete.add(getRowName(rowNamePrefix, 7).getBytes(StandardCharsets.UTF_8));
        table.deleteRows(rowsToDelete);
        try (Scanner scanner = table.scan(Bytes.toBytes(1L), null, null, null, null)) {
            Row row;
            while ((row = scanner.next()) != null) {
                String rowName = new String(row.getRow(), StandardCharsets.UTF_8);
                Assert.assertFalse(rowsToDelete.contains(rowName.getBytes(StandardCharsets.UTF_8)));
            }
        }
        service.dropTable(tableName);
    }
    // Multi-version value
    {
        Assert.assertNull(service.getTableStats().get(tableId));
        service.ensureTableExists(tableName);
        LevelDBTableCore table = new LevelDBTableCore(tableName, service);
        writeData(table, rowNamePrefix, 32, colName, 1024, 8);
        List<byte[]> rowsToDelete = new ArrayList<byte[]>();
        rowsToDelete.add(getRowName(rowNamePrefix, 1).getBytes(StandardCharsets.UTF_8));
        rowsToDelete.add(getRowName(rowNamePrefix, 5).getBytes(StandardCharsets.UTF_8));
        rowsToDelete.add(getRowName(rowNamePrefix, 7).getBytes(StandardCharsets.UTF_8));
        table.deleteRows(rowsToDelete);
        try (Scanner scanner = table.scan(null, null, null, null, null)) {
            Row row;
            while ((row = scanner.next()) != null) {
                String rowName = new String(row.getRow(), StandardCharsets.UTF_8);
                Assert.assertFalse(rowsToDelete.contains(rowName.getBytes(StandardCharsets.UTF_8)));
            }
        }
        service.dropTable(tableName);
    }
}

Example 39 with Row

use of io.cdap.cdap.api.dataset.table.Row in project cdap by caskdata.

the class TestFrameworkTestRun method testAppWithPlugin.

@Test
public void testAppWithPlugin() throws Exception {
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("app-with-plugin", "1.0.0-SNAPSHOT");
    addAppArtifact(artifactId, AppWithPlugin.class);
    ArtifactId pluginArtifactId = NamespaceId.DEFAULT.artifact("test-plugin", "1.0.0-SNAPSHOT");
    addPluginArtifact(pluginArtifactId, artifactId, ToStringPlugin.class);
    ApplicationId appId = NamespaceId.DEFAULT.app("AppWithPlugin");
    AppRequest createRequest = new AppRequest(new ArtifactSummary(artifactId.getArtifact(), artifactId.getVersion()));
    ApplicationManager appManager = deployApplication(appId, createRequest);
    final WorkerManager workerManager = appManager.getWorkerManager(AppWithPlugin.WORKER);
    workerManager.start();
    workerManager.waitForRun(ProgramRunStatus.COMPLETED, 10, TimeUnit.SECONDS);
    final ServiceManager serviceManager = appManager.getServiceManager(AppWithPlugin.SERVICE);
    serviceManager.start();
    serviceManager.waitForRun(ProgramRunStatus.RUNNING, 10, TimeUnit.SECONDS);
    URL serviceURL = serviceManager.getServiceURL(5, TimeUnit.SECONDS);
    callServiceGet(serviceURL, "dummy");
    serviceManager.stop();
    serviceManager.waitForStopped(10, TimeUnit.SECONDS);
    WorkflowManager workflowManager = appManager.getWorkflowManager(AppWithPlugin.WORKFLOW);
    workflowManager.start();
    workflowManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES);
    List<RunRecord> runRecords = workflowManager.getHistory();
    Assert.assertNotEquals(ProgramRunStatus.FAILED, runRecords.get(0).getStatus());
    DataSetManager<KeyValueTable> workflowTableManager = getDataset(AppWithPlugin.WORKFLOW_TABLE);
    String value = Bytes.toString(workflowTableManager.get().read("val"));
    Assert.assertEquals(AppWithPlugin.TEST, value);
    Map<String, String> workflowTags = ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, NamespaceId.DEFAULT.getNamespace(), Constants.Metrics.Tag.APP, "AppWithPlugin", Constants.Metrics.Tag.WORKFLOW, AppWithPlugin.WORKFLOW, Constants.Metrics.Tag.RUN_ID, runRecords.get(0).getPid());
    getMetricsManager().waitForTotalMetricCount(workflowTags, String.format("user.destroy.%s", AppWithPlugin.WORKFLOW), 1, 60, TimeUnit.SECONDS);
    // Testing Spark Plugins. First send some data to fileset for the Spark program to process
    DataSetManager<FileSet> fileSetManager = getDataset(AppWithPlugin.SPARK_INPUT);
    FileSet fileSet = fileSetManager.get();
    try (PrintStream out = new PrintStream(fileSet.getLocation("input").append("file.txt").getOutputStream(), true, "UTF-8")) {
        for (int i = 0; i < 5; i++) {
            out.println("Message " + i);
        }
    }
    Map<String, String> sparkArgs = new HashMap<>();
    FileSetArguments.setInputPath(sparkArgs, "input");
    SparkManager sparkManager = appManager.getSparkManager(AppWithPlugin.SPARK).start(sparkArgs);
    sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 2, TimeUnit.MINUTES);
    // Verify the Spark result.
    DataSetManager<Table> dataSetManager = getDataset(AppWithPlugin.SPARK_TABLE);
    Table table = dataSetManager.get();
    try (Scanner scanner = table.scan(null, null)) {
        for (int i = 0; i < 5; i++) {
            Row row = scanner.next();
            Assert.assertNotNull(row);
            String expected = "Message " + i + " " + AppWithPlugin.TEST;
            Assert.assertEquals(expected, Bytes.toString(row.getRow()));
            Assert.assertEquals(expected, Bytes.toString(row.get(expected)));
        }
        // There shouldn't be any more rows in the table.
        Assert.assertNull(scanner.next());
    }
}

Example 40 with Row

use of io.cdap.cdap.api.dataset.table.Row in project cdap by cdapio.

the class LookupTransform method transform.

@Override
public void transform(StructuredRecord input, Emitter<StructuredRecord> emitter) throws Exception {
    T lookedUpValue = lookup.lookup((String) input.get(config.lookupKey));
    // for the output schema, copy all the input fields, and add the 'destinationField'
    List<Schema.Field> outFields = new ArrayList<>();
    for (Schema.Field field : input.getSchema().getFields()) {
        outFields.add(field);
    }
    if (lookedUpValue instanceof String) {
        outFields.add(Schema.Field.of(config.destinationField, Schema.of(Schema.Type.STRING)));
    } else if (lookedUpValue instanceof Row) {
        Row lookedupRow = (Row) lookedUpValue;
        for (byte[] column : lookedupRow.getColumns().keySet()) {
            outFields.add(Schema.Field.of(Bytes.toString(column), Schema.of(Schema.Type.STRING)));
        }
    } else {
        throw new IllegalArgumentException("Unexpected value type: " + lookedUpValue.getClass());
    }
    Schema outSchema = Schema.recordOf(input.getSchema().getRecordName(), outFields);
    // copy all the values
    StructuredRecord.Builder outputBuilder = StructuredRecord.builder(outSchema);
    for (Schema.Field inField : input.getSchema().getFields()) {
        if (inField.getName().equals(config.lookupKey)) {
            if (lookedUpValue instanceof String) {
                outputBuilder.set(config.destinationField, lookedUpValue);
            } else {
                // due to the check above, we know its a Row
                Row lookedupRow = (Row) lookedUpValue;
                for (Map.Entry<byte[], byte[]> entry : lookedupRow.getColumns().entrySet()) {
                    outputBuilder.set(Bytes.toString(entry.getKey()), Bytes.toString(entry.getValue()));
                }
            }
        }
        // what if the destinationField already exists?
        outputBuilder.set(inField.getName(), input.get(inField.getName()));
    }
    emitter.emit(outputBuilder.build());
}