Examples with StructuredRecord io.cdap.cdap.api.data.format.StructuredRecord used on opensource projects

Example 96 with StructuredRecord

use of io.cdap.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class MockRuntimeDatasetSource method writeInput.

/**
 * Used to write the input records for the pipeline run. Should be called after the pipeline has been created.
 *
 * @param tableManager dataset manager used to write to the source dataset
 * @param records records that should be the input for the pipeline
 */
public static void writeInput(DataSetManager<Table> tableManager, Iterable<StructuredRecord> records) throws Exception {
    tableManager.flush();
    Table table = tableManager.get();
    // each rowkey will be a UUID.
    for (StructuredRecord record : records) {
        byte[] row = Bytes.toBytes(UUID.randomUUID());
        table.put(row, SCHEMA_COL, Bytes.toBytes(record.getSchema().toString()));
        table.put(row, RECORD_COL, Bytes.toBytes(StructuredRecordStringConverter.toJsonString(record)));
    }
    tableManager.flush();
}

Example 97 with StructuredRecord

use of io.cdap.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class MockSQLEngineWithCapabilities method writeInput.

/**
 * Used to write the input records for the pipeline run. Should be called after the pipeline has been created.
 *
 * @param fileName file to write the records into
 * @param records records that should be the input for the pipeline
 */
public static void writeInput(String fileName, Iterable<StructuredRecord> records) throws Exception {
    Function<StructuredRecord, String> mapper = input -> {
        try {
            return StructuredRecordStringConverter.toJsonString(input);
        } catch (IOException e) {
            throw new RuntimeException("Unable to set up file for test.", e);
        }
    };
    String output = Joiner.on("\n").join(Iterables.transform(records, mapper));
    Files.write(output, new File(fileName), Charsets.UTF_8);
}

Example 98 with StructuredRecord

use of io.cdap.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class MockJoiner method merge.

@Override
public StructuredRecord merge(StructuredRecord joinKey, Iterable<JoinElement<StructuredRecord>> joinRow) {
    StructuredRecord.Builder outRecordBuilder;
    outRecordBuilder = StructuredRecord.builder(outputSchema);
    for (JoinElement<StructuredRecord> joinElement : joinRow) {
        StructuredRecord record = joinElement.getInputRecord();
        for (Schema.Field field : record.getSchema().getFields()) {
            outRecordBuilder.set(field.getName(), record.get(field.getName()));
        }
    }
    return outRecordBuilder.build();
}

Example 99 with StructuredRecord

use of io.cdap.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class AbstractMockSink method readOutput.

/**
 * Used to read the records written by this sink.
 *
 * @param tableManager dataset manager used to get the sink dataset to read from
 */
public static List<StructuredRecord> readOutput(DataSetManager<Table> tableManager) throws Exception {
    tableManager.flush();
    Table table = tableManager.get();
    try (Scanner scanner = table.scan(null, null)) {
        List<StructuredRecord> records = new ArrayList<>();
        Row row;
        while ((row = scanner.next()) != null) {
            Schema schema = Schema.parseJson(row.getString(SCHEMA_COL));
            String recordStr = row.getString(RECORD_COL);
            records.add(StructuredRecordStringConverter.fromJsonString(recordStr, schema));
        }
        return records;
    }
}

Example 100 with StructuredRecord

use of io.cdap.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class FilterTransform method getOutputBuilder.

private StructuredRecord.Builder getOutputBuilder(StructuredRecord input) {
    List<Schema.Field> outFields = new ArrayList<>();
    for (Schema.Field field : input.getSchema().getFields()) {
        outFields.add(field);
    }
    Schema outSchema = Schema.recordOf(input.getSchema().getRecordName(), outFields);
    // copy all the values
    StructuredRecord.Builder outputBuilder = StructuredRecord.builder(outSchema);
    for (Schema.Field inField : input.getSchema().getFields()) {
        outFields.add(inField);
        outputBuilder.set(inField.getName(), input.get(inField.getName()));
    }
    return outputBuilder;
}