Examples with StructuredRecord co.cask.cdap.api.data.format.StructuredRecord used on opensource projects

Example 41 with StructuredRecord

use of co.cask.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class MockJoiner method joinOn.

@Override
public StructuredRecord joinOn(String stageName, StructuredRecord record) throws Exception {
    List<Schema.Field> fields = new ArrayList<>();
    Schema schema = record.getSchema();
    // TODO create output record based on fields properties
    Map<String, List<String>> stageToJoinKey = config.getJoinKeys();
    List<String> joinKeys = stageToJoinKey.get(stageName);
    int i = 1;
    for (String joinKey : joinKeys) {
        Schema.Field joinField = Schema.Field.of(String.valueOf(i++), schema.getField(joinKey).getSchema());
        fields.add(joinField);
    }
    Schema keySchema = Schema.recordOf("join.key", fields);
    StructuredRecord.Builder keyRecordBuilder = StructuredRecord.builder(keySchema);
    i = 1;
    for (String joinKey : joinKeys) {
        keyRecordBuilder.set(String.valueOf(i++), record.get(joinKey));
    }
    return keyRecordBuilder.build();
}

Example 42 with StructuredRecord

use of co.cask.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class MockJoiner method merge.

@Override
public StructuredRecord merge(StructuredRecord joinKey, Iterable<JoinElement<StructuredRecord>> joinRow) {
    StructuredRecord.Builder outRecordBuilder;
    outRecordBuilder = StructuredRecord.builder(outputSchema);
    for (JoinElement<StructuredRecord> joinElement : joinRow) {
        StructuredRecord record = joinElement.getInputRecord();
        for (Schema.Field field : record.getSchema().getFields()) {
            outRecordBuilder.set(field.getName(), record.get(field.getName()));
        }
    }
    return outRecordBuilder.build();
}

Example 43 with StructuredRecord

use of co.cask.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class MockSink method write.

@Override
public int write(Iterable<StructuredRecord> records, DataWriter dataWriter) throws Exception {
    if (dir == null) {
        return 0;
    }
    File outputFile = new File(dir, String.valueOf(count));
    File doneFile = new File(dir, String.valueOf(count) + ".done");
    List<StructuredRecord> outputRecords = new ArrayList<>();
    for (StructuredRecord record : records) {
        outputRecords.add(record);
    }
    Files.write(outputFile.toPath(), GSON.toJson(records).getBytes(StandardCharsets.UTF_8));
    doneFile.createNewFile();
    count++;
    return outputRecords.size();
}

Example 44 with StructuredRecord

use of co.cask.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class FlattenErrorTransform method transform.

@Override
public void transform(ErrorRecord<StructuredRecord> input, Emitter<StructuredRecord> emitter) throws Exception {
    StructuredRecord invalidRecord = input.getRecord();
    StructuredRecord.Builder output = StructuredRecord.builder(getOutputSchema(invalidRecord.getSchema()));
    for (Schema.Field field : invalidRecord.getSchema().getFields()) {
        output.set(field.getName(), invalidRecord.get(field.getName()));
    }
    emitter.emit(output.set("errMsg", input.getErrorMessage()).set("errCode", input.getErrorCode()).set("errStage", input.getStageName()).build());
}

Example 45 with StructuredRecord

use of co.cask.cdap.api.data.format.StructuredRecord in project cdap by caskdata.

the class JsonStructuredRecordDatumReader method decodeRecord.

@Override
protected StructuredRecord decodeRecord(Decoder decoder, Schema schema) throws IOException {
    StructuredRecord.Builder builder = StructuredRecord.builder(schema);
    JsonReader jsonReader = getJsonReader(decoder);
    jsonReader.beginObject();
    while (jsonReader.peek() != JsonToken.END_OBJECT) {
        Schema.Field field = schema.getField(jsonReader.nextName());
        if (field == null) {
            // Ignore unrecognized fields
            jsonReader.skipValue();
            continue;
        }
        builder.set(field.getName(), decode(decoder, field.getSchema()));
    }
    jsonReader.endObject();
    return builder.build();
}