Examples with RecordWithMetadata org.apache.gobblin.type.RecordWithMetadata used on opensource projects

Example 1 with RecordWithMetadata

use of org.apache.gobblin.type.RecordWithMetadata in project incubator-gobblin by apache.

the class MetadataWriterWrapper method write.

@Override
@SuppressWarnings("unchecked")
public void write(Object untypedRecord) throws IOException {
    if (untypedRecord instanceof RecordWithMetadata) {
        RecordWithMetadata record = (RecordWithMetadata) untypedRecord;
        GlobalMetadata globalMetadata = record.getMetadata().getGlobalMetadata();
        metadataCollector.processMetadata(globalMetadata);
        if (RecordWithMetadata.class.isAssignableFrom(writerDataClass)) {
            wrappedWriter.write(record);
        } else {
            wrappedWriter.write(record.getRecord());
        }
    } else {
        metadataCollector.processMetadata(null);
        wrappedWriter.write(untypedRecord);
    }
}

Example 2 with RecordWithMetadata

use of org.apache.gobblin.type.RecordWithMetadata in project incubator-gobblin by apache.

the class EncryptedSerializedRecordToSerializedRecordConverterTest method setUp.

@BeforeTest
public void setUp() {
    workUnitState = new WorkUnitState();
    converter = new EncryptedSerializedRecordToSerializedRecordConverter();
    sampleRecord = new RecordWithMetadata<>(new byte[] { 'b', 'c', 'd', 'e' }, new Metadata());
    shiftedValue = new byte[] { 'a', 'b', 'c', 'd' };
    insecureShiftTag = InsecureShiftCodec.TAG;
}

Example 3 with RecordWithMetadata

use of org.apache.gobblin.type.RecordWithMetadata in project incubator-gobblin by apache.

the class RecordWithMetadataToEnvelopedRecordWithMetadataTest method testSuccessWithString.

@Test
public void testSuccessWithString() throws DataConversionException, IOException {
    ObjectMapper objectMapper = new ObjectMapper();
    String innerRecord = "abracadabra";
    RecordWithMetadataToEnvelopedRecordWithMetadata converter = new RecordWithMetadataToEnvelopedRecordWithMetadata();
    RecordWithMetadata<String> record = new RecordWithMetadata<>(innerRecord, new Metadata());
    Iterator<RecordWithMetadata<byte[]>> recordWithMetadataIterator = converter.convertRecord("", record, null).iterator();
    RecordWithMetadata recordWithMetadata = recordWithMetadataIterator.next();
    JsonNode parsedElement = objectMapper.readValue((byte[]) recordWithMetadata.getRecord(), JsonNode.class);
    Assert.assertEquals(parsedElement.get("mId").getTextValue(), record.getMetadata().getGlobalMetadata().getId());
    Assert.assertEquals(parsedElement.get("r").getTextValue(), innerRecord);
    Assert.assertEquals(recordWithMetadata.getMetadata().getGlobalMetadata().getContentType(), "lnkd+recordWithMetadata");
    Assert.assertNull(recordWithMetadata.getMetadata().getGlobalMetadata().getInnerContentType());
}

Example 4 with RecordWithMetadata

use of org.apache.gobblin.type.RecordWithMetadata in project incubator-gobblin by apache.

the class EnvelopedRecordWithMetadataToRecordWithMetadata method convertRecord.

@Override
public Iterable<RecordWithMetadata<?>> convertRecord(Object outputSchema, RecordWithMetadata<byte[]> inputRecord, WorkUnitState workUnit) throws DataConversionException {
    try {
        try (JsonParser parser = jsonFactory.createJsonParser(inputRecord.getRecord())) {
            parser.setCodec(objectMapper);
            JsonNode jsonNode = parser.readValueAsTree();
            // extracts required record
            if (!jsonNode.has(RECORD_KEY)) {
                throw new DataConversionException("Input data does not have record.");
            }
            String record = jsonNode.get(RECORD_KEY).getTextValue();
            // Extract metadata field
            Metadata md = new Metadata();
            if (jsonNode.has(METADATA_KEY) && jsonNode.get(METADATA_KEY).has(METADATA_RECORD_KEY)) {
                md.getRecordMetadata().putAll(objectMapper.readValue(jsonNode.get(METADATA_KEY).get(METADATA_RECORD_KEY), Map.class));
            }
            return Collections.singleton(new RecordWithMetadata<>(record, md));
        }
    } catch (IOException e) {
        throw new DataConversionException(e);
    }
}

Example 5 with RecordWithMetadata

use of org.apache.gobblin.type.RecordWithMetadata in project incubator-gobblin by apache.

the class MetadataConverterWrapper method convertRecord.

@Override
public Iterable<RecordWithMetadata<DO>> convertRecord(SO outputSchema, Object inputRecord, WorkUnitState workUnit) throws DataConversionException {
    final Metadata metadata = getMetadataFromRecord(inputRecord);
    final DI innerRecord = getRecordFromRecord(inputRecord);
    Iterable<DO> outputRecords = innerConverter.convertRecord(outputSchema, innerRecord, workUnit);
    return Iterables.transform(outputRecords, new Function<DO, RecordWithMetadata<DO>>() {

        @Nullable
        @Override
        public RecordWithMetadata<DO> apply(@Nullable DO input) {
            return new RecordWithMetadata<>(input, metadata);
        }
    });
}