Examples with RecordWithMetadata org.apache.gobblin.type.RecordWithMetadata used on opensource projects

Example 11 with RecordWithMetadata

use of org.apache.gobblin.type.RecordWithMetadata in project incubator-gobblin by apache.

the class RecordWithMetadataToEnvelopedRecordWithMetadataTest method testSuccessWithInferredPrintableByteArray.

@Test
public void testSuccessWithInferredPrintableByteArray() throws DataConversionException, IOException {
    ObjectMapper objectMapper = new ObjectMapper();
    byte[] record = "abrac\\adabra".getBytes(StandardCharsets.UTF_8);
    Metadata md = new Metadata();
    md.getGlobalMetadata().setContentType("application/binary");
    md.getGlobalMetadata().addTransferEncoding("base64");
    RecordWithMetadataToEnvelopedRecordWithMetadata converter = new RecordWithMetadataToEnvelopedRecordWithMetadata();
    Iterator<RecordWithMetadata<byte[]>> recordWithMetadataIterator = converter.convertRecord("", new RecordWithMetadata<>(record, md), null).iterator();
    RecordWithMetadata recordWithMetadata = recordWithMetadataIterator.next();
    JsonNode parsedElement = objectMapper.readValue((byte[]) recordWithMetadata.getRecord(), JsonNode.class);
    Assert.assertEquals(parsedElement.get("r").getTextValue(), new String(record, StandardCharsets.UTF_8));
}

Example 12 with RecordWithMetadata

use of org.apache.gobblin.type.RecordWithMetadata in project incubator-gobblin by apache.

the class EnvelopedRecordWithMetadataToRecordWithMetadataTest method testSuccessWithRecord.

@Test
public void testSuccessWithRecord() throws DataConversionException, IOException {
    ObjectMapper objectMapper = new ObjectMapper();
    String innerRecord = "abracadabra";
    // Build the input record
    HashMap<String, Object> map = new HashMap<>();
    map.put("r", innerRecord);
    Metadata md = new Metadata();
    md.getRecordMetadata().put("test1", "test2");
    map.put("rMd", md);
    JsonNode jsonNode = objectMapper.valueToTree(map);
    RecordWithMetadata<byte[]> inputRecord = new RecordWithMetadata<>(jsonNode.toString().getBytes(), null);
    EnvelopedRecordWithMetadataToRecordWithMetadata converter = new EnvelopedRecordWithMetadataToRecordWithMetadata();
    Iterator<RecordWithMetadata<?>> iterator = converter.convertRecord(null, inputRecord, null).iterator();
    Assert.assertTrue(iterator.hasNext());
    RecordWithMetadata<?> outputRecord = iterator.next();
    Assert.assertEquals(outputRecord.getRecord(), innerRecord);
    Assert.assertEquals(outputRecord.getMetadata().getRecordMetadata().get("test1"), "test2");
}

Example 13 with RecordWithMetadata

use of org.apache.gobblin.type.RecordWithMetadata in project incubator-gobblin by apache.

the class EnvelopedRecordWithMetadataToRecordWithMetadataTest method testFailureWithoutRecord.

@Test(expectedExceptions = DataConversionException.class, expectedExceptionsMessageRegExp = "Input data does not have record.")
public void testFailureWithoutRecord() throws DataConversionException, IOException {
    ObjectMapper objectMapper = new ObjectMapper();
    // Build the input record without data
    HashMap<String, Object> map = new HashMap<>();
    Metadata md = new Metadata();
    md.getRecordMetadata().put("test1", "test2");
    map.put("rMd", md);
    JsonNode jsonNode = objectMapper.valueToTree(map);
    RecordWithMetadata<byte[]> inputRecord = new RecordWithMetadata<>(jsonNode.toString().getBytes(), null);
    EnvelopedRecordWithMetadataToRecordWithMetadata converter = new EnvelopedRecordWithMetadataToRecordWithMetadata();
    converter.convertRecord(null, inputRecord, null);
}

Example 14 with RecordWithMetadata

use of org.apache.gobblin.type.RecordWithMetadata in project incubator-gobblin by apache.

the class MetadataConverterWrapperTest method testConvertsMetadataNoOutput.

@Test
public void testConvertsMetadataNoOutput() throws DataConversionException {
    DummyConverter dummyConverter = new DummyConverter(0);
    MetadataConverterWrapper<String, String, String, String> wrapper = new MetadataConverterWrapper<>(dummyConverter);
    Iterable<RecordWithMetadata<String>> records = wrapper.convertRecord("foo", new RecordWithMetadata<String>("bar", buildMetadata(1)), new WorkUnitState());
    Assert.assertFalse(records.iterator().hasNext());
}

Example 15 with RecordWithMetadata

use of org.apache.gobblin.type.RecordWithMetadata in project incubator-gobblin by apache.

the class MetadataConverterWrapperTest method testAcceptsRawRecords.

@Test
public void testAcceptsRawRecords() throws DataConversionException {
    final int numRecordsToReturn = 1;
    DummyConverter dummyConverter = new DummyConverter(numRecordsToReturn);
    MetadataConverterWrapper<String, String, String, String> wrapper = new MetadataConverterWrapper<>(dummyConverter);
    Iterable<RecordWithMetadata<String>> records = wrapper.convertRecord("foo", "bar", new WorkUnitState());
    Iterator<RecordWithMetadata<String>> recordsIt = records.iterator();
    RecordWithMetadata<String> record = recordsIt.next();
    Assert.assertFalse(recordsIt.hasNext());
    Assert.assertEquals(record.getRecord(), "converted0");
    Assert.assertEquals(record.getMetadata().getGlobalMetadata().getId(), "0");
}