Examples with RecordWithMetadata org.apache.gobblin.type.RecordWithMetadata used on opensource projects

Example 6 with RecordWithMetadata

use of org.apache.gobblin.type.RecordWithMetadata in project incubator-gobblin by apache.

the class MetadataConverterWrapperTest method testConvertsMetadataMultiOutput.

@Test
public void testConvertsMetadataMultiOutput() throws DataConversionException {
    final int numRecordsToReturn = 2;
    DummyConverter dummyConverter = new DummyConverter(numRecordsToReturn);
    MetadataConverterWrapper<String, String, String, String> wrapper = new MetadataConverterWrapper<>(dummyConverter);
    Iterable<RecordWithMetadata<String>> records1 = wrapper.convertRecord("foo", new RecordWithMetadata<String>("bar", buildMetadata(1)), new WorkUnitState());
    Iterable<RecordWithMetadata<String>> records2 = wrapper.convertRecord("foo", new RecordWithMetadata<String>("baz", buildMetadata(2)), new WorkUnitState());
    Iterator<RecordWithMetadata<String>> record1It = records1.iterator();
    Iterator<RecordWithMetadata<String>> record2It = records2.iterator();
    for (int i = 0; i < numRecordsToReturn; i++) {
        RecordWithMetadata<String> record1 = record1It.next();
        Assert.assertEquals(record1.getRecord(), "converted" + String.valueOf(i));
        Assert.assertEquals(record1.getMetadata().getGlobalMetadata().getDatasetUrn(), "dataset-id:1");
        RecordWithMetadata<String> record2 = record2It.next();
        Assert.assertEquals(record2.getRecord(), "converted" + String.valueOf(i));
        Assert.assertEquals(record2.getMetadata().getGlobalMetadata().getDatasetUrn(), "dataset-id:2");
    }
}

Example 7 with RecordWithMetadata

use of org.apache.gobblin.type.RecordWithMetadata in project incubator-gobblin by apache.

the class MetadataWriterWrapperTest method testAppendsDefaultMetadata.

@Test
public void testAppendsDefaultMetadata() throws IOException {
    state = new WorkUnitState();
    dummyWriter = new MetadataDummyWriter();
    writer = new MetadataWriterWrapper<>(dummyWriter, byte[].class, 1, 0, state.getJobState());
    byte[] recordBytes = new byte[] { 'a', 'b', 'c', 'd' };
    Metadata md = new Metadata();
    md.getGlobalMetadata().addTransferEncoding("first");
    writer.write(new RecordWithMetadata<>(recordBytes, md));
    writer.commit();
    String writerMetadata = state.getProp(ConfigurationKeys.WRITER_METADATA_KEY);
    Assert.assertNotNull(writerMetadata, "Expected there to be metadata");
    int firstOccurrence = writerMetadata.indexOf("\"first\"");
    Assert.assertNotEquals(firstOccurrence, -1, "Expected to find record-level encoding in metadata");
    int secondOccurrence = writerMetadata.indexOf("\"default-encoding\"");
    Assert.assertNotEquals(secondOccurrence, -1, "Expected to find default metadata in metadata");
    Assert.assertTrue(firstOccurrence < secondOccurrence, "Expected recordBytes encoding to be present before default encoding");
}

Example 8 with RecordWithMetadata

use of org.apache.gobblin.type.RecordWithMetadata in project incubator-gobblin by apache.

the class MetadataWriterWrapperTest method recordsMetadata.

@Test
public void recordsMetadata() throws IOException {
    final String URN = "unit-test:dataset";
    byte[] record = new byte[] { 'a', 'b', 'c', 'd', 'e' };
    dummyWriter.setExpectedRecord(record);
    Metadata md = new Metadata();
    md.getGlobalMetadata().setDatasetUrn(URN);
    RecordWithMetadata<byte[]> mdRecord = new RecordWithMetadata<>(record, md);
    writer.write(mdRecord);
    writer.commit();
    Assert.assertEquals(dummyWriter.recordsWritten(), 1);
    String writerMetadata = state.getProp(ConfigurationKeys.WRITER_METADATA_KEY);
    Assert.assertNotNull(writerMetadata, "Expected there to be metadata");
    Assert.assertEquals(1, countOccurrences(writerMetadata, URN));
    // Write a 2nd record with the same metadata; it should _not_ be included twice in output
    byte[] record2 = new byte[] { 'e', 'f', 'g', 'h' };
    dummyWriter.setExpectedRecord(record2);
    Metadata md2 = new Metadata();
    md2.getGlobalMetadata().setDatasetUrn(URN);
    writer.write(new RecordWithMetadata<>(record2, md2));
    writer.commit();
    Assert.assertEquals(dummyWriter.recordsWritten(), 2);
    writerMetadata = state.getProp(ConfigurationKeys.WRITER_METADATA_KEY);
    Assert.assertNotNull(writerMetadata, "Expected there to be metadata");
    Assert.assertEquals(1, countOccurrences(writerMetadata, URN));
    // and now a 3rd
    // Write a 2nd record with the same metadata; it should _not_ be included twice in output
    byte[] record3 = new byte[] { 'i', 'j', 'k', 'l' };
    dummyWriter.setExpectedRecord(record3);
    Metadata md3 = new Metadata();
    md3.getGlobalMetadata().setDatasetUrn(URN + "_other");
    writer.write(new RecordWithMetadata<>(record3, md3));
    writer.commit();
    Assert.assertEquals(dummyWriter.recordsWritten(), 3);
    writerMetadata = state.getProp(ConfigurationKeys.WRITER_METADATA_KEY);
    Assert.assertNotNull(writerMetadata, "Expected there to be metadata");
    Assert.assertEquals(2, countOccurrences(writerMetadata, URN));
}

Example 9 with RecordWithMetadata

use of org.apache.gobblin.type.RecordWithMetadata in project incubator-gobblin by apache.

the class RecordWithMetadataToEnvelopedRecordWithMetadataTest method testSuccessWithJson.

@Test
public void testSuccessWithJson() throws SchemaConversionException, DataConversionException, IOException {
    final String innerContentType = "randomJsonRecord";
    ObjectMapper objectMapper = new ObjectMapper();
    RecordWithMetadataToEnvelopedRecordWithMetadata converter = new RecordWithMetadataToEnvelopedRecordWithMetadata();
    converter.convertSchema("", null);
    // Build Test Record
    HashMap<String, String> map = new HashMap<>();
    map.put("test", "test");
    map.put("value", "value");
    JsonNode jsonElement = objectMapper.valueToTree(map);
    Metadata md = new Metadata();
    md.getGlobalMetadata().setDatasetUrn("my-dataset");
    md.getGlobalMetadata().setContentType(innerContentType);
    md.getRecordMetadata().put("foo", "bar");
    RecordWithMetadata<JsonNode> record = new RecordWithMetadata(jsonElement, md);
    // Convert it
    Iterator<RecordWithMetadata<byte[]>> recordWithMetadataIterator = converter.convertRecord("", record, null).iterator();
    RecordWithMetadata recordWithMetadata = recordWithMetadataIterator.next();
    // Verify it
    JsonNode parsedElement = objectMapper.readValue((byte[]) recordWithMetadata.getRecord(), JsonNode.class);
    Assert.assertEquals(parsedElement.get("mId").getTextValue(), record.getMetadata().getGlobalMetadata().getId());
    Assert.assertEquals(parsedElement.get("r"), jsonElement);
    Assert.assertEquals(parsedElement.get("rMd").get("foo").getTextValue(), "bar");
    Assert.assertEquals(recordWithMetadata.getMetadata().getGlobalMetadata().getContentType(), "lnkd+recordWithMetadata");
    Assert.assertEquals(recordWithMetadata.getMetadata().getGlobalMetadata().getInnerContentType(), innerContentType);
}

Example 10 with RecordWithMetadata

use of org.apache.gobblin.type.RecordWithMetadata in project incubator-gobblin by apache.

the class RecordWithMetadataToEnvelopedRecordWithMetadataTest method testSuccessWithBinary.

@Test
public void testSuccessWithBinary() throws DataConversionException, IOException {
    ObjectMapper objectMapper = new ObjectMapper();
    byte[] record = "aaaa".getBytes(StandardCharsets.UTF_8);
    Metadata md = new Metadata();
    md.getGlobalMetadata().setContentType("application/binary");
    RecordWithMetadataToEnvelopedRecordWithMetadata converter = new RecordWithMetadataToEnvelopedRecordWithMetadata();
    Iterator<RecordWithMetadata<byte[]>> recordWithMetadataIterator = converter.convertRecord("", new RecordWithMetadata<>(record, md), null).iterator();
    RecordWithMetadata recordWithMetadata = recordWithMetadataIterator.next();
    JsonNode parsedElement = objectMapper.readValue((byte[]) recordWithMetadata.getRecord(), JsonNode.class);
    Assert.assertEquals(parsedElement.get("r").getTextValue(), "YWFhYQ==");
}