Examples with StreamEntity org.apache.gobblin.stream.StreamEntity used on opensource projects

Example 6 with StreamEntity

use of org.apache.gobblin.stream.StreamEntity in project incubator-gobblin by apache.

the class ConverterTest method testSingleOutputIterable.

@Test
public void testSingleOutputIterable() throws Exception {
    MyConverter converter = new MyConverter();
    BasicAckableForTesting ackable = new BasicAckableForTesting();
    RecordStreamWithMetadata<Integer, String> stream = new RecordStreamWithMetadata<>(Flowable.just(new RecordEnvelope<>(1)), GlobalMetadata.<String>builder().schema("schema").build()).mapRecords(r -> {
        r.addCallBack(ackable);
        return r;
    });
    List<StreamEntity<Integer>> outputRecords = Lists.newArrayList();
    converter.processStream(stream, new WorkUnitState()).getRecordStream().subscribe(outputRecords::add);
    Assert.assertEquals(outputRecords.size(), 1);
    // output record has not been acked
    Assert.assertEquals(ackable.acked, 0);
    outputRecords.get(0).ack();
    // output record acked
    Assert.assertEquals(ackable.acked, 1);
}

Example 7 with StreamEntity

use of org.apache.gobblin.stream.StreamEntity in project incubator-gobblin by apache.

the class ConverterTest method testEmptyOutputIterable.

@Test
public void testEmptyOutputIterable() throws Exception {
    MyConverter converter = new MyConverter();
    BasicAckableForTesting ackable = new BasicAckableForTesting();
    RecordStreamWithMetadata<Integer, String> stream = new RecordStreamWithMetadata<>(Flowable.just(new RecordEnvelope<>(0)), GlobalMetadata.<String>builder().schema("schema").build()).mapRecords(r -> {
        r.addCallBack(ackable);
        return r;
    });
    List<StreamEntity<Integer>> outputRecords = Lists.newArrayList();
    converter.processStream(stream, new WorkUnitState()).getRecordStream().subscribe(outputRecords::add);
    Assert.assertEquals(outputRecords.size(), 0);
    // record got filtered, acked immediately
    Assert.assertEquals(ackable.acked, 1);
}