Examples with BasicAckableForTesting org.apache.gobblin.ack.BasicAckableForTesting used on opensource projects

Example 1 with BasicAckableForTesting

use of org.apache.gobblin.ack.BasicAckableForTesting in project incubator-gobblin by apache.

the class ConverterTest method testMultiOutputIterable.

@Test
public void testMultiOutputIterable() throws Exception {
    MyConverter converter = new MyConverter();
    BasicAckableForTesting ackable = new BasicAckableForTesting();
    RecordStreamWithMetadata<Integer, String> stream = new RecordStreamWithMetadata<>(Flowable.just(new RecordEnvelope<>(2)), GlobalMetadata.<String>builder().schema("schema").build()).mapRecords(r -> {
        r.addCallBack(ackable);
        return r;
    });
    List<StreamEntity<Integer>> outputRecords = Lists.newArrayList();
    converter.processStream(stream, new WorkUnitState()).getRecordStream().subscribe(outputRecords::add);
    Assert.assertEquals(outputRecords.size(), 2);
    // output record has not been acked
    Assert.assertEquals(ackable.acked, 0);
    outputRecords.get(0).ack();
    // only one output record acked, still need to ack another derived record
    Assert.assertEquals(ackable.acked, 0);
    outputRecords.get(1).ack();
    // all output records acked
    Assert.assertEquals(ackable.acked, 1);
}

Example 2 with BasicAckableForTesting

use of org.apache.gobblin.ack.BasicAckableForTesting in project incubator-gobblin by apache.

the class ConverterTest method testMixedStream.

@Test
public void testMixedStream() throws Exception {
    MyConverter converter = new MyConverter();
    BasicAckableForTesting ackable = new BasicAckableForTesting();
    RecordStreamWithMetadata<Integer, String> stream = new RecordStreamWithMetadata<>(Flowable.just(new RecordEnvelope<>(1), new MyControlMessage<>()), GlobalMetadata.<String>builder().schema("schema").build()).mapRecords(r -> {
        r.addCallBack(ackable);
        return r;
    });
    List<StreamEntity<Integer>> outputRecords = Lists.newArrayList();
    converter.processStream(stream, new WorkUnitState()).getRecordStream().subscribe(outputRecords::add);
    Assert.assertEquals(outputRecords.size(), 2);
    Assert.assertEquals(((RecordEnvelope<Integer>) outputRecords.get(0)).getRecord(), new Integer(0));
    Assert.assertTrue(outputRecords.get(1) instanceof MyControlMessage);
}

Example 3 with BasicAckableForTesting

use of org.apache.gobblin.ack.BasicAckableForTesting in project incubator-gobblin by apache.

the class StreamEntityTest method testSingleCloning.

@Test
public void testSingleCloning() {
    MyStreamEntity streamEntity = new MyStreamEntity();
    BasicAckableForTesting ackable = new BasicAckableForTesting();
    streamEntity.addCallBack(ackable);
    MyStreamEntity clone = (MyStreamEntity) streamEntity.getSingleClone();
    Assert.assertEquals(clone.id, streamEntity.id);
    try {
        streamEntity.getSingleClone();
        Assert.fail();
    } catch (IllegalStateException ise) {
    // expected, cannot clone twice using getSingleClone
    }
    try {
        streamEntity.forkCloner();
        Assert.fail();
    } catch (IllegalStateException ise) {
    // expected, cannot clone twice
    }
    clone.ack();
    Assert.assertEquals(ackable.acked, 1);
}

Example 4 with BasicAckableForTesting

use of org.apache.gobblin.ack.BasicAckableForTesting in project incubator-gobblin by apache.

the class StreamEntityTest method testMultipleClones.

@Test
public void testMultipleClones() {
    MyStreamEntity streamEntity = new MyStreamEntity();
    BasicAckableForTesting ackable = new BasicAckableForTesting();
    streamEntity.addCallBack(ackable);
    StreamEntity.ForkCloner cloner = streamEntity.forkCloner();
    MyStreamEntity clone1 = (MyStreamEntity) cloner.getClone();
    Assert.assertEquals(clone1.id, streamEntity.id);
    clone1.ack();
    // cloner has not been closed, so ack does not spread
    Assert.assertEquals(ackable.acked, 0);
    MyStreamEntity clone2 = (MyStreamEntity) cloner.getClone();
    Assert.assertEquals(clone2.id, streamEntity.id);
    // close cloner to spread acks
    cloner.close();
    // ack second clone, should ack original
    clone2.ack();
    Assert.assertEquals(ackable.acked, 1);
    try {
        cloner.getClone();
        Assert.fail();
    } catch (IllegalStateException ise) {
    // cloner has been closed, cannot create new clones
    }
    try {
        streamEntity.getSingleClone();
        Assert.fail();
    } catch (IllegalStateException ise) {
    // expected, cannot clone twice
    }
    try {
        streamEntity.forkCloner();
        Assert.fail();
    } catch (IllegalStateException ise) {
    // expected, cannot clone twice
    }
}

Example 5 with BasicAckableForTesting

use of org.apache.gobblin.ack.BasicAckableForTesting in project incubator-gobblin by apache.

the class PartitionedWriterTest method testControlMessageHandler.

@Test
public void testControlMessageHandler() throws IOException {
    State state = new State();
    state.setProp(ConfigurationKeys.WRITER_PARTITIONER_CLASS, TestPartitioner.class.getCanonicalName());
    TestPartitionAwareWriterBuilder builder = new TestPartitionAwareWriterBuilder();
    PartitionedDataWriter writer = new PartitionedDataWriter<String, String>(builder, state);
    Assert.assertEquals(builder.actions.size(), 0);
    String record1 = "abc";
    writer.writeEnvelope(new RecordEnvelope(record1));
    String record2 = "123";
    writer.writeEnvelope(new RecordEnvelope(record2));
    FlushControlMessage controlMessage = FlushControlMessage.builder().build();
    BasicAckableForTesting ackable = new BasicAckableForTesting();
    controlMessage.addCallBack(ackable);
    Assert.assertEquals(ackable.acked, 0);
    // when the control message is cloned properly then this does not raise an error
    writer.getMessageHandler().handleMessage(controlMessage);
    // message handler does not ack since consumeRecordStream does acking for control messages
    // this should be revisited when control message error handling is changed
    controlMessage.ack();
    Assert.assertEquals(ackable.acked, 1);
    writer.close();
}