Examples with CommitLogReader cz.o2.proxima.direct.commitlog.CommitLogReader used on opensource projects

Example 16 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class KafkaLogReaderIT method testReadAndWriteSequentialId.

@Test(timeout = 30_000L)
public void testReadAndWriteSequentialId() throws InterruptedException {
    initializeTestWithUri("kafka://\"${broker}\"/topic", "[commit-log]");
    final EmbeddedKafkaBroker embeddedKafka = rule.getEmbeddedKafka();
    final int numPartitions = 3;
    embeddedKafka.addTopics(new NewTopic("topic", numPartitions, (short) 1));
    final CommitLogReader commitLogReader = Optionals.get(operator.getCommitLogReader(fooDescriptor));
    final TestLogObserver observer = new TestLogObserver();
    final ObserveHandle handle = commitLogReader.observe("test-reader", Position.OLDEST, observer);
    handle.waitUntilReady();
    final int numElements = 100;
    writeElements(numElements, true);
    while (observer.getNumReceivedElements() < numElements) {
        TimeUnit.MILLISECONDS.sleep(100);
    }
    observer.getReceivedElements().forEach(e -> assertTrue(e.hasSequentialId()));
    handle.close();
    Assert.assertEquals(numElements, observer.getNumReceivedElements());
    Assert.assertEquals(numElements, numCommittedElements(handle));
}

Example 17 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class ModelTest method testReadWrite.

@Test
public void testReadWrite() throws InterruptedException {
    Regular<BaseEvent> desc = model.getEvent().getDataDescriptor();
    Optional<OnlineAttributeWriter> writer = model.directOperator().getWriter(desc);
    writer.get().write(desc.upsert(UUID.randomUUID().toString(), System.currentTimeMillis(), BaseEvent.newBuilder().setUserName("user").build()), (succ, exc) -> {
    });
    Optional<CommitLogReader> reader = model.directOperator().getCommitLogReader(desc);
    List<String> users = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    reader.get().observeBulk("consumer", Position.OLDEST, true, new CommitLogObserver() {

        @Override
        public boolean onError(Throwable error) {
            return false;
        }

        @Override
        public void onCompleted() {
            latch.countDown();
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            Optional<BaseEvent> baseEvent = desc.valueOf(ingest);
            users.add(baseEvent.get().getUserName());
            return true;
        }
    });
    latch.await();
    assertEquals("user", users.get(0));
}

Example 18 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class CommitLogReadTest method testWithMultiplePartitions.

@Test(timeout = 60000)
public void testWithMultiplePartitions() throws InterruptedException {
    int numPartitions = 3;
    int numElements = 10;
    // this test has undebuggable issues on Flink, skip it for now
    if (runner.getSimpleName().equals("DirectRunner")) {
        LocalKafkaCommitLogDescriptor kafka = new LocalKafkaCommitLogDescriptor();
        KafkaAccessor accessor = kafka.createAccessor(direct, createTestFamily(event, URI.create("kafka-test://brokers/topic-" + UUID.randomUUID().toString()), ImmutableMap.of(LocalKafkaCommitLogDescriptor.CFG_NUM_PARTITIONS, numPartitions, WatermarkConfiguration.prefixedKey(WatermarkConfiguration.CFG_ESTIMATOR_FACTORY), FiniteElementsWatermarkEstimatorFactory.class.getName(), WatermarkConfiguration.prefixedKey("numElements"), numElements, WatermarkConfiguration.prefixedKey("name"), UUID.randomUUID().toString())));
        writeElementsToKafka(numElements, accessor);
        CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
        testReadingFromCommitLogMany(numElements, CommitLogRead.of("name", Position.OLDEST, Long.MAX_VALUE, repo, reader));
    }
}

Example 19 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class CommitLogReadTest method testWithMultiplePartitionsMany.

@Test(timeout = 120000)
public void testWithMultiplePartitionsMany() throws InterruptedException {
    int numPartitions = 3;
    int numElements = 1000;
    LocalKafkaCommitLogDescriptor kafka = new LocalKafkaCommitLogDescriptor();
    KafkaAccessor accessor = kafka.createAccessor(direct, createTestFamily(event, URI.create("kafka-test://brokers/topic-" + UUID.randomUUID()), ImmutableMap.of(LocalKafkaCommitLogDescriptor.CFG_NUM_PARTITIONS, numPartitions, WatermarkConfiguration.prefixedKey(WatermarkConfiguration.CFG_ESTIMATOR_FACTORY), FiniteElementsWatermarkEstimatorFactory.class.getName(), WatermarkConfiguration.prefixedKey("numElements"), numElements, WatermarkConfiguration.prefixedKey("name"), UUID.randomUUID().toString())));
    writeElementsToKafka(numElements, accessor);
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    testReadingFromCommitLogMany(numElements, CommitLogRead.of("name", Position.OLDEST, Long.MAX_VALUE, repo, reader));
}

Example 20 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class DirectUnboundedSourceTest method testCheckpointWithNoAdvance.

@Test
public void testCheckpointWithNoAdvance() throws Exception {
    OnlineAttributeWriter writer = Optionals.get(direct.getWriter(armed));
    CommitLogReader commitLogReader = Optionals.get(direct.getCommitLogReader(armed));
    DirectUnboundedSource source = DirectUnboundedSource.of(repo.asFactory(), "name", commitLogReader, Position.OLDEST, true, Long.MAX_VALUE);
    long now = System.currentTimeMillis();
    writer.write(StreamElement.upsert(gateway, armed, UUID.randomUUID().toString(), "key", armed.getName(), now, new byte[] {}), (succ, exc) -> {
    });
    PipelineOptions opts = PipelineOptionsFactory.create();
    UnboundedSource<StreamElement, Checkpoint> split = source.split(1, opts).get(0);
    UnboundedReader<StreamElement> reader = split.createReader(opts, null);
    boolean start = reader.start();
    while (!start && !reader.advance()) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    Checkpoint mark = (Checkpoint) reader.getCheckpointMark();
    assertNotNull(mark);
    reader = split.createReader(opts, mark);
    TestUtils.assertSerializable(source);
    TestUtils.assertHashCodeAndEquals(source, SerializableUtils.clone(source));
    assertEquals(mark, reader.getCheckpointMark());
    TestUtils.assertSerializable(mark);
    TestUtils.assertHashCodeAndEquals(mark, SerializableUtils.clone(mark));
}