Examples with CommitLogReader cz.o2.proxima.direct.commitlog.CommitLogReader used on opensource projects

Example 51 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class LocalKafkaCommitLogDescriptorTest method testSlowPollMovesWatermarkSlowly.

@Test(timeout = 10000)
public void testSlowPollMovesWatermarkSlowly() throws InterruptedException {
    Accessor accessor = kafka.createAccessor(direct, createTestFamily(entity, storageUri, and(partitionsCfg(3), cfg(Pair.of(KafkaAccessor.EMPTY_POLL_TIME, "1000")))));
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(context()));
    final long now = System.currentTimeMillis();
    AtomicLong watermark = new AtomicLong();
    CountDownLatch latch = new CountDownLatch(30);
    reader.observe("test", Position.NEWEST, new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            return true;
        }

        @Override
        public void onCompleted() {
            fail("This should not be called");
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }

        @Override
        public void onIdle(OnIdleContext context) {
            watermark.set(context.getWatermark());
            latch.countDown();
        }
    }).waitUntilReady();
    // for two seconds we have empty data
    TimeUnit.SECONDS.sleep(2);
    latch.await();
    // watermark should be moved
    assertTrue(watermark.get() > 0);
    assertTrue(watermark.get() < now * 10);
}

Example 52 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class LocalKafkaCommitLogDescriptorTest method testPollFromNConsumersMovesWatermark.

void testPollFromNConsumersMovesWatermark(Accessor accessor, int numObservers, boolean expectMoved) throws InterruptedException {
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(context()));
    final long now = System.currentTimeMillis();
    CountDownLatch latch = new CountDownLatch(numObservers);
    Set<CommitLogObserver> movedConsumers = Collections.synchronizedSet(new HashSet<>());
    Map<CommitLogObserver, Long> observerWatermarks = new ConcurrentHashMap<>();
    AtomicInteger readyObservers = new AtomicInteger();
    for (int i = 0; i < numObservers; i++) {
        reader.observe("test-" + expectMoved, Position.OLDEST, new CommitLogObserver() {

            @Override
            public boolean onNext(StreamElement ingest, OnNextContext context) {
                log.debug("Received element {} on watermark {}", ingest, context.getWatermark());
                return true;
            }

            @Override
            public boolean onError(Throwable error) {
                throw new RuntimeException(error);
            }

            @Override
            public void onIdle(OnIdleContext context) {
                if (readyObservers.get() == numObservers) {
                    observerWatermarks.compute(this, (k, v) -> Math.max(MoreObjects.firstNonNull(v, Long.MIN_VALUE), context.getWatermark()));
                    if ((!expectMoved || observerWatermarks.get(this) > 0) && movedConsumers.add(this)) {
                        latch.countDown();
                    }
                }
            }
        }).waitUntilReady();
        readyObservers.incrementAndGet();
    }
    assertTrue(latch.await(10, TimeUnit.SECONDS));
    assertEquals(numObservers, observerWatermarks.size());
    long watermark = observerWatermarks.values().stream().min(Long::compare).orElse(Long.MIN_VALUE);
    // watermark should be moved
    assertTrue(!expectMoved || watermark > 0);
    assertTrue("Watermark should not be too far, got " + watermark + " calculated from " + observerWatermarks, watermark < now * 10);
}

Example 53 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveNonExternalizableMultipleConsumers.

@Test(timeout = 10000)
public void testObserveNonExternalizableMultipleConsumers() throws InterruptedException {
    int numElements = 1000;
    CommitLogReader reader = ListCommitLog.ofNonExternalizable(data(numElements), direct.getContext());
    ExecutorService executor = Executors.newCachedThreadPool();
    String name = "name" + UUID.randomUUID().toString();
    List<StreamElement> list = Collections.synchronizedList(new ArrayList<>());
    int numThreads = 10;
    CountDownLatch latch = new CountDownLatch(numThreads);
    for (int i = 0; i < numThreads; i++) {
        executor.submit(() -> {
            reader.observe(name, toList(list, ign -> latch.countDown()));
        });
    }
    latch.await();
    assertEquals("Duplicate: " + list.stream().collect(Collectors.toMap(Function.identity(), e -> 1, Integer::sum)).entrySet().stream().filter(e -> e.getValue() > 1).collect(Collectors.toList()).toString(), numElements, list.size());
}

Example 54 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveExternalizableUnnamedPauseContinue.

@Test(timeout = 10000)
public void testObserveExternalizableUnnamedPauseContinue() throws InterruptedException {
    CommitLogReader reader = ListCommitLog.of(data(10), direct.getContext());
    List<StreamElement> data = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    ObserveHandle handle = reader.observe(null, toList(data, b -> latch.countDown(), v -> v.getValue()[0] < 5));
    latch.await();
    assertEquals(6, data.size());
    assertTrue(handle.getCommittedOffsets().isEmpty());
    assertFalse(handle.getCurrentOffsets().isEmpty());
    CountDownLatch nextLatch = new CountDownLatch(1);
    reader.observeBulkOffsets(handle.getCurrentOffsets(), toList(data, b -> nextLatch.countDown()));
    nextLatch.await();
    assertEquals(11, data.size());
}

Example 55 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveExternalizableUnnamed.

@Test(timeout = 10000)
public void testObserveExternalizableUnnamed() throws InterruptedException {
    CommitLogReader reader = ListCommitLog.of(data(10), direct.getContext());
    List<StreamElement> data = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    ObserveHandle handle = reader.observe(null, toList(data, b -> latch.countDown()));
    latch.await();
    assertEquals(10, data.size());
    assertTrue(handle.getCommittedOffsets().isEmpty());
    assertFalse(handle.getCurrentOffsets().isEmpty());
}