Examples with ObserveHandle cz.o2.proxima.direct.commitlog.ObserveHandle used on opensource projects

Example 16 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class PubSubReaderTest method testPartitionsSplit.

@Test
public void testPartitionsSplit() throws InterruptedException {
    List<Partition> partitions = reader.getPartitions();
    assertEquals(1, partitions.size());
    partitions = new ArrayList<>(partitions.get(0).split(3));
    assertEquals(3, partitions.size());
    reader.setSupplier(() -> {
        LockSupport.park();
        return null;
    });
    CommitLogObserver observer = new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            context.confirm();
            return false;
        }

        @Override
        public void onCancelled() {
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    };
    try (ObserveHandle handle = reader.observeBulkPartitions(partitions, Position.NEWEST, observer)) {
        handle.waitUntilReady();
    }
}

Example 17 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class PubSubReaderTest method testObserveCommittedOffset.

@Test
public void testObserveCommittedOffset() throws InterruptedException {
    long now = System.currentTimeMillis();
    Deque<PubsubMessage> inputs = new LinkedList<>(Arrays.asList(update("key1", "attr", new byte[] { 1, 2 }, now), delete("key2", "attr", now + 1000), deleteWildcard("key3", wildcard, now)));
    reader.setSupplier(() -> {
        if (inputs.isEmpty()) {
            LockSupport.park();
        }
        return inputs.pop();
    });
    CountDownLatch latch = new CountDownLatch(1);
    ObserveHandle handle = reader.observe("dummy", new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            timestampSupplier.addAndGet(1000);
            context.confirm();
            latch.countDown();
            return false;
        }

        @Override
        public void onCancelled() {
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    latch.await();
    assertEquals(1, handle.getCommittedOffsets().size());
    assertTrue(handle.getCommittedOffsets().get(0).getWatermark() > 0);
}

Example 18 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class PubSubReaderTest method testObserveBulk.

@Test(timeout = 10000)
public void testObserveBulk() throws InterruptedException {
    long now = System.currentTimeMillis();
    Deque<PubsubMessage> inputs = new LinkedList<>(Arrays.asList(update("key1", "attr", new byte[] { 1, 2 }, now), delete("key2", "attr", now + 1000), deleteWildcard("key3", wildcard, now)));
    reader.setSupplier(() -> {
        if (inputs.isEmpty()) {
            LockSupport.park();
        }
        return inputs.pop();
    });
    List<StreamElement> elems = new ArrayList<>();
    AtomicBoolean cancelled = new AtomicBoolean();
    CountDownLatch latch = new CountDownLatch(3);
    AtomicReference<OffsetCommitter> commit = new AtomicReference<>();
    CommitLogObserver observer = new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            elems.add(ingest);
            commit.set(context);
            latch.countDown();
            return true;
        }

        @Override
        public void onCancelled() {
            cancelled.set(true);
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    };
    try (ObserveHandle handle = reader.observeBulk("dummy", observer)) {
        latch.await();
        commit.get().confirm();
    }
    assertEquals(3, elems.size());
    StreamElement elem = elems.get(0);
    assertEquals("key1", elem.getKey());
    assertEquals("attr", elem.getAttribute());
    assertFalse(elem.isDelete());
    assertFalse(elem.isDeleteWildcard());
    assertArrayEquals(new byte[] { 1, 2 }, elem.getValue());
    assertEquals(now, elem.getStamp());
    elem = elems.get(1);
    assertEquals("key2", elem.getKey());
    assertEquals("attr", elem.getAttribute());
    assertTrue(elem.isDelete());
    assertFalse(elem.isDeleteWildcard());
    assertEquals(now + 1000L, elem.getStamp());
    elem = elems.get(2);
    assertEquals("key3", elem.getKey());
    assertEquals(wildcard.toAttributePrefix() + "*", elem.getAttribute());
    assertTrue(elem.isDelete());
    assertTrue(elem.isDeleteWildcard());
    assertEquals(now, elem.getStamp());
    assertTrue(cancelled.get());
    assertEquals(Sets.newHashSet(0, 1, 2), reader.acked);
}

Example 19 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class KafkaLogReaderIT method testReadAndWriteSequentialId.

@Test(timeout = 30_000L)
public void testReadAndWriteSequentialId() throws InterruptedException {
    initializeTestWithUri("kafka://\"${broker}\"/topic", "[commit-log]");
    final EmbeddedKafkaBroker embeddedKafka = rule.getEmbeddedKafka();
    final int numPartitions = 3;
    embeddedKafka.addTopics(new NewTopic("topic", numPartitions, (short) 1));
    final CommitLogReader commitLogReader = Optionals.get(operator.getCommitLogReader(fooDescriptor));
    final TestLogObserver observer = new TestLogObserver();
    final ObserveHandle handle = commitLogReader.observe("test-reader", Position.OLDEST, observer);
    handle.waitUntilReady();
    final int numElements = 100;
    writeElements(numElements, true);
    while (observer.getNumReceivedElements() < numElements) {
        TimeUnit.MILLISECONDS.sleep(100);
    }
    observer.getReceivedElements().forEach(e -> assertTrue(e.hasSequentialId()));
    handle.close();
    Assert.assertEquals(numElements, observer.getNumReceivedElements());
    Assert.assertEquals(numElements, numCommittedElements(handle));
}

Example 20 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class PubSubReader method observeBulk.

private ObserveHandle observeBulk(@Nullable String name, Position position, boolean stopAtCurrent, long minWatermark, CommitLogObserver observer) {
    validateNotStopAtCurrent(stopAtCurrent);
    validatePosition(position);
    AtomicReference<List<AckReplyConsumer>> unconfirmed = new AtomicReference<>(new ArrayList<>());
    Object lock = new Object();
    Object listLock = new Object();
    AtomicLong globalOffset = new AtomicLong();
    String consumerName = asConsumerName(name);
    AtomicLong committedWatermark = new AtomicLong(minWatermark);
    PubSubPartition partition = new PubSubPartition(consumerName);
    return consume(consumerName, (e, w, c) -> {
        final long confirmUntil;
        synchronized (listLock) {
            List<AckReplyConsumer> list = unconfirmed.get();
            list.add(c);
            confirmUntil = list.size() + globalOffset.get();
        }
        OffsetCommitter committer = createBulkCommitter(listLock, confirmUntil, globalOffset, unconfirmed, w, committedWatermark);
        // ensure explicit synchronization here
        synchronized (lock) {
            try {
                Offset offset = new PubSubOffset(consumerName, w.getWatermark());
                if (!observer.onNext(e, asOnNextContext(committer, offset))) {
                    observer.onCompleted();
                    return false;
                }
                return true;
            } catch (Exception ex) {
                log.error("Error calling on next", ex);
                committer.fail(ex);
                throw new RuntimeException(ex);
            }
        }
    }, observer::onError, () -> observer.onRepartition(asRepartitionContext(Arrays.asList(partition))), () -> observer.onRepartition(asRepartitionContext(Arrays.asList(partition))), observer::onCancelled, committedWatermark);
}