Examples with OnNextContext cz.o2.proxima.direct.commitlog.CommitLogObserver.OnNextContext used on opensource projects

Example 21 with OnNextContext

use of cz.o2.proxima.direct.commitlog.CommitLogObserver.OnNextContext in project proxima-platform by O2-Czech-Republic.

the class PubSubReaderTest method testObserveBulk.

@Test(timeout = 10000)
public void testObserveBulk() throws InterruptedException {
    long now = System.currentTimeMillis();
    Deque<PubsubMessage> inputs = new LinkedList<>(Arrays.asList(update("key1", "attr", new byte[] { 1, 2 }, now), delete("key2", "attr", now + 1000), deleteWildcard("key3", wildcard, now)));
    reader.setSupplier(() -> {
        if (inputs.isEmpty()) {
            LockSupport.park();
        }
        return inputs.pop();
    });
    List<StreamElement> elems = new ArrayList<>();
    AtomicBoolean cancelled = new AtomicBoolean();
    CountDownLatch latch = new CountDownLatch(3);
    AtomicReference<OffsetCommitter> commit = new AtomicReference<>();
    CommitLogObserver observer = new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            elems.add(ingest);
            commit.set(context);
            latch.countDown();
            return true;
        }

        @Override
        public void onCancelled() {
            cancelled.set(true);
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    };
    try (ObserveHandle handle = reader.observeBulk("dummy", observer)) {
        latch.await();
        commit.get().confirm();
    }
    assertEquals(3, elems.size());
    StreamElement elem = elems.get(0);
    assertEquals("key1", elem.getKey());
    assertEquals("attr", elem.getAttribute());
    assertFalse(elem.isDelete());
    assertFalse(elem.isDeleteWildcard());
    assertArrayEquals(new byte[] { 1, 2 }, elem.getValue());
    assertEquals(now, elem.getStamp());
    elem = elems.get(1);
    assertEquals("key2", elem.getKey());
    assertEquals("attr", elem.getAttribute());
    assertTrue(elem.isDelete());
    assertFalse(elem.isDeleteWildcard());
    assertEquals(now + 1000L, elem.getStamp());
    elem = elems.get(2);
    assertEquals("key3", elem.getKey());
    assertEquals(wildcard.toAttributePrefix() + "*", elem.getAttribute());
    assertTrue(elem.isDelete());
    assertTrue(elem.isDeleteWildcard());
    assertEquals(now, elem.getStamp());
    assertTrue(cancelled.get());
    assertEquals(Sets.newHashSet(0, 1, 2), reader.acked);
}

Example 22 with OnNextContext

use of cz.o2.proxima.direct.commitlog.CommitLogObserver.OnNextContext in project proxima-platform by O2-Czech-Republic.

the class ModelTest method testReadWrite.

@Test
public void testReadWrite() throws InterruptedException {
    Regular<BaseEvent> desc = model.getEvent().getDataDescriptor();
    Optional<OnlineAttributeWriter> writer = model.directOperator().getWriter(desc);
    writer.get().write(desc.upsert(UUID.randomUUID().toString(), System.currentTimeMillis(), BaseEvent.newBuilder().setUserName("user").build()), (succ, exc) -> {
    });
    Optional<CommitLogReader> reader = model.directOperator().getCommitLogReader(desc);
    List<String> users = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    reader.get().observeBulk("consumer", Position.OLDEST, true, new CommitLogObserver() {

        @Override
        public boolean onError(Throwable error) {
            return false;
        }

        @Override
        public void onCompleted() {
            latch.countDown();
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            Optional<BaseEvent> baseEvent = desc.valueOf(ingest);
            users.add(baseEvent.get().getUserName());
            return true;
        }
    });
    latch.await();
    assertEquals("user", users.get(0));
}

Example 23 with OnNextContext

use of cz.o2.proxima.direct.commitlog.CommitLogObserver.OnNextContext in project proxima-platform by O2-Czech-Republic.

the class BatchLogReaderTest method testObserveWithContext.

@Test(timeout = 5000)
public void testObserveWithContext() throws InterruptedException {
    write("gw", new byte[] { 1 });
    BatchLogReader reader = getBatchReader();
    BlockingQueue<StreamElement> read = new SynchronousQueue<>();
    reader.observe(reader.getPartitions(), Collections.singletonList(attr), new BatchLogObserver() {

        @Override
        public boolean onNext(StreamElement element, OnNextContext context) {
            assertEquals(0, context.getPartition().getId());
            assertEquals(Long.MIN_VALUE, context.getWatermark());
            ExceptionUtils.unchecked(() -> read.put(element));
            return false;
        }
    });
    StreamElement element = read.take();
    assertEquals("gw", element.getKey());
}

Example 24 with OnNextContext

use of cz.o2.proxima.direct.commitlog.CommitLogObserver.OnNextContext in project proxima-platform by O2-Czech-Republic.

the class BatchLogReaderTest method testObserveReadOffset.

@Test
public void testObserveReadOffset() throws InterruptedException {
    final List<StreamElement> firstPartition = newPartition("first_", 10);
    final List<StreamElement> secondPartition = newPartition("second_", 20);
    final List<StreamElement> thirdPartition = newPartition("third_", 30);
    final ListBatchReader reader = ListBatchReader.ofPartitioned(direct.getContext(), Arrays.asList(firstPartition, secondPartition, thirdPartition));
    final ConcurrentMap<Partition, Offset> lastOffsets = new ConcurrentHashMap<>();
    final CountDownLatch doneConsuming = new CountDownLatch(1);
    reader.observe(Arrays.asList(Partition.of(0), Partition.of(1), Partition.of(2)), Collections.singletonList(attr), new BatchLogObserver() {

        @Override
        public boolean onNext(StreamElement element, OnNextContext context) {
            lastOffsets.merge(context.getPartition(), context.getOffset(), (oldValue, newValue) -> {
                assertTrue(oldValue.getElementIndex() < newValue.getElementIndex());
                return newValue;
            });
            return true;
        }

        @Override
        public void onCompleted() {
            doneConsuming.countDown();
        }
    });
    doneConsuming.await();
    assertEquals(Offset.of(Partition.of(0), 9, true), lastOffsets.get(Partition.of(0)));
    assertEquals(Offset.of(Partition.of(1), 19, true), lastOffsets.get(Partition.of(1)));
    assertEquals(Offset.of(Partition.of(2), 29, true), lastOffsets.get(Partition.of(2)));
}

Example 25 with OnNextContext

use of cz.o2.proxima.direct.commitlog.CommitLogObserver.OnNextContext in project proxima-platform by O2-Czech-Republic.

the class CommitLogReaderTest method testObserveWithRetry.

@Test(timeout = 10000)
public void testObserveWithRetry() throws InterruptedException {
    List<StreamElement> received = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    AtomicInteger count = new AtomicInteger();
    final CommitLogObserver observer = CommitLogObservers.withNumRetriedExceptions("test", 2, new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext confirm) {
            if (count.incrementAndGet() == 0) {
                throw new RuntimeException("fail");
            }
            received.add(ingest);
            latch.countDown();
            return true;
        }

        @Override
        public boolean onError(Throwable error) {
            return false;
        }
    });
    reader.observe("test", observer);
    writer.online().write(StreamElement.upsert(entity, attr, UUID.randomUUID().toString(), "key", attr.getName(), System.currentTimeMillis(), new byte[] { 1, 2 }), (succ, exc) -> {
    });
    latch.await();
    assertEquals(1, received.size());
}