Examples with ObserveHandle cz.o2.proxima.direct.commitlog.ObserveHandle used on opensource projects

Example 41 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class OffsetTrackingBatchLogReaderTest method test.

@Test
void test() throws InterruptedException {
    final Repository repository = Repository.ofTest(ConfigFactory.parseString(MODEL));
    final DirectDataOperator direct = repository.getOrCreateOperator(DirectDataOperator.class);
    final List<StreamElement> firstPartition = Arrays.asList(newData(repository, "first", Instant.now(), "value"), newData(repository, "first", Instant.now(), "value2"), newData(repository, "first", Instant.now(), "value3"));
    final List<StreamElement> secondPartition = Arrays.asList(newData(repository, "second", Instant.now(), "value"), newData(repository, "second", Instant.now(), "value2"), newData(repository, "second", Instant.now(), "value3"));
    final BatchLogReader reader = OffsetTrackingBatchLogReader.of(ListBatchReader.ofPartitioned(direct.getContext(), firstPartition, secondPartition));
    final CountDownLatch latch = new CountDownLatch(1);
    final ObserveHandle handle = reader.observe(reader.getPartitions(), Collections.singletonList(repository.getEntity("test").getAttribute("data")), new BatchLogObserver() {

        @Override
        public boolean onNext(StreamElement element, OnNextContext context) {
            final OffsetTrackingBatchLogReader.OffsetCommitter committer = (OffsetTrackingBatchLogReader.OffsetCommitter) context;
            committer.markOffsetAsConsumed();
            return !context.getOffset().isLast();
        }

        @Override
        public void onCancelled() {
            latch.countDown();
        }

        @Override
        public void onCompleted() {
            latch.countDown();
        }
    });
    latch.await();
    final OffsetTrackingBatchLogReader.OffsetTrackingObserveHandle otHandle = (OffsetTrackingBatchLogReader.OffsetTrackingObserveHandle) handle;
    Assertions.assertEquals(Arrays.asList(Offset.of(Partition.of(0), firstPartition.size() - 1, true), Offset.of(Partition.of(1), -1, false)), otHandle.getCurrentOffsets());
}

Example 42 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class PubSubReaderTest method testObserveBulkOffsetsWithWatermark.

@Test
public void testObserveBulkOffsetsWithWatermark() throws InterruptedException {
    long now = System.currentTimeMillis();
    Offset off = new PubSubOffset("dummy", now);
    reader.setSupplier(() -> {
        LockSupport.park();
        return null;
    });
    List<Offset> offsets = Collections.singletonList(off);
    CommitLogObserver observer = new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            context.confirm();
            return false;
        }

        @Override
        public void onCancelled() {
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    };
    try (ObserveHandle handle = reader.observeBulkOffsets(offsets, observer)) {
        handle.waitUntilReady();
        assertEquals(1, handle.getCommittedOffsets().size());
        assertEquals(now, handle.getCommittedOffsets().get(0).getWatermark());
    }
}

Example 43 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class PubSubReaderTest method testObserveError.

@Test(timeout = 10000)
public void testObserveError() throws InterruptedException {
    long now = System.currentTimeMillis();
    Deque<PubsubMessage> inputs = new LinkedList<>(Arrays.asList(update("key1", "attr", new byte[] { 1, 2 }, now), delete("key2", "attr", now + 1000), deleteWildcard("key3", wildcard, now)));
    reader.setSupplier(() -> {
        if (inputs.isEmpty()) {
            LockSupport.park();
        }
        return inputs.pop();
    });
    final AtomicBoolean cancelled = new AtomicBoolean(false);
    final CountDownLatch latch = new CountDownLatch(3);
    CommitLogObserver observer = new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            throw new RuntimeException("Fail");
        }

        @Override
        public void onCancelled() {
            cancelled.set(true);
        }

        @Override
        public boolean onError(Throwable error) {
            assertEquals("Fail", error.getCause().getMessage());
            latch.countDown();
            return true;
        }
    };
    try (final ObserveHandle handle = reader.observe("dummy", observer)) {
        latch.await();
        assertFalse(cancelled.get());
        assertTrue(reader.acked.isEmpty());
        assertFalse(reader.nacked.isEmpty());
    }
}

Example 44 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class BatchLogSourceFunction method observePartitions.

@Override
UnifiedObserveHandle<Offset> observePartitions(BatchLogReader reader, List<Partition> partitions, List<AttributeDescriptor<?>> attributeDescriptors, LogObserver<OutputT> observer) {
    final ObserveHandle batchReaderHandle = reader.observe(partitions, attributeDescriptors, wrapSourceObserver(observer));
    // We've wrapped BatchLogReader with the OffsetTrackingBatchLogReader, so we can safely cast its
    // handle, to get access to offsets.
    final OffsetTrackingBatchLogReader.OffsetTrackingObserveHandle offsetTrackingHandle = (OffsetTrackingBatchLogReader.OffsetTrackingObserveHandle) batchReaderHandle;
    return new UnifiedObserveHandle<Offset>() {

        @Override
        public List<Offset> getConsumedOffsets() {
            // Filter out finished partitions, as we don't need them for restoring the state.
            return offsetTrackingHandle.getCurrentOffsets().stream().filter(offset -> !offset.isLast()).collect(Collectors.toList());
        }

        @Override
        public void close() {
            offsetTrackingHandle.close();
        }
    };
}

Example 45 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class ListCommitLog method pushToObserverBulk.

private ObserveHandle pushToObserverBulk(@Nonnull String name, UnaryPredicate<Integer> allowOffsetPredicate, CommitLogObserver observer) {
    observer.onRepartition(asRepartitionContext(Collections.singletonList(PARTITION)));
    Consumer consumer = CONSUMERS.get(uuid).computeIfAbsent(name, k -> new Consumer(uuid, name, watermarkEstimator));
    ListObserveHandle handle = new ListObserveHandle(uuid, name);
    Set<Integer> consumerFedOffsets = Collections.synchronizedSet(new HashSet<>());
    pushTo((element, offset) -> {
        if (handle.isClosed()) {
            return false;
        }
        final CommitLogObserver.OffsetCommitter committer = (succ, exc) -> {
            if (exc != null) {
                observer.onError(exc);
            }
        };
        final boolean acceptable;
        OnNextContext context = null;
        synchronized (consumer) {
            acceptable = (externalizableOffsets || !consumer.getAckedOffsets().contains(offset) && !consumer.getInflightOffsets().contains(offset)) && allowOffsetPredicate.apply(offset);
            if (acceptable) {
                context = consumer.asOnNextContextBulk(committer, offset, consumerFedOffsets);
            }
        }
        if (acceptable) {
            return observer.onNext(element, context);
        }
        return true;
    }, externalizableOffsets ? () -> true : allMatchOffset(consumer::isAcked), observer::onCompleted, observer::onCancelled);
    return handle;
}