Examples with ObserveHandle cz.o2.proxima.direct.commitlog.ObserveHandle used on opensource projects

Example 36 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class CassandraDBAccessorTest method testBatchReaderCancelled.

@Test(timeout = 10000)
public void testBatchReaderCancelled() throws InterruptedException {
    TestDBAccessor accessor = new TestDBAccessor(entity, URI.create("cassandra://host:9042/table/?primary=data"), getCfg(TestCqlFactory.class, 2));
    CassandraLogReader reader = accessor.newBatchReader(direct.getContext());
    int numElements = 2;
    ResultSet result = mockResultSet(numElements);
    accessor.setRes(result);
    CountDownLatch latch = new CountDownLatch(1);
    AtomicReference<ObserveHandle> handle = new AtomicReference<>();
    handle.set(reader.observe(reader.getPartitions(), Collections.singletonList(attr), new BatchLogObserver() {

        @Override
        public boolean onNext(StreamElement element) {
            handle.get().close();
            return true;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }

        @Override
        public void onCancelled() {
            latch.countDown();
        }

        @Override
        public void onCompleted() {
            fail("onCompleted should have not been called");
        }
    }));
    latch.await();
}

Example 37 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class CassandraLogReader method observe.

@Override
public ObserveHandle observe(List<Partition> partitions, List<AttributeDescriptor<?>> attributes, BatchLogObserver observer) {
    TerminationContext terminationContext = new TerminationContext(observer);
    observeInternal(partitions, attributes, observer, terminationContext);
    return terminationContext.asObserveHandle();
}

Example 38 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class LocalKafkaCommitLogDescriptorTest method testBulkObserveWithException.

@Test(timeout = 10000)
public void testBulkObserveWithException() throws InterruptedException {
    Accessor accessor = kafka.createAccessor(direct, createTestFamily(entity, storageUri, partitionsCfg(3)));
    LocalKafkaWriter writer = accessor.newWriter();
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(context()));
    final AtomicInteger restarts = new AtomicInteger();
    final AtomicReference<Throwable> exc = new AtomicReference<>();
    final CountDownLatch latch = new CountDownLatch(2);
    final StreamElement update = StreamElement.upsert(entity, attr, UUID.randomUUID().toString(), "key", attr.getName(), System.currentTimeMillis(), new byte[] { 1, 2 });
    final ObserveHandle handle = reader.observeBulk("test", Position.NEWEST, new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            restarts.incrementAndGet();
            throw new RuntimeException("FAIL!");
        }

        @Override
        public void onCompleted() {
            fail("This should not be called");
        }

        @Override
        public boolean onError(Throwable error) {
            exc.set(error);
            latch.countDown();
            throw new RuntimeException(error);
        }
    });
    writer.write(update, (succ, e) -> {
        assertTrue(succ);
        latch.countDown();
    });
    latch.await();
    assertEquals("FAIL!", exc.get().getMessage());
    assertEquals(1, restarts.get());
    assertEquals(3, handle.getCommittedOffsets().size());
    List<Long> startedOffsets = handle.getCurrentOffsets().stream().map(o -> ((TopicOffset) o).getOffset()).filter(o -> o >= 0).collect(Collectors.toList());
    assertEquals(Collections.singletonList(0L), startedOffsets);
}

Example 39 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class PubSubReader method observe.

private ObserveHandle observe(@Nullable String name, Position position, long minWatermark, CommitLogObserver observer) {
    validatePosition(position);
    String consumerName = asConsumerName(name);
    AtomicLong committedWatermark = new AtomicLong(minWatermark);
    return consume(consumerName, (e, w, c) -> {
        OffsetCommitter committer = (succ, exc) -> {
            if (succ) {
                log.debug("Confirming message {} to PubSub", e);
                committedWatermark.set(w.getWatermark());
                c.ack();
            } else {
                if (exc != null) {
                    log.warn("Error during processing of {}", e, exc);
                } else {
                    log.info("Nacking message {} by request", e);
                }
                c.nack();
            }
        };
        try {
            long watermark = w.getWatermark();
            Offset offset = new PubSubOffset(consumerName, watermark);
            boolean ret = observer.onNext(e, asOnNextContext(committer, offset));
            if (!ret) {
                observer.onCompleted();
            }
            return ret;
        } catch (Exception ex) {
            log.error("Error calling onNext", ex);
            committer.fail(ex);
            throw new RuntimeException(ex);
        }
    }, observer::onError, null, () -> {
    }, observer::onCancelled, committedWatermark);
}

Example 40 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class PubSubReader method consume.

private ObserveHandle consume(String consumerName, PubSubConsumer consumer, UnaryFunction<Throwable, Boolean> errorHandler, @Nullable Runnable onInit, Runnable onRestart, Runnable onCancel, AtomicLong committedWatermark) {
    ProjectSubscriptionName subscription = ProjectSubscriptionName.of(project, consumerName);
    AtomicReference<Subscriber> subscriber = new AtomicReference<>();
    AtomicBoolean stopProcessing = new AtomicBoolean();
    AtomicReference<MessageReceiver> receiver = new AtomicReference<>();
    WatermarkEstimator watermarkEstimator = createWatermarkEstimator(committedWatermark.get());
    receiver.set(createMessageReceiver(subscription, subscriber, stopProcessing, consumer, watermarkEstimator, errorHandler, onRestart, receiver));
    subscriber.set(newSubscriber(subscription, receiver.get()));
    subscriber.get().startAsync();
    if (onInit != null) {
        executor().submit(() -> {
            subscriber.get().awaitRunning();
            if (onInit != null) {
                onInit.run();
            }
        });
    }
    return new ObserveHandle() {

        @Override
        public void close() {
            log.debug("Cancelling observer {}", consumerName);
            stopProcessing.set(true);
            Subscriber sub = stopAsync(subscriber);
            if (sub != null) {
                sub.awaitTerminated();
            }
            onCancel.run();
        }

        @Override
        public List<Offset> getCommittedOffsets() {
            return Collections.singletonList(new PubSubOffset(consumerName, committedWatermark.get()));
        }

        @Override
        public void resetOffsets(List<Offset> offsets) {
        // nop
        }

        @Override
        public List<Offset> getCurrentOffsets() {
            return getCommittedOffsets();
        }

        @Override
        public void waitUntilReady() {
            subscriber.get().awaitRunning();
        }
    };
}