Examples with WatermarkEstimator cz.o2.proxima.time.WatermarkEstimator used on opensource projects

Example 1 with WatermarkEstimator

use of cz.o2.proxima.time.WatermarkEstimator in project proxima-platform by O2-Czech-Republic.

the class KafkaLogReader method listener.

// create rebalance listener from consumer
private ConsumerRebalanceListener listener(String name, AtomicReference<KafkaConsumer<Object, Object>> kafka, ElementConsumer<Object, Object> consumer, Map<TopicPartition, Integer> emptyPollCount, Map<TopicPartition, Integer> topicPartitionToId, AtomicReference<PartitionedWatermarkEstimator> watermarkEstimator) {
    return new ConsumerRebalanceListener() {

        private final Set<TopicPartition> currentlyAssigned = new HashSet<>();

        @Override
        public void onPartitionsRevoked(Collection<TopicPartition> parts) {
            currentlyAssigned.removeAll(parts);
        }

        @Override
        public void onPartitionsAssigned(Collection<TopicPartition> parts) {
            currentlyAssigned.addAll(parts);
            log.info("Consumer {} has assigned partitions {}", name, currentlyAssigned);
            emptyPollCount.clear();
            topicPartitionToId.clear();
            AtomicInteger id = new AtomicInteger();
            currentlyAssigned.forEach(p -> {
                topicPartitionToId.put(p, id.getAndIncrement());
                emptyPollCount.put(p, 0);
            });
            if (currentlyAssigned.isEmpty()) {
                watermarkEstimator.set(createWatermarkEstimatorForEmptyParts());
            } else {
                watermarkEstimator.set(new MinimalPartitionWatermarkEstimator(currentlyAssigned.stream().collect(toMap(topicPartitionToId::get, item -> createWatermarkEstimator()))));
            }
            Optional.ofNullable(kafka.get()).ifPresent(c -> consumer.onAssign(c, name != null ? getCommittedTopicOffsets(currentlyAssigned, c) : getCurrentTopicOffsets(currentlyAssigned, c)));
        }

        List<TopicOffset> getCurrentTopicOffsets(Collection<TopicPartition> parts, KafkaConsumer<Object, Object> c) {
            return parts.stream().map(tp -> new TopicOffset(new PartitionWithTopic(tp.topic(), tp.partition()), c.position(tp), watermarkEstimator.get().getWatermark())).collect(Collectors.toList());
        }

        List<TopicOffset> getCommittedTopicOffsets(Collection<TopicPartition> parts, KafkaConsumer<Object, Object> c) {
            Map<TopicPartition, OffsetAndMetadata> committed = new HashMap<>(c.committed(new HashSet<>(parts)));
            for (TopicPartition tp : parts) {
                committed.putIfAbsent(tp, null);
            }
            return committed.entrySet().stream().map(entry -> {
                final long offset = entry.getValue() == null ? 0L : entry.getValue().offset();
                return new TopicOffset(new PartitionWithTopic(entry.getKey().topic(), entry.getKey().partition()), offset, watermarkEstimator.get().getWatermark());
            }).collect(Collectors.toList());
        }

        private WatermarkEstimator createWatermarkEstimator() {
            final WatermarkIdlePolicyFactory idlePolicyFactory = accessor.getWatermarkConfiguration().getWatermarkIdlePolicyFactory();
            final WatermarkEstimatorFactory estimatorFactory = accessor.getWatermarkConfiguration().getWatermarkEstimatorFactory();
            return estimatorFactory.create(cfg, idlePolicyFactory);
        }
    };
}

Example 2 with WatermarkEstimator

use of cz.o2.proxima.time.WatermarkEstimator in project proxima-platform by O2-Czech-Republic.

the class PubSubReader method createWatermarkEstimator.

WatermarkEstimator createWatermarkEstimator(long minWatermark) {
    final WatermarkIdlePolicyFactory idlePolicyFactory = watermarkConfiguration.getWatermarkIdlePolicyFactory();
    final WatermarkEstimatorFactory estimatorFactory = watermarkConfiguration.getWatermarkEstimatorFactory();
    final WatermarkEstimator estimator = estimatorFactory.create(cfg, idlePolicyFactory);
    estimator.setMinWatermark(minWatermark);
    return estimator;
}

Example 3 with WatermarkEstimator

use of cz.o2.proxima.time.WatermarkEstimator in project proxima-platform by O2-Czech-Republic.

the class CommitLogReadTest method testWatermarkEstimator.

@Test(timeout = 60000)
public void testWatermarkEstimator() {
    int numElements = 1000;
    WatermarkEstimator estimator = new TestWatermarkEstimator(numElements);
    List<StreamElement> input = createInput(numElements);
    ListCommitLog commitLog = ListCommitLog.of(input, estimator, direct.getContext());
    testReadingFromCommitLogMany(numElements, commitLog);
}

Example 4 with WatermarkEstimator

use of cz.o2.proxima.time.WatermarkEstimator in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testObserveWithEndOfTime.

@Test
public void testObserveWithEndOfTime() throws InterruptedException {
    URI uri = URI.create("inmem:///inmemstoragetest");
    InMemStorage storage = new InMemStorage();
    InMemStorage.setWatermarkEstimatorFactory(uri, (stamp, name, offset) -> new WatermarkEstimator() {

        {
            Preconditions.checkArgument(offset != null);
        }

        @Override
        public long getWatermark() {
            return Watermarks.MAX_WATERMARK - InMemStorage.getBoundedOutOfOrderness();
        }

        @Override
        public void update(StreamElement element) {
        }

        @Override
        public void setMinWatermark(long minWatermark) {
        }
    });
    DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(uri));
    CommitLogReader reader = accessor.getCommitLogReader(direct.getContext()).orElseThrow(() -> new IllegalStateException("Missing commit log reader"));
    CountDownLatch completed = new CountDownLatch(1);
    reader.observe("observer", new CommitLogObserver() {

        @Override
        public void onCompleted() {
            completed.countDown();
        }

        @Override
        public boolean onError(Throwable error) {
            return false;
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            return false;
        }
    });
    assertTrue(completed.await(1, TimeUnit.SECONDS));
}

Example 5 with WatermarkEstimator

use of cz.o2.proxima.time.WatermarkEstimator in project proxima-platform by O2-Czech-Republic.

the class ListCommitLog method observe.

@Override
public ObserveHandle observe(@Nullable String name, Position position, CommitLogObserver observer) {
    String consumerName = name == null ? UUID.randomUUID().toString() : name;
    Consumer consumer = CONSUMERS.get(uuid).computeIfAbsent(consumerName, k -> new Consumer(uuid, consumerName, watermarkEstimator));
    ListObserveHandle handle = new ListObserveHandle(uuid, consumerName);
    pushTo((element, offset) -> {
        if (handle.isClosed()) {
            return false;
        }
        final CommitLogObserver.OffsetCommitter committer = (succ, exc) -> {
            if (exc != null) {
                observer.onError(exc);
            }
        };
        final boolean acceptable;
        OnNextContext context = null;
        synchronized (consumer) {
            acceptable = (externalizableOffsets || !consumer.getAckedOffsets().contains(offset) && !consumer.getInflightOffsets().contains(offset));
            if (acceptable) {
                context = consumer.asOnNextContext(committer, offset);
            }
        }
        if (acceptable) {
            return observer.onNext(element, context);
        }
        return true;
    }, externalizableOffsets ? () -> true : allMatchOffset(consumer::isAcked), observer::onCompleted, observer::onCancelled);
    return handle;
}