Examples with WatermarkIdlePolicyFactory cz.o2.proxima.time.WatermarkIdlePolicyFactory used on opensource projects

Example 1 with WatermarkIdlePolicyFactory

use of cz.o2.proxima.time.WatermarkIdlePolicyFactory in project proxima-platform by O2-Czech-Republic.

the class KafkaLogReader method listener.

// create rebalance listener from consumer
private ConsumerRebalanceListener listener(String name, AtomicReference<KafkaConsumer<Object, Object>> kafka, ElementConsumer<Object, Object> consumer, Map<TopicPartition, Integer> emptyPollCount, Map<TopicPartition, Integer> topicPartitionToId, AtomicReference<PartitionedWatermarkEstimator> watermarkEstimator) {
    return new ConsumerRebalanceListener() {

        private final Set<TopicPartition> currentlyAssigned = new HashSet<>();

        @Override
        public void onPartitionsRevoked(Collection<TopicPartition> parts) {
            currentlyAssigned.removeAll(parts);
        }

        @Override
        public void onPartitionsAssigned(Collection<TopicPartition> parts) {
            currentlyAssigned.addAll(parts);
            log.info("Consumer {} has assigned partitions {}", name, currentlyAssigned);
            emptyPollCount.clear();
            topicPartitionToId.clear();
            AtomicInteger id = new AtomicInteger();
            currentlyAssigned.forEach(p -> {
                topicPartitionToId.put(p, id.getAndIncrement());
                emptyPollCount.put(p, 0);
            });
            if (currentlyAssigned.isEmpty()) {
                watermarkEstimator.set(createWatermarkEstimatorForEmptyParts());
            } else {
                watermarkEstimator.set(new MinimalPartitionWatermarkEstimator(currentlyAssigned.stream().collect(toMap(topicPartitionToId::get, item -> createWatermarkEstimator()))));
            }
            Optional.ofNullable(kafka.get()).ifPresent(c -> consumer.onAssign(c, name != null ? getCommittedTopicOffsets(currentlyAssigned, c) : getCurrentTopicOffsets(currentlyAssigned, c)));
        }

        List<TopicOffset> getCurrentTopicOffsets(Collection<TopicPartition> parts, KafkaConsumer<Object, Object> c) {
            return parts.stream().map(tp -> new TopicOffset(new PartitionWithTopic(tp.topic(), tp.partition()), c.position(tp), watermarkEstimator.get().getWatermark())).collect(Collectors.toList());
        }

        List<TopicOffset> getCommittedTopicOffsets(Collection<TopicPartition> parts, KafkaConsumer<Object, Object> c) {
            Map<TopicPartition, OffsetAndMetadata> committed = new HashMap<>(c.committed(new HashSet<>(parts)));
            for (TopicPartition tp : parts) {
                committed.putIfAbsent(tp, null);
            }
            return committed.entrySet().stream().map(entry -> {
                final long offset = entry.getValue() == null ? 0L : entry.getValue().offset();
                return new TopicOffset(new PartitionWithTopic(entry.getKey().topic(), entry.getKey().partition()), offset, watermarkEstimator.get().getWatermark());
            }).collect(Collectors.toList());
        }

        private WatermarkEstimator createWatermarkEstimator() {
            final WatermarkIdlePolicyFactory idlePolicyFactory = accessor.getWatermarkConfiguration().getWatermarkIdlePolicyFactory();
            final WatermarkEstimatorFactory estimatorFactory = accessor.getWatermarkConfiguration().getWatermarkEstimatorFactory();
            return estimatorFactory.create(cfg, idlePolicyFactory);
        }
    };
}

Example 2 with WatermarkIdlePolicyFactory

use of cz.o2.proxima.time.WatermarkIdlePolicyFactory in project proxima-platform by O2-Czech-Republic.

the class PubSubReader method createWatermarkEstimator.

WatermarkEstimator createWatermarkEstimator(long minWatermark) {
    final WatermarkIdlePolicyFactory idlePolicyFactory = watermarkConfiguration.getWatermarkIdlePolicyFactory();
    final WatermarkEstimatorFactory estimatorFactory = watermarkConfiguration.getWatermarkEstimatorFactory();
    final WatermarkEstimator estimator = estimatorFactory.create(cfg, idlePolicyFactory);
    estimator.setMinWatermark(minWatermark);
    return estimator;
}

Example 3 with WatermarkIdlePolicyFactory

use of cz.o2.proxima.time.WatermarkIdlePolicyFactory in project proxima-platform by O2-Czech-Republic.

the class NotProgressingWatermarkIdlePolicyTest method testFactory.

@Test
public void testFactory() {
    WatermarkIdlePolicyFactory factory = new NotProgressingWatermarkIdlePolicy.Factory();
    WatermarkIdlePolicy policy = factory.create(Collections.emptyMap());
    assertNotNull(policy);
    assertEquals(NotProgressingWatermarkIdlePolicy.class, policy.getClass());
}

Example 4 with WatermarkIdlePolicyFactory

use of cz.o2.proxima.time.WatermarkIdlePolicyFactory in project proxima-platform by O2-Czech-Republic.

the class SkewedProcessingTimeIdlePolicyTest method testFactory.

@Test
public void testFactory() {
    WatermarkIdlePolicyFactory factory = new SkewedProcessingTimeIdlePolicy.Factory();
    SkewedProcessingTimeIdlePolicy policy = (SkewedProcessingTimeIdlePolicy) factory.create(Collections.emptyMap());
    assertNotNull(policy);
    assertEquals(DEFAULT_TIMESTAMP_SKEW, policy.getTimestampSkew());
}

Example 5 with WatermarkIdlePolicyFactory

use of cz.o2.proxima.time.WatermarkIdlePolicyFactory in project proxima-platform by O2-Czech-Republic.

the class PubSubWatermarkConfigurationTest method testConfigureDefault.

@Test
public void testConfigureDefault() {
    // Check backward compatibility with legacy behaviour
    Map<String, Object> cfg = emptyMap();
    PubSubWatermarkConfiguration configuration = new PubSubWatermarkConfiguration(cfg, ESTIMATE_DURATION, TIMESTAMP_SKEW);
    WatermarkIdlePolicyFactory policyFactory = configuration.getWatermarkIdlePolicyFactory();
    UnboundedOutOfOrdernessWatermarkEstimator estimator = (UnboundedOutOfOrdernessWatermarkEstimator) configuration.getWatermarkEstimatorFactory().create(cfg, policyFactory);
    SkewedProcessingTimeIdlePolicy policy = (SkewedProcessingTimeIdlePolicy) policyFactory.create(cfg);
    assertNotNull(estimator);
    assertNotNull(policy);
    assertEquals(ESTIMATE_DURATION, estimator.getEstimateDurationMs());
    assertEquals(TIMESTAMP_SKEW, estimator.getAllowedTimestampSkew());
}