Examples with Offset cz.o2.proxima.direct.batch.Offset used on opensource projects

Example 1 with Offset

use of cz.o2.proxima.direct.batch.Offset in project proxima-platform by O2-Czech-Republic.

the class RandomHBaseReader method listEntities.

@Override
public void listEntities(RandomOffset offset, int limit, Consumer<Pair<RandomOffset, String>> consumer) {
    ensureClient();
    Scan s = offset == null ? new Scan() : new Scan((((RawOffset) offset).getOffset() + '\00').getBytes(StandardCharsets.UTF_8));
    s.addFamily(family);
    s.setFilter(new KeyOnlyFilter());
    s.setCaching(keyCaching);
    try (ResultScanner scanner = client.getScanner(s)) {
        int taken = 0;
        while (limit <= 0 || taken++ < limit) {
            Result res = scanner.next();
            if (res != null) {
                String key = new String(res.getRow());
                consumer.accept(Pair.of(new RawOffset(key), key));
            } else {
                break;
            }
        }
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}

Example 2 with Offset

use of cz.o2.proxima.direct.batch.Offset in project proxima-platform by O2-Czech-Republic.

the class HadoopBatchLogReader method processPath.

private boolean processPath(BatchLogObserver observer, long watermark, HadoopPartition partition, HadoopPath path, TerminationContext terminationContext) {
    try {
        try (Reader reader = accessor.getFormat().openReader(path, accessor.getEntityDesc())) {
            long elementIndex = 0;
            final Iterator<StreamElement> iterator = reader.iterator();
            while (iterator.hasNext()) {
                final StreamElement element = iterator.next();
                final Offset offset = Offset.of(partition, elementIndex++, !iterator.hasNext());
                if (terminationContext.isCancelled() || !observer.onNext(element, BatchLogObservers.withWatermark(partition, offset, watermark))) {
                    return false;
                }
            }
        }
    } catch (IOException ex) {
        throw new RuntimeException("Failed to read file " + partition, ex);
    }
    return true;
}

Example 3 with Offset

use of cz.o2.proxima.direct.batch.Offset in project proxima-platform by O2-Czech-Republic.

the class KafkaLogReader method listener.

// create rebalance listener from consumer
private ConsumerRebalanceListener listener(String name, AtomicReference<KafkaConsumer<Object, Object>> kafka, ElementConsumer<Object, Object> consumer, Map<TopicPartition, Integer> emptyPollCount, Map<TopicPartition, Integer> topicPartitionToId, AtomicReference<PartitionedWatermarkEstimator> watermarkEstimator) {
    return new ConsumerRebalanceListener() {

        private final Set<TopicPartition> currentlyAssigned = new HashSet<>();

        @Override
        public void onPartitionsRevoked(Collection<TopicPartition> parts) {
            currentlyAssigned.removeAll(parts);
        }

        @Override
        public void onPartitionsAssigned(Collection<TopicPartition> parts) {
            currentlyAssigned.addAll(parts);
            log.info("Consumer {} has assigned partitions {}", name, currentlyAssigned);
            emptyPollCount.clear();
            topicPartitionToId.clear();
            AtomicInteger id = new AtomicInteger();
            currentlyAssigned.forEach(p -> {
                topicPartitionToId.put(p, id.getAndIncrement());
                emptyPollCount.put(p, 0);
            });
            if (currentlyAssigned.isEmpty()) {
                watermarkEstimator.set(createWatermarkEstimatorForEmptyParts());
            } else {
                watermarkEstimator.set(new MinimalPartitionWatermarkEstimator(currentlyAssigned.stream().collect(toMap(topicPartitionToId::get, item -> createWatermarkEstimator()))));
            }
            Optional.ofNullable(kafka.get()).ifPresent(c -> consumer.onAssign(c, name != null ? getCommittedTopicOffsets(currentlyAssigned, c) : getCurrentTopicOffsets(currentlyAssigned, c)));
        }

        List<TopicOffset> getCurrentTopicOffsets(Collection<TopicPartition> parts, KafkaConsumer<Object, Object> c) {
            return parts.stream().map(tp -> new TopicOffset(new PartitionWithTopic(tp.topic(), tp.partition()), c.position(tp), watermarkEstimator.get().getWatermark())).collect(Collectors.toList());
        }

        List<TopicOffset> getCommittedTopicOffsets(Collection<TopicPartition> parts, KafkaConsumer<Object, Object> c) {
            Map<TopicPartition, OffsetAndMetadata> committed = new HashMap<>(c.committed(new HashSet<>(parts)));
            for (TopicPartition tp : parts) {
                committed.putIfAbsent(tp, null);
            }
            return committed.entrySet().stream().map(entry -> {
                final long offset = entry.getValue() == null ? 0L : entry.getValue().offset();
                return new TopicOffset(new PartitionWithTopic(entry.getKey().topic(), entry.getKey().partition()), offset, watermarkEstimator.get().getWatermark());
            }).collect(Collectors.toList());
        }

        private WatermarkEstimator createWatermarkEstimator() {
            final WatermarkIdlePolicyFactory idlePolicyFactory = accessor.getWatermarkConfiguration().getWatermarkIdlePolicyFactory();
            final WatermarkEstimatorFactory estimatorFactory = accessor.getWatermarkConfiguration().getWatermarkEstimatorFactory();
            return estimatorFactory.create(cfg, idlePolicyFactory);
        }
    };
}

Example 4 with Offset

use of cz.o2.proxima.direct.batch.Offset in project proxima-platform by O2-Czech-Republic.

the class LocalKafkaCommitLogDescriptorTest method testBulkObserveOffsets2.

@Test(timeout = 10000)
public void testBulkObserveOffsets2() throws InterruptedException {
    final Accessor accessor = kafka.createAccessor(direct, createTestFamily(entity, storageUri, partitionsCfg(3)));
    final LocalKafkaWriter writer = accessor.newWriter();
    final CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(context()));
    final List<KafkaStreamElement> input = new ArrayList<>();
    final AtomicReference<CountDownLatch> latch = new AtomicReference<>(new CountDownLatch(3));
    final StreamElement update = StreamElement.upsert(entity, attr, UUID.randomUUID().toString(), "key", attr.getName(), System.currentTimeMillis(), new byte[] { 1, 2 });
    final CommitLogObserver observer = new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            input.add((KafkaStreamElement) ingest);
            latch.get().countDown();
            // terminate after reading first record
            return false;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    };
    final List<Offset> offsets;
    try (final ObserveHandle handle = reader.observeBulkPartitions(reader.getPartitions(), Position.NEWEST, observer)) {
        // write two elements
        for (int i = 0; i < 2; i++) {
            writer.write(update, (succ, e) -> {
                assertTrue(succ);
                latch.get().countDown();
            });
        }
        latch.get().await();
        latch.set(new CountDownLatch(1));
        offsets = handle.getCurrentOffsets();
    }
    // restart from old offset
    reader.observeBulkOffsets(Lists.newArrayList(offsets), observer);
    latch.get().await();
    assertEquals(2, input.size());
    assertEquals(0, input.get(0).getOffset());
    assertEquals(0, input.get(1).getOffset());
}

Example 5 with Offset

use of cz.o2.proxima.direct.batch.Offset in project proxima-platform by O2-Czech-Republic.

the class LocalKafkaCommitLogDescriptorTest method testObserveBulkCommitsCorrectly.

@Test(timeout = 10000)
public void testObserveBulkCommitsCorrectly() throws InterruptedException {
    Accessor accessor = kafka.createAccessor(direct, createTestFamily(entity, storageUri, cfg(Pair.of(KafkaAccessor.ASSIGNMENT_TIMEOUT_MS, 1L), Pair.of(LocalKafkaCommitLogDescriptor.CFG_NUM_PARTITIONS, 3))));
    LocalKafkaWriter writer = accessor.newWriter();
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(context()));
    long now = System.currentTimeMillis();
    for (int i = 0; i < 100; i++) {
        StreamElement update = StreamElement.upsert(entity, attr, UUID.randomUUID().toString(), "key-" + i, attr.getName(), now + 2000, new byte[] { 1, 2 });
        // then we write single element
        writer.write(update, (succ, e) -> {
        });
    }
    CountDownLatch latch = new CountDownLatch(1);
    ObserveHandle handle = reader.observeBulk("test", Position.OLDEST, true, new CommitLogObserver() {

        int processed = 0;

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            if (++processed == 100) {
                context.confirm();
            }
            return true;
        }

        @Override
        public void onCompleted() {
            latch.countDown();
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    latch.await();
    long offsetSum = handle.getCommittedOffsets().stream().mapToLong(o -> ((TopicOffset) o).getOffset()).sum();
    assertEquals(100, offsetSum);
    KafkaConsumer<Object, Object> consumer = ((LocalKafkaCommitLogDescriptor.LocalKafkaLogReader) reader).getConsumer();
    String topic = accessor.getTopic();
    assertEquals(100, consumer.committed(handle.getCommittedOffsets().stream().map(o -> new TopicPartition(topic, o.getPartition().getId())).collect(Collectors.toSet())).values().stream().mapToLong(OffsetAndMetadata::offset).sum());
}