Examples with Item cz.o2.proxima.beam.tools.proto.service.Collect.Item used on opensource projects

Example 1 with Item

use of cz.o2.proxima.beam.tools.proto.service.Collect.Item in project proxima-platform by O2-Czech-Republic.

the class KafkaLogReader method listener.

// create rebalance listener from consumer
private ConsumerRebalanceListener listener(String name, AtomicReference<KafkaConsumer<Object, Object>> kafka, ElementConsumer<Object, Object> consumer, Map<TopicPartition, Integer> emptyPollCount, Map<TopicPartition, Integer> topicPartitionToId, AtomicReference<PartitionedWatermarkEstimator> watermarkEstimator) {
    return new ConsumerRebalanceListener() {

        private final Set<TopicPartition> currentlyAssigned = new HashSet<>();

        @Override
        public void onPartitionsRevoked(Collection<TopicPartition> parts) {
            currentlyAssigned.removeAll(parts);
        }

        @Override
        public void onPartitionsAssigned(Collection<TopicPartition> parts) {
            currentlyAssigned.addAll(parts);
            log.info("Consumer {} has assigned partitions {}", name, currentlyAssigned);
            emptyPollCount.clear();
            topicPartitionToId.clear();
            AtomicInteger id = new AtomicInteger();
            currentlyAssigned.forEach(p -> {
                topicPartitionToId.put(p, id.getAndIncrement());
                emptyPollCount.put(p, 0);
            });
            if (currentlyAssigned.isEmpty()) {
                watermarkEstimator.set(createWatermarkEstimatorForEmptyParts());
            } else {
                watermarkEstimator.set(new MinimalPartitionWatermarkEstimator(currentlyAssigned.stream().collect(toMap(topicPartitionToId::get, item -> createWatermarkEstimator()))));
            }
            Optional.ofNullable(kafka.get()).ifPresent(c -> consumer.onAssign(c, name != null ? getCommittedTopicOffsets(currentlyAssigned, c) : getCurrentTopicOffsets(currentlyAssigned, c)));
        }

        List<TopicOffset> getCurrentTopicOffsets(Collection<TopicPartition> parts, KafkaConsumer<Object, Object> c) {
            return parts.stream().map(tp -> new TopicOffset(new PartitionWithTopic(tp.topic(), tp.partition()), c.position(tp), watermarkEstimator.get().getWatermark())).collect(Collectors.toList());
        }

        List<TopicOffset> getCommittedTopicOffsets(Collection<TopicPartition> parts, KafkaConsumer<Object, Object> c) {
            Map<TopicPartition, OffsetAndMetadata> committed = new HashMap<>(c.committed(new HashSet<>(parts)));
            for (TopicPartition tp : parts) {
                committed.putIfAbsent(tp, null);
            }
            return committed.entrySet().stream().map(entry -> {
                final long offset = entry.getValue() == null ? 0L : entry.getValue().offset();
                return new TopicOffset(new PartitionWithTopic(entry.getKey().topic(), entry.getKey().partition()), offset, watermarkEstimator.get().getWatermark());
            }).collect(Collectors.toList());
        }

        private WatermarkEstimator createWatermarkEstimator() {
            final WatermarkIdlePolicyFactory idlePolicyFactory = accessor.getWatermarkConfiguration().getWatermarkIdlePolicyFactory();
            final WatermarkEstimatorFactory estimatorFactory = accessor.getWatermarkConfiguration().getWatermarkEstimatorFactory();
            return estimatorFactory.create(cfg, idlePolicyFactory);
        }
    };
}

Example 2 with Item

use of cz.o2.proxima.beam.tools.proto.service.Collect.Item in project proxima-platform by O2-Czech-Republic.

the class InMemStorage method createAccessor.

@Override
public DataAccessor createAccessor(DirectDataOperator op, AttributeFamilyDescriptor familyDescriptor) {
    final EntityDescriptor entity = familyDescriptor.getEntity();
    final URI uri = familyDescriptor.getStorageUri();
    final Map<String, Object> cfg = familyDescriptor.getCfg();
    log.info("Creating accessor {} for URI {}", getClass(), uri);
    holder().observers.computeIfAbsent(uri, k -> Collections.synchronizedNavigableMap(new TreeMap<>()));
    final int numPartitions = Optional.ofNullable(cfg.get(NUM_PARTITIONS)).map(v -> Integer.parseInt(v.toString())).orElse(1);
    final Partitioner partitioner = Optional.ofNullable(cfg.get(ConfigConstants.PARTITIONER)).map(name -> Classpath.newInstance(name.toString(), Partitioner.class)).orElseGet(KeyAttributePartitioner::new);
    final Repository opRepo = op.getRepository();
    final RepositoryFactory repositoryFactory = opRepo.asFactory();
    final OnlineAttributeWriter.Factory<?> writerFactory = new Writer(entity, uri, numPartitions, partitioner).asFactory();
    final CommitLogReader.Factory<?> commitLogReaderFactory = new InMemCommitLogReader(entity, uri, op.getContext().getExecutorFactory(), partitioner, numPartitions).asFactory();
    final RandomAccessReader.Factory<Reader> randomAccessReaderFactory;
    final BatchLogReader.Factory<Reader> batchLogReaderFactory;
    final CachedView.Factory cachedViewFactory;
    if (numPartitions > 1) {
        randomAccessReaderFactory = null;
        batchLogReaderFactory = null;
        cachedViewFactory = null;
    } else {
        final ReaderFactory readerFactory = new Reader(entity, uri, op.getContext().getExecutorFactory()).asFactory();
        randomAccessReaderFactory = readerFactory;
        batchLogReaderFactory = readerFactory;
        cachedViewFactory = new LocalCachedPartitionedView(entity, commitLogReaderFactory.apply(opRepo), writerFactory.apply(opRepo)).asFactory();
    }
    return new DataAccessor() {

        private static final long serialVersionUID = 1L;

        @Nullable
        private transient Repository repo = opRepo;

        @Override
        public URI getUri() {
            return uri;
        }

        @Override
        public Optional<AttributeWriterBase> getWriter(Context context) {
            Objects.requireNonNull(context);
            return Optional.of(writerFactory.apply(repo()));
        }

        @Override
        public Optional<CommitLogReader> getCommitLogReader(Context context) {
            Objects.requireNonNull(context);
            return Optional.of(commitLogReaderFactory.apply(repo()));
        }

        @Override
        public Optional<RandomAccessReader> getRandomAccessReader(Context context) {
            Objects.requireNonNull(context);
            return Optional.ofNullable(randomAccessReaderFactory).map(item -> item.apply(repo()));
        }

        @Override
        public Optional<CachedView> getCachedView(Context context) {
            Objects.requireNonNull(context);
            return Optional.ofNullable(cachedViewFactory).map(item -> item.apply(repo()));
        }

        @Override
        public Optional<BatchLogReader> getBatchLogReader(Context context) {
            Objects.requireNonNull(context);
            return Optional.ofNullable(batchLogReaderFactory).map(item -> item.apply(repo()));
        }

        private Repository repo() {
            if (this.repo == null) {
                this.repo = repositoryFactory.apply();
            }
            return this.repo;
        }
    };
}

Example 3 with Item

use of cz.o2.proxima.beam.tools.proto.service.Collect.Item in project proxima-platform by O2-Czech-Republic.

the class ThreadPooledObserverTest method testParallelObserveWithRepartition.

@Test(timeout = 20_000)
public void testParallelObserveWithRepartition() throws InterruptedException {
    List<Integer> elements = Collections.synchronizedList(new ArrayList<>());
    Set<Integer> confirmed = Collections.synchronizedSet(new TreeSet<>());
    ExecutorService executor = Executors.newCachedThreadPool();
    int parallelism = 20;
    int numElements = 50_000;
    ThreadPooledObserver observer = new ThreadPooledObserver(executor, (elem, context) -> {
        elements.add(Integer.valueOf(elem.getKey().substring(3)));
        context.confirm();
        return true;
    }, parallelism);
    long now = System.currentTimeMillis();
    CountDownLatch confirmedLatch = new CountDownLatch(numElements);
    for (int i = 0; i < numElements; i++) {
        int item = i;
        observer.onNext(device.upsert("key" + i, String.valueOf(i), now + i, new byte[] {}), asOnNextContext((succ, exc) -> {
            assertTrue(succ);
            assertNull(exc);
            confirmed.add(item);
            confirmedLatch.countDown();
        }, null));
        if (item % 83 == 0) {
            observer.onRepartition(Collections::emptyList);
        }
    }
    confirmedLatch.await();
    observer.onCompleted();
    assertEquals(numElements, elements.stream().distinct().count());
    assertEquals(numElements, confirmed.size());
}

Example 4 with Item

use of cz.o2.proxima.beam.tools.proto.service.Collect.Item in project proxima-platform by O2-Czech-Republic.

the class RemoteConsumer method observer.

StreamObserver<Item> observer() {
    if (channel == null) {
        channel = ManagedChannelBuilder.forAddress(hostname, port).usePlaintext().build();
        CountDownLatch connected = new CountDownLatch(1);
        channel.notifyWhenStateChanged(ConnectivityState.READY, connected::countDown);
        ExceptionUtils.ignoringInterrupted(() -> {
            if (!connected.await(1, TimeUnit.SECONDS)) {
                log.warn("Timeout waiting for channel to become connected. Skipping.");
            }
        });
    }
    if (stub == null) {
        stub = CollectServiceGrpc.newStub(channel);
    }
    if (observer == null) {
        terminateFuture = new CompletableFuture<>();
        CountDownLatch initLatch = new CountDownLatch(1);
        observer = stub.collect(new StreamObserver<Response>() {

            @Override
            public void onNext(Response response) {
                if (response.getStatus() == CONTINUE) {
                    initLatch.countDown();
                } else if (response.getStatus() == OK) {
                    terminateFuture.complete(null);
                }
            }

            @Override
            public void onError(Throwable throwable) {
                terminateFuture.completeExceptionally(throwable);
                observer = null;
            }

            @Override
            public void onCompleted() {
            // nop
            }
        });
        initLatch.countDown();
    }
    return observer;
}

Example 5 with Item

use of cz.o2.proxima.beam.tools.proto.service.Collect.Item in project proxima-platform by O2-Czech-Republic.

the class RemoteConsumer method accept.

@Override
public void accept(T what) {
    try {
        Item item = Item.newBuilder().setSerialized(ByteString.readFrom(serialize(what))).build();
        observer().onNext(item);
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}