Examples with ObserveHandle cz.o2.proxima.direct.commitlog.ObserveHandle used on opensource projects

Example 26 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testObserveMultiplePartitions.

@Test
public void testObserveMultiplePartitions() throws InterruptedException {
    final int numPartitions = 3;
    final InMemStorage storage = new InMemStorage();
    final DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem:///test"), numPartitions));
    final CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    final AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    final int numElements = 1_000;
    final ConcurrentMap<Partition, Long> partitionHistogram = new ConcurrentHashMap<>();
    final CountDownLatch elementsReceived = new CountDownLatch(numElements);
    // Start observer.
    final ObserveHandle observeHandle = reader.observePartitions(reader.getPartitions(), new CommitLogObserver() {

        @Override
        public void onRepartition(OnRepartitionContext context) {
            assertEquals(numPartitions, context.partitions().size());
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            partitionHistogram.merge(context.getPartition(), 1L, Long::sum);
            context.confirm();
            elementsReceived.countDown();
            return elementsReceived.getCount() > 0;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    // Write data.
    for (int i = 0; i < numElements; i++) {
        writer.online().write(StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key_" + i, data.getName(), System.currentTimeMillis(), new byte[] { 1, 2, 3 }), CommitCallback.noop());
    }
    // Wait for all elements to be received.
    elementsReceived.await();
    assertEquals(3, partitionHistogram.size());
    assertEquals(3, observeHandle.getCurrentOffsets().size());
}

Example 27 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testObserveOffsets.

@Test(timeout = 10000)
public void testObserveOffsets() throws InterruptedException {
    InMemStorage storage = new InMemStorage();
    DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem:///inmemstoragetest")));
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    List<Byte> received = new ArrayList<>();
    CommitLogObserver observer = new CommitLogObserver() {

        @Override
        public void onRepartition(CommitLogObserver.OnRepartitionContext context) {
            assertEquals(1, context.partitions().size());
        }

        @Override
        public boolean onNext(StreamElement ingest, CommitLogObserver.OnNextContext context) {
            assertEquals(0, context.getPartition().getId());
            received.add(ingest.getValue()[0]);
            return false;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    };
    ObserveHandle handle = reader.observePartitions(reader.getPartitions(), observer);
    writer.online().write(StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key", data.getName(), System.currentTimeMillis(), new byte[] { 1 }), (succ, exc) -> {
    });
    List<Offset> offsets = handle.getCurrentOffsets();
    assertEquals(1, offsets.size());
    assertTrue(offsets.get(0).getWatermark() > 0);
    assertEquals(Collections.singletonList((byte) 1), received);
    handle.close();
    handle = reader.observeBulkOffsets(offsets, observer);
    handle.waitUntilReady();
    offsets = handle.getCurrentOffsets();
    assertEquals(1, offsets.size());
    assertTrue("Expected positive watermark, got " + offsets.get(0).getWatermark(), offsets.get(0).getWatermark() > 0);
    writer.online().write(StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key", data.getName(), System.currentTimeMillis(), new byte[] { 2 }), (succ, exc) -> {
    });
    assertEquals(Arrays.asList((byte) 1, (byte) 1), received);
    assertEquals(0, ((ConsumedOffset) handle.getCurrentOffsets().get(0)).getConsumedKeyAttr().size());
    handle.close();
}

Example 28 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class ListBatchReader method observe.

@Override
public ObserveHandle observe(List<Partition> partitions, List<AttributeDescriptor<?>> attributes, BatchLogObserver observer) {
    Preconditions.checkArgument(partitions != null);
    Preconditions.checkArgument(attributes != null);
    Preconditions.checkArgument(observer != null);
    Set<AttributeDescriptor<?>> attrSet = new HashSet<>(attributes);
    TerminationContext terminationContext = new TerminationContext(observer);
    context.getExecutorService().submit(() -> {
        terminationContext.setRunningThread();
        for (int i = 0; i < partitions.size() && !terminationContext.isCancelled(); i++) {
            long elementIndex = 0;
            final Partition partition = partitions.get(i);
            final Iterator<StreamElement> iterator = data.get(partition.getId()).iterator();
            while (!Thread.currentThread().isInterrupted() && !terminationContext.isCancelled() && iterator.hasNext()) {
                final StreamElement element = iterator.next();
                final Offset offset = Offset.of(partition, elementIndex++, !iterator.hasNext());
                if (attrSet.contains(element.getAttributeDescriptor()) && !observer.onNext(element, BatchLogObservers.withWatermark(partition, offset, Watermarks.MIN_WATERMARK))) {
                    terminationContext.cancel();
                    break;
                }
            }
        }
        terminationContext.finished();
    });
    return terminationContext.asObserveHandle();
}

Example 29 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class ListCommitLog method observe.

@Override
public ObserveHandle observe(@Nullable String name, Position position, CommitLogObserver observer) {
    String consumerName = name == null ? UUID.randomUUID().toString() : name;
    Consumer consumer = CONSUMERS.get(uuid).computeIfAbsent(consumerName, k -> new Consumer(uuid, consumerName, watermarkEstimator));
    ListObserveHandle handle = new ListObserveHandle(uuid, consumerName);
    pushTo((element, offset) -> {
        if (handle.isClosed()) {
            return false;
        }
        final CommitLogObserver.OffsetCommitter committer = (succ, exc) -> {
            if (exc != null) {
                observer.onError(exc);
            }
        };
        final boolean acceptable;
        OnNextContext context = null;
        synchronized (consumer) {
            acceptable = (externalizableOffsets || !consumer.getAckedOffsets().contains(offset) && !consumer.getInflightOffsets().contains(offset));
            if (acceptable) {
                context = consumer.asOnNextContext(committer, offset);
            }
        }
        if (acceptable) {
            return observer.onNext(element, context);
        }
        return true;
    }, externalizableOffsets ? () -> true : allMatchOffset(consumer::isAcked), observer::onCompleted, observer::onCancelled);
    return handle;
}

Example 30 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveWithCustomWatemarkEstimator.

@Test(timeout = 1000)
public void testObserveWithCustomWatemarkEstimator() throws InterruptedException {
    long now = System.currentTimeMillis() - 1000;
    int numElements = 10;
    CommitLogReader reader = ListCommitLog.of(data(numElements), new TestWatermarkEstimator(now), direct.getContext());
    CountDownLatch latch = new CountDownLatch(1);
    List<Long> watermarks = new ArrayList<>();
    ObserveHandle handle = reader.observe(null, new CommitLogObserver() {

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            context.confirm();
            watermarks.add(context.getWatermark());
            return true;
        }

        @Override
        public void onCompleted() {
            latch.countDown();
        }
    });
    latch.await();
    assertEquals(numElements, watermarks.size());
    List<Long> expected = IntStream.range(0, numElements).mapToObj(i -> now + i).collect(Collectors.toList());
    assertEquals(expected, watermarks);
}