Examples with ObserveHandle cz.o2.proxima.direct.commitlog.ObserveHandle used on opensource projects

Example 21 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class OffsetTrackingBatchLogReader method observe.

@Override
public ObserveHandle observe(List<Partition> partitions, List<AttributeDescriptor<?>> attributes, BatchLogObserver observer) {
    final OffsetTrackingBatchLogObserver wrappedObserver = new OffsetTrackingBatchLogObserver(observer);
    final ObserveHandle handle = delegate.observe(partitions, attributes, wrappedObserver);
    return new OffsetTrackingObserveHandle() {

        @Override
        public List<Offset> getCurrentOffsets() {
            final Map<Partition, Offset> result = new HashMap<>();
            partitions.forEach(p -> result.put(p, Offset.of(p, -1, false)));
            wrappedObserver.getConsumedOffsets().forEach((p, o) -> result.merge(p, o, OffsetTrackingBatchLogReader::mergeOffsets));
            return new ArrayList<>(result.values());
        }

        @Override
        public void close() {
            handle.close();
        }
    };
}

Example 22 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class BlobLogReaderTest method testObservePartitionsCancelled.

@Test
public void testObservePartitionsCancelled() throws InterruptedException {
    List<Pair<Long, Long>> stamps = Lists.newArrayList(Pair.of(1234566000000L, 1234566000000L + 3_600_000L), Pair.of(1234566000000L + 3_600_000L, (1234566000000L + 2 * 3_600_000L)));
    writePartitions(stamps.stream().map(p -> (p.getSecond() + p.getFirst()) / 2).collect(Collectors.toList()));
    BlobReader reader = accessor.new BlobReader(context);
    CountDownLatch latch = new CountDownLatch(1);
    AtomicReference<ObserveHandle> handle = new AtomicReference<>();
    handle.set(reader.observe(reader.getPartitions(), Collections.singletonList(status), new BatchLogObserver() {

        @Override
        public boolean onNext(StreamElement element) {
            handle.get().close();
            return true;
        }

        @Override
        public void onCancelled() {
            latch.countDown();
        }

        @Override
        public void onCompleted() {
            fail("onCompleted should not have been called");
        }
    }));
    latch.await();
}

Example 23 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class CassandraDBAccessorTest method testBatchReader.

@Test(timeout = 10000)
public void testBatchReader() throws InterruptedException {
    TestDBAccessor accessor = new TestDBAccessor(entity, URI.create("cassandra://host:9042/table/?primary=data"), getCfg(TestCqlFactory.class, 2));
    CassandraLogReader reader = accessor.newBatchReader(direct.getContext());
    int numElements = 100;
    ResultSet result = mockResultSet(numElements);
    accessor.setRes(result);
    AtomicInteger numConsumed = new AtomicInteger();
    CountDownLatch latch = new CountDownLatch(1);
    try (ObserveHandle handle = reader.observe(reader.getPartitions(), Collections.singletonList(attr), new BatchLogObserver() {

        @Override
        public boolean onNext(StreamElement element) {
            numConsumed.incrementAndGet();
            return true;
        }

        @Override
        public boolean onError(Throwable error) {
            while (latch.getCount() > 0) {
                latch.countDown();
            }
            throw new RuntimeException(error);
        }

        @Override
        public void onCompleted() {
            latch.countDown();
        }
    })) {
        latch.await();
        assertEquals(numElements, numConsumed.get());
        List<Statement> executed = accessor.getExecuted();
        assertEquals(2, executed.size());
    }
    assertTrue("Expected empty CLUSTER_MAP, got " + CassandraDBAccessor.getCLUSTER_MAP(), CassandraDBAccessor.getCLUSTER_MAP().isEmpty());
}

Example 24 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testObservePartitions.

@Test(timeout = 10000)
public void testObservePartitions() throws InterruptedException {
    InMemStorage storage = new InMemStorage();
    DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem:///inmemstoragetest")));
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    AtomicReference<CountDownLatch> latch = new AtomicReference<>();
    ObserveHandle handle = reader.observePartitions(reader.getPartitions(), new CommitLogObserver() {

        @Override
        public void onRepartition(OnRepartitionContext context) {
            assertEquals(1, context.partitions().size());
            latch.set(new CountDownLatch(1));
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            assertEquals(0, context.getPartition().getId());
            assertEquals("key", ingest.getKey());
            context.confirm();
            latch.get().countDown();
            return false;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    assertTrue(ObserveHandleUtils.isAtHead(handle, reader));
    writer.online().write(StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key", data.getName(), System.currentTimeMillis(), new byte[] { 1, 2, 3 }), (succ, exc) -> {
    });
    latch.get().await();
}

Example 25 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testObserveSinglePartitionOutOfMultiplePartitions.

@Test
public void testObserveSinglePartitionOutOfMultiplePartitions() throws InterruptedException {
    final int numPartitions = 3;
    final InMemStorage storage = new InMemStorage();
    final DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem:///test"), numPartitions));
    final CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    final AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    final int numElements = 999;
    final ConcurrentMap<Partition, Long> partitionHistogram = new ConcurrentHashMap<>();
    // Elements are uniformly distributed between partitions.
    final CountDownLatch elementsReceived = new CountDownLatch(numElements / numPartitions);
    // Start observer.
    final List<Partition> consumedPartitions = reader.getPartitions().subList(0, 1);
    final ObserveHandle observeHandle = reader.observePartitions(reader.getPartitions().subList(0, 1), new CommitLogObserver() {

        @Override
        public void onRepartition(OnRepartitionContext context) {
            assertEquals(numPartitions, context.partitions().size());
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            partitionHistogram.merge(context.getPartition(), 1L, Long::sum);
            context.confirm();
            elementsReceived.countDown();
            return elementsReceived.getCount() > 0;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    // Write data.
    final Partitioner partitioner = new KeyAttributePartitioner();
    final Map<Partition, Long> expectedPartitionHistogram = new HashMap<>();
    for (int i = 0; i < numElements; i++) {
        final StreamElement element = StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key_" + i, data.getName(), System.currentTimeMillis(), new byte[] { 1, 2, 3 });
        expectedPartitionHistogram.merge(Partition.of(Partitioners.getTruncatedPartitionId(partitioner, element, numPartitions)), 1L, Long::sum);
        writer.online().write(element, CommitCallback.noop());
    }
    assertEquals(3, expectedPartitionHistogram.size());
    // Wait for all elements to be received.
    elementsReceived.await();
    assertEquals(1, partitionHistogram.size());
    assertEquals(1, observeHandle.getCurrentOffsets().size());
    assertEquals(expectedPartitionHistogram.get(Iterables.getOnlyElement(consumedPartitions)), partitionHistogram.get(Iterables.getOnlyElement(consumedPartitions)));
}