Examples with CommitLogObserver cz.o2.proxima.direct.commitlog.CommitLogObserver used on opensource projects

Example 31 with CommitLogObserver

use of cz.o2.proxima.direct.commitlog.CommitLogObserver in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testReadWriteSequentialIds.

@Test(timeout = 10000)
public void testReadWriteSequentialIds() throws InterruptedException {
    InMemStorage storage = new InMemStorage();
    DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem:///inmemstoragetest")));
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    List<StreamElement> result = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    CommitLogObserver observer = LogObserverUtils.toList(result, Assert::assertTrue, elem -> {
        latch.countDown();
        return false;
    });
    reader.observe("test", observer);
    writer.online().write(StreamElement.upsert(entity, data, 1L, "key", data.getName(), System.currentTimeMillis(), new byte[] { 1, 2, 3 }), (succ, exc) -> {
    });
    latch.await();
    assertEquals(1, result.size());
    assertTrue(result.get(0).hasSequentialId());
    assertEquals(1L, result.get(0).getSequentialId());
}

Example 32 with CommitLogObserver

use of cz.o2.proxima.direct.commitlog.CommitLogObserver in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testObserveError.

@Test(timeout = 1000L)
public void testObserveError() throws InterruptedException {
    final URI uri = URI.create("inmem:///inmemstoragetest");
    final InMemStorage storage = new InMemStorage();
    final DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(uri));
    final CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    final AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    final CountDownLatch failingObserverErrorReceived = new CountDownLatch(1);
    final AtomicInteger failingObserverMessages = new AtomicInteger(0);
    reader.observe("failing-observer", new CommitLogObserver() {

        @Override
        public void onCompleted() {
            throw new UnsupportedOperationException("This should never happen.");
        }

        @Override
        public boolean onError(Throwable error) {
            failingObserverErrorReceived.countDown();
            return false;
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            failingObserverMessages.incrementAndGet();
            throw new RuntimeException("Test exception.");
        }
    });
    final int numElements = 100;
    final CountDownLatch successObserverAllReceived = new CountDownLatch(numElements);
    reader.observe("success-observer", new CommitLogObserver() {

        @Override
        public void onCompleted() {
            throw new UnsupportedOperationException("This should never happen.");
        }

        @Override
        public boolean onError(Throwable error) {
            return false;
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            successObserverAllReceived.countDown();
            return true;
        }
    });
    for (int i = 0; i < numElements; i++) {
        writer.online().write(StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key_" + i, data.getName(), System.currentTimeMillis(), new byte[] { 2 }), (success, error) -> {
        });
    }
    failingObserverErrorReceived.await();
    successObserverAllReceived.await();
    assertEquals(1, failingObserverMessages.get());
}

Example 33 with CommitLogObserver

use of cz.o2.proxima.direct.commitlog.CommitLogObserver in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testObserveWithEndOfTime.

@Test
public void testObserveWithEndOfTime() throws InterruptedException {
    URI uri = URI.create("inmem:///inmemstoragetest");
    InMemStorage storage = new InMemStorage();
    InMemStorage.setWatermarkEstimatorFactory(uri, (stamp, name, offset) -> new WatermarkEstimator() {

        {
            Preconditions.checkArgument(offset != null);
        }

        @Override
        public long getWatermark() {
            return Watermarks.MAX_WATERMARK - InMemStorage.getBoundedOutOfOrderness();
        }

        @Override
        public void update(StreamElement element) {
        }

        @Override
        public void setMinWatermark(long minWatermark) {
        }
    });
    DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(uri));
    CommitLogReader reader = accessor.getCommitLogReader(direct.getContext()).orElseThrow(() -> new IllegalStateException("Missing commit log reader"));
    CountDownLatch completed = new CountDownLatch(1);
    reader.observe("observer", new CommitLogObserver() {

        @Override
        public void onCompleted() {
            completed.countDown();
        }

        @Override
        public boolean onError(Throwable error) {
            return false;
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            return false;
        }
    });
    assertTrue(completed.await(1, TimeUnit.SECONDS));
}

Example 34 with CommitLogObserver

use of cz.o2.proxima.direct.commitlog.CommitLogObserver in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testObserveSinglePartitionOutOfMultiplePartitions.

@Test
public void testObserveSinglePartitionOutOfMultiplePartitions() throws InterruptedException {
    final int numPartitions = 3;
    final InMemStorage storage = new InMemStorage();
    final DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem:///test"), numPartitions));
    final CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    final AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    final int numElements = 999;
    final ConcurrentMap<Partition, Long> partitionHistogram = new ConcurrentHashMap<>();
    // Elements are uniformly distributed between partitions.
    final CountDownLatch elementsReceived = new CountDownLatch(numElements / numPartitions);
    // Start observer.
    final List<Partition> consumedPartitions = reader.getPartitions().subList(0, 1);
    final ObserveHandle observeHandle = reader.observePartitions(reader.getPartitions().subList(0, 1), new CommitLogObserver() {

        @Override
        public void onRepartition(OnRepartitionContext context) {
            assertEquals(numPartitions, context.partitions().size());
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            partitionHistogram.merge(context.getPartition(), 1L, Long::sum);
            context.confirm();
            elementsReceived.countDown();
            return elementsReceived.getCount() > 0;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    // Write data.
    final Partitioner partitioner = new KeyAttributePartitioner();
    final Map<Partition, Long> expectedPartitionHistogram = new HashMap<>();
    for (int i = 0; i < numElements; i++) {
        final StreamElement element = StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key_" + i, data.getName(), System.currentTimeMillis(), new byte[] { 1, 2, 3 });
        expectedPartitionHistogram.merge(Partition.of(Partitioners.getTruncatedPartitionId(partitioner, element, numPartitions)), 1L, Long::sum);
        writer.online().write(element, CommitCallback.noop());
    }
    assertEquals(3, expectedPartitionHistogram.size());
    // Wait for all elements to be received.
    elementsReceived.await();
    assertEquals(1, partitionHistogram.size());
    assertEquals(1, observeHandle.getCurrentOffsets().size());
    assertEquals(expectedPartitionHistogram.get(Iterables.getOnlyElement(consumedPartitions)), partitionHistogram.get(Iterables.getOnlyElement(consumedPartitions)));
}

Example 35 with CommitLogObserver

use of cz.o2.proxima.direct.commitlog.CommitLogObserver in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testObserveMultiplePartitions.

@Test
public void testObserveMultiplePartitions() throws InterruptedException {
    final int numPartitions = 3;
    final InMemStorage storage = new InMemStorage();
    final DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem:///test"), numPartitions));
    final CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    final AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    final int numElements = 1_000;
    final ConcurrentMap<Partition, Long> partitionHistogram = new ConcurrentHashMap<>();
    final CountDownLatch elementsReceived = new CountDownLatch(numElements);
    // Start observer.
    final ObserveHandle observeHandle = reader.observePartitions(reader.getPartitions(), new CommitLogObserver() {

        @Override
        public void onRepartition(OnRepartitionContext context) {
            assertEquals(numPartitions, context.partitions().size());
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            partitionHistogram.merge(context.getPartition(), 1L, Long::sum);
            context.confirm();
            elementsReceived.countDown();
            return elementsReceived.getCount() > 0;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    // Write data.
    for (int i = 0; i < numElements; i++) {
        writer.online().write(StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key_" + i, data.getName(), System.currentTimeMillis(), new byte[] { 1, 2, 3 }), CommitCallback.noop());
    }
    // Wait for all elements to be received.
    elementsReceived.await();
    assertEquals(3, partitionHistogram.size());
    assertEquals(3, observeHandle.getCurrentOffsets().size());
}