Examples with CommitLogReader cz.o2.proxima.direct.commitlog.CommitLogReader used on opensource projects

Example 56 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveNonExternalizableUnnamedPauseContinue.

@Test(timeout = 10000)
public void testObserveNonExternalizableUnnamedPauseContinue() throws InterruptedException {
    CommitLogReader reader = ListCommitLog.ofNonExternalizable(data(10), direct.getContext());
    List<StreamElement> data = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    ObserveHandle handle = reader.observe(null, toList(data, b -> latch.countDown(), v -> v.getValue()[0] < 5));
    latch.await();
    assertEquals(6, data.size());
    assertFalse(handle.getCommittedOffsets().isEmpty());
    assertFalse(handle.getCurrentOffsets().isEmpty());
    CountDownLatch nextLatch = new CountDownLatch(1);
    reader.observeBulkOffsets(handle.getCurrentOffsets(), toList(data, b -> nextLatch.countDown()));
    nextLatch.await();
    assertEquals(10, data.size());
}

Example 57 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveNonExternalizableUnnamed.

@Test(timeout = 10000)
public void testObserveNonExternalizableUnnamed() throws InterruptedException {
    CommitLogReader reader = ListCommitLog.ofNonExternalizable(data(10), direct.getContext());
    List<StreamElement> data = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    ObserveHandle handle = reader.observe(null, toList(data, b -> latch.countDown()));
    latch.await();
    assertEquals(10, data.size());
    assertFalse(handle.getCommittedOffsets().isEmpty());
    assertFalse(handle.getCurrentOffsets().isEmpty());
    ListObserveHandle listObserveHandle = (ListObserveHandle) handle;
    ListCommitLog.Consumer consumer = listObserveHandle.getConsumer();
    assertTrue(consumer.getInflightOffsets().isEmpty());
}

Example 58 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveBulkNonExternalizableUnnamed.

@Test(timeout = 10000)
public void testObserveBulkNonExternalizableUnnamed() throws InterruptedException {
    CommitLogReader reader = ListCommitLog.ofNonExternalizable(data(10), direct.getContext());
    List<StreamElement> data = new ArrayList<>();
    CountDownLatch first = new CountDownLatch(1);
    CountDownLatch second = new CountDownLatch(1);
    ObserveHandle handle = reader.observeBulk(null, new CommitLogObserver() {

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            context.nack();
            return false;
        }

        @Override
        public void onCancelled() {
            first.countDown();
        }
    });
    first.await();
    List<Offset> offsets = handle.getCurrentOffsets();
    handle = reader.observeBulkOffsets(offsets, toList(data, b -> second.countDown()));
    second.await();
    assertEquals(10, data.size());
    assertFalse(handle.getCommittedOffsets().isEmpty());
    assertFalse(handle.getCurrentOffsets().isEmpty());
    ListObserveHandle listObserveHandle = (ListObserveHandle) handle;
    ListCommitLog.Consumer consumer = listObserveHandle.getConsumer();
    assertTrue(consumer.getInflightOffsets().isEmpty());
}

Example 59 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testFetchOffsets.

private void testFetchOffsets(int numPartitions) throws InterruptedException {
    InMemStorage storage = new InMemStorage();
    DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem:///test"), numPartitions));
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    long now = System.currentTimeMillis();
    List<StreamElement> updates = new ArrayList<>();
    for (int i = 0; i < 2 * numPartitions; i++) {
        updates.add(StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key" + (i + 1), data.getName(), now + i, new byte[] { 1, 2, 3 }));
    }
    updates.forEach(el -> writer.online().write(el, (succ, exc) -> {
    }));
    Map<Partition, Offset> startingOffsets = reader.fetchOffsets(Position.OLDEST, reader.getPartitions());
    assertEquals(numPartitions, startingOffsets.size());
    List<StreamElement> elements = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    CountDownLatch startLatch = new CountDownLatch(1);
    CommitLogObserver observer = LogObserverUtils.toList(elements, ign -> latch.countDown(), el -> {
        ExceptionUtils.ignoringInterrupted(startLatch::await);
        return true;
    });
    ObserveHandle handle = reader.observeBulkOffsets(startingOffsets.values(), true, observer);
    assertFalse(ObserveHandleUtils.isAtHead(handle, reader));
    startLatch.countDown();
    latch.await();
    assertEquals(2 * numPartitions, elements.size());
    List<Offset> committed = handle.getCommittedOffsets();
    Map<Partition, Offset> endOffsets2 = reader.fetchOffsets(Position.NEWEST, committed.stream().map(Offset::getPartition).collect(Collectors.toList()));
    assertTrue(ObserveHandleUtils.isAtHead(handle, reader));
    assertEquals(IntStream.range(0, 2 * numPartitions).mapToObj(i -> "key" + (i + 1)).collect(Collectors.toList()), elements.stream().map(StreamElement::getKey).collect(Collectors.toList()));
    elements.clear();
    Map<Partition, Offset> endOffsets = reader.fetchOffsets(Position.NEWEST, reader.getPartitions());
    assertEquals(numPartitions, endOffsets.size());
    CountDownLatch latch2 = new CountDownLatch(1);
    observer = LogObserverUtils.toList(elements, ign -> latch2.countDown());
    reader.observeBulkOffsets(endOffsets.values(), true, observer);
    latch2.await();
    assertEquals(numPartitions, elements.size());
    assertEquals(IntStream.range(numPartitions, 2 * numPartitions).mapToObj(i -> "key" + (i + 1)).collect(Collectors.toList()), elements.stream().map(StreamElement::getKey).collect(Collectors.toList()));
}

Example 60 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testObserveCancel.

@Test(timeout = 10000)
public void testObserveCancel() {
    InMemStorage storage = new InMemStorage();
    DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem:///inmemstoragetest")));
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    List<Byte> received = new ArrayList<>();
    ObserveHandle handle = reader.observePartitions(reader.getPartitions(), new CommitLogObserver() {

        @Override
        public void onRepartition(CommitLogObserver.OnRepartitionContext context) {
            assertEquals(1, context.partitions().size());
        }

        @Override
        public boolean onNext(StreamElement ingest, CommitLogObserver.OnNextContext context) {
            assertEquals(0, context.getPartition().getId());
            assertEquals("key", ingest.getKey());
            context.confirm();
            received.add(ingest.getValue()[0]);
            return false;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    writer.online().write(StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key", data.getName(), System.currentTimeMillis(), new byte[] { 1 }), (succ, exc) -> {
    });
    List<Offset> offsets = handle.getCurrentOffsets();
    assertEquals(1, offsets.size());
    assertEquals(Collections.singletonList((byte) 1), received);
    handle.close();
    writer.online().write(StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key", data.getName(), System.currentTimeMillis(), new byte[] { 2 }), (succ, exc) -> {
    });
    assertEquals(Collections.singletonList((byte) 1), received);
}