Examples with ObserveHandle cz.o2.proxima.direct.commitlog.ObserveHandle used on opensource projects

Example 51 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testFetchOffsets.

private void testFetchOffsets(int numPartitions) throws InterruptedException {
    InMemStorage storage = new InMemStorage();
    DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem:///test"), numPartitions));
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    long now = System.currentTimeMillis();
    List<StreamElement> updates = new ArrayList<>();
    for (int i = 0; i < 2 * numPartitions; i++) {
        updates.add(StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key" + (i + 1), data.getName(), now + i, new byte[] { 1, 2, 3 }));
    }
    updates.forEach(el -> writer.online().write(el, (succ, exc) -> {
    }));
    Map<Partition, Offset> startingOffsets = reader.fetchOffsets(Position.OLDEST, reader.getPartitions());
    assertEquals(numPartitions, startingOffsets.size());
    List<StreamElement> elements = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    CountDownLatch startLatch = new CountDownLatch(1);
    CommitLogObserver observer = LogObserverUtils.toList(elements, ign -> latch.countDown(), el -> {
        ExceptionUtils.ignoringInterrupted(startLatch::await);
        return true;
    });
    ObserveHandle handle = reader.observeBulkOffsets(startingOffsets.values(), true, observer);
    assertFalse(ObserveHandleUtils.isAtHead(handle, reader));
    startLatch.countDown();
    latch.await();
    assertEquals(2 * numPartitions, elements.size());
    List<Offset> committed = handle.getCommittedOffsets();
    Map<Partition, Offset> endOffsets2 = reader.fetchOffsets(Position.NEWEST, committed.stream().map(Offset::getPartition).collect(Collectors.toList()));
    assertTrue(ObserveHandleUtils.isAtHead(handle, reader));
    assertEquals(IntStream.range(0, 2 * numPartitions).mapToObj(i -> "key" + (i + 1)).collect(Collectors.toList()), elements.stream().map(StreamElement::getKey).collect(Collectors.toList()));
    elements.clear();
    Map<Partition, Offset> endOffsets = reader.fetchOffsets(Position.NEWEST, reader.getPartitions());
    assertEquals(numPartitions, endOffsets.size());
    CountDownLatch latch2 = new CountDownLatch(1);
    observer = LogObserverUtils.toList(elements, ign -> latch2.countDown());
    reader.observeBulkOffsets(endOffsets.values(), true, observer);
    latch2.await();
    assertEquals(numPartitions, elements.size());
    assertEquals(IntStream.range(numPartitions, 2 * numPartitions).mapToObj(i -> "key" + (i + 1)).collect(Collectors.toList()), elements.stream().map(StreamElement::getKey).collect(Collectors.toList()));
}

Example 52 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testObserveCancel.

@Test(timeout = 10000)
public void testObserveCancel() {
    InMemStorage storage = new InMemStorage();
    DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem:///inmemstoragetest")));
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    List<Byte> received = new ArrayList<>();
    ObserveHandle handle = reader.observePartitions(reader.getPartitions(), new CommitLogObserver() {

        @Override
        public void onRepartition(CommitLogObserver.OnRepartitionContext context) {
            assertEquals(1, context.partitions().size());
        }

        @Override
        public boolean onNext(StreamElement ingest, CommitLogObserver.OnNextContext context) {
            assertEquals(0, context.getPartition().getId());
            assertEquals("key", ingest.getKey());
            context.confirm();
            received.add(ingest.getValue()[0]);
            return false;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    writer.online().write(StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key", data.getName(), System.currentTimeMillis(), new byte[] { 1 }), (succ, exc) -> {
    });
    List<Offset> offsets = handle.getCurrentOffsets();
    assertEquals(1, offsets.size());
    assertEquals(Collections.singletonList((byte) 1), received);
    handle.close();
    writer.online().write(StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key", data.getName(), System.currentTimeMillis(), new byte[] { 2 }), (succ, exc) -> {
    });
    assertEquals(Collections.singletonList((byte) 1), received);
}

Example 53 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class ReplicationControllerTest method testSimpleEventReplicationWithFilter.

@Test
public void testSimpleEventReplicationWithFilter() {
    final List<StreamElement> written = new ArrayList<>();
    final CommitLogReader reader = Optionals.get(direct.getCommitLogReader(data));
    final CommitLogObserver observer = controller.createOnlineObserver("consumer", direct.getCommitLogReader(data).orElseThrow(() -> new IllegalArgumentException("Missing commit log reader for data")), Sets.newHashSet(data), ingest -> false, fakeOnlineWriter(written));
    try (ObserveHandle ignored = reader.observe("consumer", observer)) {
        writeEvent();
        assertEquals(0, written.size());
    }
}

Example 54 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class ReplicationRunner method runAttributeReplicas.

/**
 * Run replications of attributes to replica attribute families.
 *
 * @param direct {@link DirectDataOperator} direct data operator
 * @param onReplicated callback called for each replicated element
 */
public static void runAttributeReplicas(DirectDataOperator direct, Consumer<StreamElement> onReplicated) {
    direct.getAllFamilies().filter(af -> af.getDesc().getType() == StorageType.REPLICA).filter(af -> !af.getDesc().getAccess().isReadonly() && !af.getDesc().isProxy()).forEach(af -> {
        List<AttributeDescriptor<?>> attributes = af.getAttributes();
        final AttributeWriterBase writer = Optionals.get(af.getWriter());
        final CommitLogReader primaryCommitLogReader = Optionals.get(attributes.stream().flatMap(a -> direct.getFamiliesForAttribute(a).stream()).filter(f -> f.getDesc().getType() == StorageType.PRIMARY).distinct().findFirst().flatMap(DirectAttributeFamilyDescriptor::getCommitLogReader));
        final ObserveHandle handle;
        if (writer instanceof OnlineAttributeWriter) {
            final OnlineAttributeWriter onlineWriter = writer.online();
            handle = primaryCommitLogReader.observe(af.getDesc().getName(), (CommitLogObserver) (ingest, context) -> {
                log.debug("Replicating input {} to {}", ingest, writer);
                onlineWriter.write(ingest, (succ, exc) -> {
                    context.commit(succ, exc);
                    onReplicated.accept(ingest);
                });
                return true;
            });
        } else {
            final BulkAttributeWriter bulkWriter = writer.bulk();
            handle = primaryCommitLogReader.observe(af.getDesc().getName(), (CommitLogObserver) (ingest, context) -> {
                log.debug("Replicating input {} to {}", ingest, writer);
                bulkWriter.write(ingest, context.getWatermark(), (succ, exc) -> {
                    context.commit(succ, exc);
                    onReplicated.accept(ingest);
                });
                return true;
            });
        }
        ExceptionUtils.unchecked(handle::waitUntilReady);
        log.info("Started attribute replica {}", af.getDesc().getName());
    });
}

Example 55 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class BlobLogReader method observe.

@Override
public ObserveHandle observe(List<Partition> partitions, List<AttributeDescriptor<?>> attributes, BatchLogObserver observer) {
    TerminationContext terminationContext = new TerminationContext(observer);
    observeInternal(partitions, attributes, observer, terminationContext);
    return terminationContext.asObserveHandle();
}