Examples with ObserveHandle cz.o2.proxima.direct.commitlog.ObserveHandle used on opensource projects

Example 31 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class ReplicationControllerTest method testBulkReplication.

@Test
public void testBulkReplication() throws InterruptedException {
    final List<StreamElement> written = new ArrayList<>();
    final CommitLogReader reader = Optionals.get(direct.getCommitLogReader(data));
    final CommitLogObserver observer = controller.createBulkObserver("consumer", reader, Sets.newHashSet(data), new PassthroughFilter(), fakeBulkWriter(written, stamp -> stamp == now - InMemStorage.getBoundedOutOfOrderness() + 100));
    try (ObserveHandle ignored = reader.observeBulk("consumer", observer)) {
        for (int i = 0; i < 10; i++) {
            writeEvent(now + 20 * i);
        }
        assertEquals(6, written.size());
    }
    livenessLatch.await();
    assertEquals(1.0, Metrics.LIVENESS.getValue(), 0.0001);
}

Example 32 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class ReplicationControllerTest method testSimpleEventReplication.

@Test(timeout = 5000)
public void testSimpleEventReplication() throws InterruptedException {
    List<StreamElement> written = new ArrayList<>();
    final CommitLogReader reader = Optionals.get(direct.getCommitLogReader(data));
    final CommitLogObserver observer = controller.createOnlineObserver("consumer", direct.getCommitLogReader(data).orElseThrow(() -> new IllegalArgumentException("Missing commit log reader for data")), Sets.newHashSet(data), new PassthroughFilter(), fakeOnlineWriter(written));
    try (ObserveHandle ignored = reader.observe("consumer", observer)) {
        writeEvent();
        assertEquals(1, written.size());
    }
    livenessLatch.await();
    assertEquals(1.0, Metrics.LIVENESS.getValue(), 0.0001);
}

Example 33 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveNonExternalizableWatermark.

@Test
public void testObserveNonExternalizableWatermark() throws InterruptedException {
    int numElements = 10;
    CommitLogReader reader = ListCommitLog.ofNonExternalizable(data(numElements), direct.getContext());
    CountDownLatch first = new CountDownLatch(1);
    List<Long> watermarks = new ArrayList<>();
    ObserveHandle handle = reader.observeBulk(null, new CommitLogObserver() {

        int received = 0;

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            watermarks.add(context.getWatermark());
            if (++received == numElements) {
                context.confirm();
            }
            return true;
        }

        @Override
        public void onCompleted() {
            first.countDown();
        }
    });
    first.await();
    assertEquals(numElements, watermarks.size());
    long min = watermarks.get(0);
    for (int i = 1; i < numElements; i++) {
        assertEquals(min, (long) watermarks.get(i));
    }
}

Example 34 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveNonExternalizableUnnamedPauseContinueNoCommit.

@Test(timeout = 10000)
public void testObserveNonExternalizableUnnamedPauseContinueNoCommit() throws InterruptedException {
    CommitLogReader reader = ListCommitLog.ofNonExternalizable(data(10), direct.getContext());
    List<StreamElement> data = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    ObserveHandle handle = reader.observe(null, toList(data, b -> latch.countDown(), v -> v.getValue()[0] < 5));
    latch.await();
    assertEquals(6, data.size());
    assertFalse(handle.getCommittedOffsets().isEmpty());
    assertFalse(handle.getCurrentOffsets().isEmpty());
    CountDownLatch nextLatch = new CountDownLatch(1);
    reader.observeBulkOffsets(handle.getCurrentOffsets(), toList(data, b -> nextLatch.countDown()));
    nextLatch.await();
    assertEquals(10, data.size());
}

Example 35 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class TransformationRunner method runTransformation.

/**
 * Run given transformation in local JVM.
 *
 * @param direct the operator to run transformations with
 * @param name name of the transformation
 * @param desc the transformation to run
 * @param onReplicated callback to be called before write to replicated target
 * @return {@link ObserveHandle} of the transformation
 */
public static ObserveHandle runTransformation(DirectDataOperator direct, String name, TransformationDescriptor desc, Consumer<StreamElement> onReplicated) {
    final CommitLogObserver observer;
    if (desc.getTransformation().isContextual()) {
        observer = new TransformationObserver.Contextual(direct, name, desc.getTransformation().as(DirectElementWiseTransform.class), desc.getOutputTransactionMode() == OutputTransactionMode.ENABLED, desc.getFilter()) {

            @Override
            protected void onReplicated(StreamElement element) {
                onReplicated.accept(element);
            }
        };
    } else {
        observer = new TransformationObserver.NonContextual(direct, name, desc.getTransformation().asElementWiseTransform(), desc.getOutputTransactionMode() == OutputTransactionMode.ENABLED, desc.getFilter()) {

            @Override
            protected void onReplicated(StreamElement element) {
                onReplicated.accept(element);
            }
        };
    }
    CommitLogReader reader = desc.getAttributes().stream().flatMap(attr -> findFamilyDescriptorForAttribute(direct, attr)).findAny().flatMap(DirectAttributeFamilyDescriptor::getCommitLogReader).orElseThrow(() -> new IllegalStateException("No commit log reader for attributes of transformation " + desc));
    log.debug("Starting to observe reader {} with observer {} as {}", reader, observer, name);
    return reader.observe(name, observer);
}