Examples with ObserveHandle cz.o2.proxima.direct.commitlog.ObserveHandle used on opensource projects

Example 71 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class KafkaLogReaderIT method testReadFromOldestVerifyIsAtHead.

@Test(timeout = 30_000L)
public void testReadFromOldestVerifyIsAtHead() throws InterruptedException {
    final EmbeddedKafkaBroker embeddedKafka = rule.getEmbeddedKafka();
    final int numPartitions = 3;
    embeddedKafka.addTopics(new NewTopic("foo", numPartitions, (short) 1));
    final CommitLogReader commitLogReader = Optionals.get(operator.getCommitLogReader(fooDescriptor));
    // Write everything up front, so we can be sure that we really seek all the way to beginning.
    final int numElements = 100;
    await(writeElements(numElements));
    AtomicReference<ObserveHandle> handle = new AtomicReference<>();
    CountDownLatch firstLatch = new CountDownLatch(1);
    CountDownLatch lastLatch = new CountDownLatch(1);
    CommitLogObserver observer = new CommitLogObserver() {

        int consumed = 0;

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            context.confirm();
            if (consumed == 0) {
                ExceptionUtils.ignoringInterrupted(firstLatch::await);
            }
            if (++consumed == numElements) {
                lastLatch.countDown();
            }
            return true;
        }
    };
    handle.set(commitLogReader.observe("test-reader", Position.OLDEST, observer));
    assertFalse(ObserveHandleUtils.isAtHead(handle.get(), commitLogReader));
    firstLatch.countDown();
    lastLatch.await();
    assertTrue(ObserveHandleUtils.isAtHead(handle.get(), commitLogReader));
}

Example 72 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class LocalKafkaCommitLogDescriptorTest method testFetchOffsets.

@Test(timeout = 10_000)
public void testFetchOffsets() throws InterruptedException {
    Accessor accessor = kafka.createAccessor(direct, createTestFamily(entity, storageUri, partitionsCfg(3)));
    LocalKafkaWriter writer = accessor.newWriter();
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(context()));
    final CountDownLatch latch = new CountDownLatch(2);
    final StreamElement[] updates = new StreamElement[] { StreamElement.upsert(entity, attr, UUID.randomUUID().toString(), "key", attr.getName(), System.currentTimeMillis(), new byte[] { 1, 2 }), StreamElement.upsert(entity, attr, UUID.randomUUID().toString(), "key2", attr.getName(), System.currentTimeMillis(), new byte[] { 1, 2, 3 }) };
    Arrays.stream(updates).forEach(update -> writer.write(update, (succ, exc) -> {
    }));
    CommitLogObserver observer = new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            context.confirm();
            latch.countDown();
            return true;
        }

        @Override
        public void onCompleted() {
            fail("This should not be called");
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    };
    ObserveHandle handle = reader.observeBulkOffsets(reader.fetchOffsets(Position.OLDEST, reader.getPartitions()).values(), observer);
    latch.await();
    assertEquals(handle.getCommittedOffsets().stream().map(TopicOffset.class::cast).sorted(Comparator.comparing(tp -> tp.getPartition().getId())).collect(Collectors.toList()), reader.fetchOffsets(Position.NEWEST, reader.getPartitions()).values().stream().map(TopicOffset.class::cast).sorted(Comparator.comparing(tp -> tp.getPartition().getId())).collect(Collectors.toList()));
}

Example 73 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class HBaseLogReaderTest method testObserveCancel.

@Test(timeout = 5000)
public void testObserveCancel() throws InterruptedException, IOException {
    long now = 1500000000000L;
    write("a", "dummy", "a", now);
    write("firs", "wildcard.1", "firs", now);
    write("fir", "dummy", "fir", now);
    write("first", "dummy", "first", now);
    List<Partition> partitions = reader.getPartitions();
    CountDownLatch latch = new CountDownLatch(1);
    AtomicReference<ObserveHandle> handle = new AtomicReference<>();
    handle.set(reader.observe(partitions.subList(0, 1), Collections.singletonList(attr), new BatchLogObserver() {

        @Override
        public boolean onNext(StreamElement element) {
            handle.get().close();
            return true;
        }

        @Override
        public void onCancelled() {
            latch.countDown();
        }

        @Override
        public void onCompleted() {
            fail("onCompleted should not heve been called");
        }
    }));
    latch.await();
}

Example 74 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class WebsocketReader method observe.

private ObserveHandle observe(Consumer<StreamElement> onNext, Consumer<Throwable> onError) {
    WebSocketClient client = new WebSocketClient(getUri()) {

        @Override
        public void onOpen(ServerHandshake sh) {
            send(hello);
        }

        @Override
        public void onMessage(String m) {
            StreamElement elem = StreamElement.upsert(getEntityDescriptor(), attr, UUID.randomUUID().toString(), keyExtractor.apply(m), attr.getName(), System.currentTimeMillis(), m.getBytes(StandardCharsets.UTF_8));
            onNext.accept(elem);
        }

        @Override
        public void onClose(int code, String reason, boolean remote) {
            if (remote) {
                onError.accept(new RuntimeException("Server error: " + code + ": " + reason));
            }
        }

        @Override
        public void onError(Exception excptn) {
            onError.accept(excptn);
        }
    };
    client.connect();
    return new ObserveHandle() {

        @Override
        public void close() {
            client.close();
        }

        @Override
        public List<Offset> getCommittedOffsets() {
            return Arrays.asList(OFFSET);
        }

        @Override
        public void resetOffsets(List<Offset> offsets) {
        // nop
        }

        @Override
        public List<Offset> getCurrentOffsets() {
            return getCommittedOffsets();
        }

        @Override
        public void waitUntilReady() throws InterruptedException {
        // nop
        }
    };
}