Examples with OnNextContext cz.o2.proxima.direct.commitlog.CommitLogObserver.OnNextContext used on opensource projects

Example 36 with OnNextContext

use of cz.o2.proxima.direct.commitlog.CommitLogObserver.OnNextContext in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testObservePartitionsWithSamePath.

@Test(timeout = 10000)
public void testObservePartitionsWithSamePath() throws InterruptedException {
    InMemStorage storage = new InMemStorage();
    DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem://test1")));
    DataAccessor accessor2 = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem://test2")));
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    CountDownLatch latch = new CountDownLatch(1);
    StreamElement element = StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key", data.getName(), System.currentTimeMillis(), new byte[] { 1, 2, 3 });
    writer.online().write(element, (succ, exc) -> {
    });
    accessor2.getWriter(direct.getContext()).orElseThrow(() -> new IllegalStateException("Missing writer2")).online().write(element, (succ, exc) -> {
    });
    AtomicInteger count = new AtomicInteger();
    reader.observePartitions(reader.getPartitions(), Position.OLDEST, true, new CommitLogObserver() {

        @Override
        public void onCompleted() {
            latch.countDown();
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            assertEquals("key", ingest.getKey());
            context.confirm();
            count.incrementAndGet();
            return true;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    latch.await();
    assertEquals(1, count.get());
}

Example 37 with OnNextContext

use of cz.o2.proxima.direct.commitlog.CommitLogObserver.OnNextContext in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testObserveBatchWithSamePath.

@Test(timeout = 10000)
public void testObserveBatchWithSamePath() throws InterruptedException {
    InMemStorage storage = new InMemStorage();
    DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem://test1")));
    DataAccessor accessor2 = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem://test2")));
    BatchLogReader reader = Optionals.get(accessor.getBatchLogReader(direct.getContext()));
    AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    CountDownLatch latch = new CountDownLatch(1);
    StreamElement element = StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key", data.getName(), System.currentTimeMillis(), new byte[] { 1, 2, 3 });
    writer.online().write(element, (succ, exc) -> {
    });
    accessor2.getWriter(direct.getContext()).orElseThrow(() -> new IllegalStateException("Missing writer2")).online().write(element, (succ, exc) -> {
    });
    AtomicInteger count = new AtomicInteger();
    reader.observe(reader.getPartitions(), Collections.singletonList(data), new BatchLogObserver() {

        @Override
        public void onCompleted() {
            latch.countDown();
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            assertEquals(0, context.getPartition().getId());
            assertEquals("key", ingest.getKey());
            count.incrementAndGet();
            return true;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    latch.await();
    assertEquals(1, count.get());
}

Example 38 with OnNextContext

use of cz.o2.proxima.direct.commitlog.CommitLogObserver.OnNextContext in project proxima-platform by O2-Czech-Republic.

the class ListCommitLog method observe.

@Override
public ObserveHandle observe(@Nullable String name, Position position, CommitLogObserver observer) {
    String consumerName = name == null ? UUID.randomUUID().toString() : name;
    Consumer consumer = CONSUMERS.get(uuid).computeIfAbsent(consumerName, k -> new Consumer(uuid, consumerName, watermarkEstimator));
    ListObserveHandle handle = new ListObserveHandle(uuid, consumerName);
    pushTo((element, offset) -> {
        if (handle.isClosed()) {
            return false;
        }
        final CommitLogObserver.OffsetCommitter committer = (succ, exc) -> {
            if (exc != null) {
                observer.onError(exc);
            }
        };
        final boolean acceptable;
        OnNextContext context = null;
        synchronized (consumer) {
            acceptable = (externalizableOffsets || !consumer.getAckedOffsets().contains(offset) && !consumer.getInflightOffsets().contains(offset));
            if (acceptable) {
                context = consumer.asOnNextContext(committer, offset);
            }
        }
        if (acceptable) {
            return observer.onNext(element, context);
        }
        return true;
    }, externalizableOffsets ? () -> true : allMatchOffset(consumer::isAcked), observer::onCompleted, observer::onCancelled);
    return handle;
}

Example 39 with OnNextContext

use of cz.o2.proxima.direct.commitlog.CommitLogObserver.OnNextContext in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveWithCustomWatemarkEstimator.

@Test(timeout = 1000)
public void testObserveWithCustomWatemarkEstimator() throws InterruptedException {
    long now = System.currentTimeMillis() - 1000;
    int numElements = 10;
    CommitLogReader reader = ListCommitLog.of(data(numElements), new TestWatermarkEstimator(now), direct.getContext());
    CountDownLatch latch = new CountDownLatch(1);
    List<Long> watermarks = new ArrayList<>();
    ObserveHandle handle = reader.observe(null, new CommitLogObserver() {

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            context.confirm();
            watermarks.add(context.getWatermark());
            return true;
        }

        @Override
        public void onCompleted() {
            latch.countDown();
        }
    });
    latch.await();
    assertEquals(numElements, watermarks.size());
    List<Long> expected = IntStream.range(0, numElements).mapToObj(i -> now + i).collect(Collectors.toList());
    assertEquals(expected, watermarks);
}

Example 40 with OnNextContext

use of cz.o2.proxima.direct.commitlog.CommitLogObserver.OnNextContext in project proxima-platform by O2-Czech-Republic.

the class DirectDataOperatorTest method testProxyWrite.

@Test(timeout = 10000)
public void testProxyWrite() throws InterruptedException {
    EntityDescriptor proxied = repo.getEntity("proxied");
    AttributeDescriptor<?> target = proxied.getAttribute("_e.*", true);
    AttributeDescriptor<?> source = proxied.getAttribute("event.*");
    Set<DirectAttributeFamilyDescriptor> families = direct.getFamiliesForAttribute(target);
    Set<DirectAttributeFamilyDescriptor> proxiedFamilies = direct.getFamiliesForAttribute(source);
    assertEquals(families.stream().map(a -> "proxy::" + a.getDesc().getName() + "::" + a.getDesc().getName()).collect(Collectors.toList()), proxiedFamilies.stream().map(a -> a.getDesc().getName()).collect(Collectors.toList()));
    // verify that writing to attribute event.abc ends up as _e.abc
    CountDownLatch latch = new CountDownLatch(2);
    proxiedFamilies.iterator().next().getCommitLogReader().get().observe("dummy", new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            assertNotNull(ingest.getValue());
            assertEquals("test", new String(ingest.getValue()));
            assertEquals("event.abc", ingest.getAttribute());
            assertEquals(source, ingest.getAttributeDescriptor());
            latch.countDown();
            return false;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    assertTrue(direct.getWriter(source).isPresent());
    direct.getWriter(source).get().write(StreamElement.upsert(proxied, source, UUID.randomUUID().toString(), "key", "event.abc", System.currentTimeMillis(), "test".getBytes(StandardCharsets.UTF_8)), (s, exc) -> {
        latch.countDown();
    });
    latch.await();
    KeyValue<?> kv = families.iterator().next().getRandomAccessReader().get().get("key", "_e.raw-abc", target).orElseGet(() -> {
        fail("Missing _e.raw-abc stored");
        return null;
    });
    assertEquals("test", new String(Objects.requireNonNull(kv.getValue())));
}