Examples with UnaryFunction cz.o2.proxima.functional.UnaryFunction used on opensource projects

Example 1 with UnaryFunction

use of cz.o2.proxima.functional.UnaryFunction in project proxima-platform by O2-Czech-Republic.

the class LocalKafkaCommitLogDescriptorTest method testObserveMovesWatermark.

@Test(timeout = 10000)
public void testObserveMovesWatermark() throws InterruptedException {
    Accessor accessor = kafka.createAccessor(direct, createTestFamily(entity, storageUri, partitionsCfg(3)));
    LocalKafkaWriter writer = accessor.newWriter();
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(context()));
    long now = System.currentTimeMillis();
    final UnaryFunction<Integer, StreamElement> update = pos -> StreamElement.upsert(entity, attr, UUID.randomUUID().toString(), "key" + pos, attr.getName(), now + pos, new byte[] { 1, 2 });
    AtomicLong watermark = new AtomicLong();
    CountDownLatch latch = new CountDownLatch(100);
    reader.observe("test", Position.NEWEST, new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            watermark.set(context.getWatermark());
            latch.countDown();
            return true;
        }

        @Override
        public void onCompleted() {
            fail("This should not be called");
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    }).waitUntilReady();
    for (int i = 0; i < 100; i++) {
        writer.write(update.apply(i), (succ, e) -> {
        });
    }
    latch.await();
    assertTrue(watermark.get() > 0);
    assertTrue(watermark.get() < now * 10);
}

Example 2 with UnaryFunction

use of cz.o2.proxima.functional.UnaryFunction in project proxima-platform by O2-Czech-Republic.

the class BeamStream method writeUsingOnlineWriterFactory.

void writeUsingOnlineWriterFactory(String name, UnaryFunction<StreamElement, OnlineAttributeWriter> writerFactory) {
    @SuppressWarnings("unchecked") BeamStream<StreamElement> elements = (BeamStream<StreamElement>) this;
    elements.forEach(name, el -> {
        OnlineAttributeWriter writer = writerFactory.apply(el);
        CountDownLatch latch = new CountDownLatch(1);
        AtomicReference<Throwable> err = new AtomicReference<>();
        writer.online().write(el, (succ, exc) -> {
            latch.countDown();
            err.set(exc);
        });
        try {
            latch.await();
        } catch (InterruptedException ex) {
            Thread.currentThread().interrupt();
            throw new RuntimeException(ex);
        }
        if (err.get() != null) {
            throw new RuntimeException(err.get());
        }
    }, false);
}

Example 3 with UnaryFunction

use of cz.o2.proxima.functional.UnaryFunction in project proxima-platform by O2-Czech-Republic.

the class PubSubReader method consume.

private ObserveHandle consume(String consumerName, PubSubConsumer consumer, UnaryFunction<Throwable, Boolean> errorHandler, @Nullable Runnable onInit, Runnable onRestart, Runnable onCancel, AtomicLong committedWatermark) {
    ProjectSubscriptionName subscription = ProjectSubscriptionName.of(project, consumerName);
    AtomicReference<Subscriber> subscriber = new AtomicReference<>();
    AtomicBoolean stopProcessing = new AtomicBoolean();
    AtomicReference<MessageReceiver> receiver = new AtomicReference<>();
    WatermarkEstimator watermarkEstimator = createWatermarkEstimator(committedWatermark.get());
    receiver.set(createMessageReceiver(subscription, subscriber, stopProcessing, consumer, watermarkEstimator, errorHandler, onRestart, receiver));
    subscriber.set(newSubscriber(subscription, receiver.get()));
    subscriber.get().startAsync();
    if (onInit != null) {
        executor().submit(() -> {
            subscriber.get().awaitRunning();
            if (onInit != null) {
                onInit.run();
            }
        });
    }
    return new ObserveHandle() {

        @Override
        public void close() {
            log.debug("Cancelling observer {}", consumerName);
            stopProcessing.set(true);
            Subscriber sub = stopAsync(subscriber);
            if (sub != null) {
                sub.awaitTerminated();
            }
            onCancel.run();
        }

        @Override
        public List<Offset> getCommittedOffsets() {
            return Collections.singletonList(new PubSubOffset(consumerName, committedWatermark.get()));
        }

        @Override
        public void resetOffsets(List<Offset> offsets) {
        // nop
        }

        @Override
        public List<Offset> getCurrentOffsets() {
            return getCommittedOffsets();
        }

        @Override
        public void waitUntilReady() {
            subscriber.get().awaitRunning();
        }
    };
}

Example 4 with UnaryFunction

use of cz.o2.proxima.functional.UnaryFunction in project proxima-platform by O2-Czech-Republic.

the class CommitLogReaderTest method testThroughputLimitedCommitLogIdles.

@Test(timeout = 10000)
public void testThroughputLimitedCommitLogIdles() throws InterruptedException {
    ThroughputLimiter limiter = ThroughputLimiter.NoOpThroughputLimiter.INSTANCE;
    CommitLogReader limitedReader = CommitLogReaders.withThroughputLimit(reader, limiter);
    UnaryFunction<CountDownLatch, CommitLogObserver> observerFactory = myLatch -> new CommitLogObserver() {

        @Override
        public boolean onError(Throwable error) {
            return false;
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            return false;
        }

        @Override
        public void onIdle(OnIdleContext context) {
            myLatch.countDown();
        }
    };
    CountDownLatch latch = new CountDownLatch(1);
    CommitLogObserver observer = observerFactory.apply(latch);
    long nanoTime = System.nanoTime();
    ObserveHandle handle = limitedReader.observe("dummy", observer);
    latch.await();
    handle.close();
    latch = new CountDownLatch(1);
    observer = observerFactory.apply(latch);
    long durationMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - nanoTime);
    handle.close();
    limitedReader = CommitLogReaders.withThroughputLimit(reader, withNumRecordsPerSec(1));
    handle = limitedReader.observe("dummy", observer);
    // no idle called this time
    assertFalse(latch.await(2 * durationMillis, TimeUnit.MILLISECONDS));
}

Example 5 with UnaryFunction

use of cz.o2.proxima.functional.UnaryFunction in project proxima-platform by O2-Czech-Republic.

the class LocalKafkaCommitLogDescriptorTest method testCustomWatermarkEstimator.

@Test(timeout = 10000)
public void testCustomWatermarkEstimator() throws InterruptedException {
    Map<String, Object> cfg = partitionsCfg(3);
    cfg.put("watermark.estimator-factory", FixedWatermarkEstimatorFactory.class.getName());
    Accessor accessor = kafka.createAccessor(direct, createTestFamily(entity, storageUri, cfg));
    LocalKafkaWriter writer = accessor.newWriter();
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(context()));
    long now = System.currentTimeMillis();
    final UnaryFunction<Integer, StreamElement> update = pos -> StreamElement.upsert(entity, attr, UUID.randomUUID().toString(), "key" + pos, attr.getName(), now + pos, new byte[] { 1, 2 });
    AtomicLong watermark = new AtomicLong();
    CountDownLatch latch = new CountDownLatch(100);
    reader.observe("test", Position.NEWEST, new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            watermark.set(context.getWatermark());
            latch.countDown();
            return true;
        }

        @Override
        public void onCompleted() {
            fail("This should not be called");
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    }).waitUntilReady();
    for (int i = 0; i < 100; i++) {
        writer.write(update.apply(i), (succ, e) -> {
        });
    }
    latch.await();
    assertEquals(FixedWatermarkEstimatorFactory.FIXED_WATERMARK, watermark.get());
}