Examples with ThroughputLimiter cz.o2.proxima.storage.ThroughputLimiter used on opensource projects

Example 1 with ThroughputLimiter

use of cz.o2.proxima.storage.ThroughputLimiter in project proxima-platform by O2-Czech-Republic.

the class CommitLogReaderTest method withNumRecordsPerSec.

public static ThroughputLimiter withNumRecordsPerSec(int recordsPerSec) {
    final Duration pauseDuration = Duration.ofMillis(1000L / recordsPerSec);
    return new ThroughputLimiter() {

        @Override
        public Duration getPauseTime(Context context) {
            assertEquals(1, context.getConsumedPartitions().size());
            assertEquals(0, Iterables.getOnlyElement(context.getConsumedPartitions()).getId());
            assertTrue(context.getMinWatermark() < Long.MAX_VALUE);
            return pauseDuration;
        }

        @Override
        public void close() {
        }
    };
}

Example 2 with ThroughputLimiter

use of cz.o2.proxima.storage.ThroughputLimiter in project proxima-platform by O2-Czech-Republic.

the class CommitLogReaderTest method testThroughputLimitedCommitLogReaderAsFactory.

@Test
public void testThroughputLimitedCommitLogReaderAsFactory() {
    ThroughputLimiter limiter = ThroughputLimiter.NoOpThroughputLimiter.INSTANCE;
    CommitLogReader limitedReader = CommitLogReaders.withThroughputLimit(reader, limiter);
    CommitLogReader cloned = limitedReader.asFactory().apply(repo);
    assertTrue(limitedReader.getClass().isAssignableFrom(cloned.getClass()));
}

Example 3 with ThroughputLimiter

use of cz.o2.proxima.storage.ThroughputLimiter in project proxima-platform by O2-Czech-Republic.

the class DirectDataOperatorTest method testLimiterDataAccessor.

@Test
public void testLimiterDataAccessor() {
    repo.reloadConfig(true, ConfigFactory.load("test-limiter.conf").withFallback(ConfigFactory.load("test-reference.conf")).resolve());
    DirectDataOperator direct = repo.getOrCreateOperator(DirectDataOperator.class);
    DirectAttributeFamilyDescriptor family = direct.getFamilyByName("event-storage-stream");
    Optional<CommitLogReader> maybeReader = family.getCommitLogReader();
    assertTrue(maybeReader.isPresent());
    CommitLogReader reader = maybeReader.get();
    assertTrue("Expected reader of class LimitedCommitLogReader, got " + reader.getClass(), reader instanceof LimitedCommitLogReader);
    LimitedCommitLogReader limitedReader = (LimitedCommitLogReader) reader;
    ThroughputLimiter limiter = limitedReader.getLimiter();
    assertNotNull(limiter);
    assertTrue(limiter instanceof GlobalWatermarkThroughputLimiter);
    GlobalWatermarkThroughputLimiter watermarkLimiter = (GlobalWatermarkThroughputLimiter) limiter;
    GlobalWatermarkTracker tracker = watermarkLimiter.getTracker();
    assertTrue(tracker instanceof TestTracker);
    TestTracker testTracker = (TestTracker) tracker;
    assertEquals(2, testTracker.getTestConf());
}

Example 4 with ThroughputLimiter

use of cz.o2.proxima.storage.ThroughputLimiter in project proxima-platform by O2-Czech-Republic.

the class CommitLogReaderTest method testThroughputLimitedCommitLogIdles.

@Test(timeout = 10000)
public void testThroughputLimitedCommitLogIdles() throws InterruptedException {
    ThroughputLimiter limiter = ThroughputLimiter.NoOpThroughputLimiter.INSTANCE;
    CommitLogReader limitedReader = CommitLogReaders.withThroughputLimit(reader, limiter);
    UnaryFunction<CountDownLatch, CommitLogObserver> observerFactory = myLatch -> new CommitLogObserver() {

        @Override
        public boolean onError(Throwable error) {
            return false;
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            return false;
        }

        @Override
        public void onIdle(OnIdleContext context) {
            myLatch.countDown();
        }
    };
    CountDownLatch latch = new CountDownLatch(1);
    CommitLogObserver observer = observerFactory.apply(latch);
    long nanoTime = System.nanoTime();
    ObserveHandle handle = limitedReader.observe("dummy", observer);
    latch.await();
    handle.close();
    latch = new CountDownLatch(1);
    observer = observerFactory.apply(latch);
    long durationMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - nanoTime);
    handle.close();
    limitedReader = CommitLogReaders.withThroughputLimit(reader, withNumRecordsPerSec(1));
    handle = limitedReader.observe("dummy", observer);
    // no idle called this time
    assertFalse(latch.await(2 * durationMillis, TimeUnit.MILLISECONDS));
}

Example 5 with ThroughputLimiter

use of cz.o2.proxima.storage.ThroughputLimiter in project proxima-platform by O2-Czech-Republic.

the class CommitLogReaderTest method testThroughputLimitedCommitLogWithObserve.

private void testThroughputLimitedCommitLogWithObserve(BiFunction<CommitLogReader, Runnable, ObserveHandle> observe) throws InterruptedException {
    final int numElements = 50;
    ThroughputLimiter limiter = withNumRecordsPerSec(2 * numElements);
    CommitLogReader limitedReader = CommitLogReaders.withThroughputLimit(reader, limiter);
    assertEquals(limitedReader.getPartitions(), reader.getPartitions());
    assertEquals(limitedReader.getUri(), reader.getUri());
    CountDownLatch latch = new CountDownLatch(numElements);
    AtomicLong lastMillis = new AtomicLong(System.currentTimeMillis());
    List<Long> durations = new ArrayList<>();
    ObserveHandle handle = observe.apply(limitedReader, () -> {
        latch.countDown();
        long now = System.currentTimeMillis();
        long lastGet = lastMillis.getAndUpdate(current -> now);
        durations.add(now - lastGet);
    });
    for (int i = 0; i < 50; i++) {
        writer.online().write(StreamElement.upsert(entity, attr, UUID.randomUUID().toString(), "key", attr.getName(), System.currentTimeMillis(), new byte[] { 1, 2 }), (succ, exc) -> {
        });
    }
    latch.await();
    assertEquals(numElements, durations.size());
    assertEquals(durations.toString(), numElements, durations.stream().filter(s -> s >= 500 / numElements).count());
    handle.close();
}