Examples with CommitLogReader cz.o2.proxima.direct.commitlog.CommitLogReader used on opensource projects

Example 66 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class DirectDataOperatorTest method testWriteIntoReplicatedProxyAttribute.

@Test(timeout = 10000)
public void testWriteIntoReplicatedProxyAttribute() throws InterruptedException {
    repo.reloadConfig(true, ConfigFactory.load().withFallback(ConfigFactory.load("test-replication.conf")).withFallback(ConfigFactory.load("test-reference.conf")).resolve());
    EntityDescriptor dummy = repo.getEntity("dummy");
    AttributeDescriptor<Object> data = dummy.getAttribute("data", true);
    AttributeDescriptor<Object> dataReplicated = dummy.getAttribute("_dummyReplicationProxiedSlave_replicated$_d", true);
    CountDownLatch latch = new CountDownLatch(2);
    CommitLogReader reader = direct.getCommitLogReader(data).orElseThrow(() -> new IllegalStateException("Missing commit log reader for data"));
    reader.observe("dummy", new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            assertEquals(ingest.getAttributeDescriptor(), data);
            latch.countDown();
            return true;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    OnlineAttributeWriter writer = direct.getWriter(dataReplicated).get();
    writer.write(StreamElement.upsert(dummy, dataReplicated, "uuid", "gw", dataReplicated.getName(), System.currentTimeMillis(), new byte[] { 1, 2 }), (succ, exc) -> {
        assertTrue(succ);
        latch.countDown();
    });
    latch.await();
}

Example 67 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class DirectDataOperatorTest method testReplicationWriteReadonlyObserve.

@SuppressWarnings("unchecked")
@Test(timeout = 10000)
public void testReplicationWriteReadonlyObserve() throws InterruptedException {
    repo.reloadConfig(true, ConfigFactory.load().withFallback(ConfigFactory.parseString("replications.gateway-replication.read-only = true")).withFallback(ConfigFactory.load("test-replication.conf")).withFallback(ConfigFactory.load("test-reference.conf")).resolve());
    EntityDescriptor gateway = repo.getEntity("gateway");
    AttributeDescriptor<Object> armed = gateway.getAttribute("armed");
    TimeUnit.MILLISECONDS.sleep(300);
    CommitLogReader reader = direct.getFamiliesForAttribute(armed).stream().filter(af -> af.getDesc().getAccess().canReadCommitLog()).findAny().flatMap(DirectAttributeFamilyDescriptor::getCommitLogReader).orElseThrow(() -> new IllegalStateException("Missing random access reader for armed"));
    CountDownLatch latch = new CountDownLatch(1);
    reader.observe("dummy", new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            assertEquals(ingest.getAttributeDescriptor(), armed);
            latch.countDown();
            context.confirm();
            return true;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    assertTrue(direct.getWriter(armed).isPresent());
    OnlineAttributeWriter writer = direct.getWriter(armed).get();
    writer.write(StreamElement.upsert(gateway, armed, "uuid", "gw", armed.getName(), System.currentTimeMillis(), new byte[] { 1, 2 }), (succ, exc) -> {
        assertTrue(succ);
    });
    latch.await();
}

Example 68 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class DirectDataOperatorTest method testLimiterDataAccessor.

@Test
public void testLimiterDataAccessor() {
    repo.reloadConfig(true, ConfigFactory.load("test-limiter.conf").withFallback(ConfigFactory.load("test-reference.conf")).resolve());
    DirectDataOperator direct = repo.getOrCreateOperator(DirectDataOperator.class);
    DirectAttributeFamilyDescriptor family = direct.getFamilyByName("event-storage-stream");
    Optional<CommitLogReader> maybeReader = family.getCommitLogReader();
    assertTrue(maybeReader.isPresent());
    CommitLogReader reader = maybeReader.get();
    assertTrue("Expected reader of class LimitedCommitLogReader, got " + reader.getClass(), reader instanceof LimitedCommitLogReader);
    LimitedCommitLogReader limitedReader = (LimitedCommitLogReader) reader;
    ThroughputLimiter limiter = limitedReader.getLimiter();
    assertNotNull(limiter);
    assertTrue(limiter instanceof GlobalWatermarkThroughputLimiter);
    GlobalWatermarkThroughputLimiter watermarkLimiter = (GlobalWatermarkThroughputLimiter) limiter;
    GlobalWatermarkTracker tracker = watermarkLimiter.getTracker();
    assertTrue(tracker instanceof TestTracker);
    TestTracker testTracker = (TestTracker) tracker;
    assertEquals(2, testTracker.getTestConf());
}

Example 69 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class CommitLogReaderTest method testThroughputLimitedCommitLogIdles.

@Test(timeout = 10000)
public void testThroughputLimitedCommitLogIdles() throws InterruptedException {
    ThroughputLimiter limiter = ThroughputLimiter.NoOpThroughputLimiter.INSTANCE;
    CommitLogReader limitedReader = CommitLogReaders.withThroughputLimit(reader, limiter);
    UnaryFunction<CountDownLatch, CommitLogObserver> observerFactory = myLatch -> new CommitLogObserver() {

        @Override
        public boolean onError(Throwable error) {
            return false;
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            return false;
        }

        @Override
        public void onIdle(OnIdleContext context) {
            myLatch.countDown();
        }
    };
    CountDownLatch latch = new CountDownLatch(1);
    CommitLogObserver observer = observerFactory.apply(latch);
    long nanoTime = System.nanoTime();
    ObserveHandle handle = limitedReader.observe("dummy", observer);
    latch.await();
    handle.close();
    latch = new CountDownLatch(1);
    observer = observerFactory.apply(latch);
    long durationMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - nanoTime);
    handle.close();
    limitedReader = CommitLogReaders.withThroughputLimit(reader, withNumRecordsPerSec(1));
    handle = limitedReader.observe("dummy", observer);
    // no idle called this time
    assertFalse(latch.await(2 * durationMillis, TimeUnit.MILLISECONDS));
}

Example 70 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class CommitLogReaderTest method testThroughputLimitedCommitLogWithObserve.

private void testThroughputLimitedCommitLogWithObserve(BiFunction<CommitLogReader, Runnable, ObserveHandle> observe) throws InterruptedException {
    final int numElements = 50;
    ThroughputLimiter limiter = withNumRecordsPerSec(2 * numElements);
    CommitLogReader limitedReader = CommitLogReaders.withThroughputLimit(reader, limiter);
    assertEquals(limitedReader.getPartitions(), reader.getPartitions());
    assertEquals(limitedReader.getUri(), reader.getUri());
    CountDownLatch latch = new CountDownLatch(numElements);
    AtomicLong lastMillis = new AtomicLong(System.currentTimeMillis());
    List<Long> durations = new ArrayList<>();
    ObserveHandle handle = observe.apply(limitedReader, () -> {
        latch.countDown();
        long now = System.currentTimeMillis();
        long lastGet = lastMillis.getAndUpdate(current -> now);
        durations.add(now - lastGet);
    });
    for (int i = 0; i < 50; i++) {
        writer.online().write(StreamElement.upsert(entity, attr, UUID.randomUUID().toString(), "key", attr.getName(), System.currentTimeMillis(), new byte[] { 1, 2 }), (succ, exc) -> {
        });
    }
    latch.await();
    assertEquals(numElements, durations.size());
    assertEquals(durations.toString(), numElements, durations.stream().filter(s -> s >= 500 / numElements).count());
    handle.close();
}