Examples with CommitLogReader cz.o2.proxima.direct.commitlog.CommitLogReader used on opensource projects

Example 36 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class DirectDataOperatorTest method testReplicationWriteObserveInternal.

private void testReplicationWriteObserveInternal(Config config, boolean localWrite, boolean expectNonEmpty) throws InterruptedException {
    repo.reloadConfig(true, config);
    EntityDescriptor gateway = repo.getEntity("gateway");
    AttributeDescriptor<Object> armed = gateway.getAttribute("armed");
    AttributeDescriptor<Object> armedWrite = gateway.getAttribute(localWrite ? "_gatewayReplication_write$armed" : "_gatewayReplication_replicated$armed", true);
    // observe stream
    CommitLogReader reader = direct.getFamiliesForAttribute(armed).stream().filter(af -> af.getDesc().getAccess().canReadCommitLog()).findAny().flatMap(DirectAttributeFamilyDescriptor::getCommitLogReader).orElseThrow(() -> new IllegalStateException("Missing commit log reader for armed"));
    List<StreamElement> observed = new ArrayList<>();
    reader.observe("dummy", new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            if (!expectNonEmpty) {
                fail("No input was expected.");
            }
            observed.add(ingest);
            return true;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    // start replications
    TransformationRunner.runTransformations(repo, direct);
    assertTrue(direct.getWriter(armedWrite).isPresent());
    OnlineAttributeWriter writer = direct.getWriter(armedWrite).get();
    writer.write(StreamElement.upsert(gateway, armedWrite, "uuid", "gw", armedWrite.getName(), System.currentTimeMillis(), new byte[] { 1, 2 }), (succ, exc) -> {
        assertTrue(succ);
    });
    // wait till write propagates
    TimeUnit.MILLISECONDS.sleep(300);
    if (expectNonEmpty) {
        assertEquals(1, observed.size());
        assertEquals(armed, observed.get(0).getAttributeDescriptor());
    }
}

Example 37 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class DirectDataOperatorTest method testIncomingReplicationDoesntCycle.

@Test(timeout = 10000)
public void testIncomingReplicationDoesntCycle() throws InterruptedException {
    repo.reloadConfig(true, ConfigFactory.load().withFallback(ConfigFactory.load("test-replication.conf")).withFallback(ConfigFactory.load("test-reference.conf")).resolve());
    final EntityDescriptor gateway = repo.getEntity("gateway");
    final AttributeDescriptor<Object> status = gateway.getAttribute("status");
    final AttributeDescriptor<Object> statusRead = gateway.getAttribute("_gatewayReplication_read$status", true);
    final AttributeDescriptor<Object> statusWrite = gateway.getAttribute("_gatewayReplication_write$status", true);
    TransformationRunner.runTransformations(repo, direct);
    CommitLogReader reader = direct.getCommitLogReader(status).orElseThrow(() -> new IllegalStateException("Missing random access reader for " + status.getName()));
    CountDownLatch latch = new CountDownLatch(1);
    reader.observe("dummy", new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            assertEquals(ingest.getAttributeDescriptor(), status);
            latch.countDown();
            context.confirm();
            return true;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    assertTrue(direct.getWriter(statusRead).isPresent());
    OnlineAttributeWriter writer = direct.getWriter(statusRead).get();
    writer.write(StreamElement.upsert(gateway, status, "uuid", "gw", status.getName(), System.currentTimeMillis(), new byte[] { 1, 2 }), (succ, exc) -> {
        assertTrue(succ);
    });
    latch.await();
    RandomAccessReader localReader = direct.getFamiliesForAttribute(statusWrite).stream().filter(af -> af.getDesc().getType() == StorageType.PRIMARY).findAny().flatMap(DirectAttributeFamilyDescriptor::getRandomAccessReader).orElseThrow(() -> new IllegalStateException("Missing primary random access family for status write"));
    assertFalse(localReader.get("gw", statusWrite).isPresent());
}

Example 38 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class DirectDataOperatorTest method testObserveReplicatedWithProxy.

@Test(timeout = 10000)
public void testObserveReplicatedWithProxy() throws InterruptedException {
    repo.reloadConfig(true, ConfigFactory.load().withFallback(ConfigFactory.load("test-replication.conf")).withFallback(ConfigFactory.load("test-reference.conf")).resolve());
    final EntityDescriptor dummy = repo.getEntity("dummy");
    final AttributeDescriptor<Object> data = dummy.getAttribute("data");
    final AttributeDescriptor<Object> dataRead = dummy.getAttribute("_dummyReplicationProxiedSlave_read$data", true);
    final AttributeDescriptor<Object> dataWrite = dummy.getAttribute("_dummyReplicationProxiedSlave_write$_d", true);
    TransformationRunner.runTransformations(repo, direct);
    CommitLogReader reader = direct.getCommitLogReader(data).orElseThrow(() -> new IllegalStateException("Missing commit log reader for " + data.getName()));
    CountDownLatch latch = new CountDownLatch(1);
    reader.observe("dummy", new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            assertEquals(ingest.getAttributeDescriptor(), data);
            latch.countDown();
            context.confirm();
            return true;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    OnlineAttributeWriter writer = direct.getWriter(dataRead).get();
    writer.write(StreamElement.upsert(dummy, data, "uuid", "gw", data.getName(), System.currentTimeMillis(), new byte[] { 1, 2 }), (succ, exc) -> {
        assertTrue(succ);
    });
    latch.await();
    assertFalse(direct.getFamiliesForAttribute(dataWrite).stream().filter(af -> af.getDesc().getType() == StorageType.PRIMARY).findAny().flatMap(DirectAttributeFamilyDescriptor::getRandomAccessReader).orElseThrow(() -> new IllegalStateException("Missing random access for " + dataWrite)).get("gw", dataWrite).isPresent());
}

Example 39 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class DirectDataOperatorTest method testProxyObserveBulk.

@Test
public void testProxyObserveBulk() throws InterruptedException {
    EntityDescriptor proxied = repo.getEntity("proxied");
    AttributeDescriptor<?> source = proxied.getAttribute("event.*");
    CommitLogReader reader = direct.getFamiliesForAttribute(source).stream().filter(af -> af.getDesc().getAccess().canReadCommitLog()).findAny().flatMap(DirectAttributeFamilyDescriptor::getCommitLogReader).get();
    List<StreamElement> read = new ArrayList<>();
    reader.observeBulk("dummy", new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            read.add(ingest);
            context.confirm();
            return true;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    }).waitUntilReady();
    assertTrue(direct.getWriter(source).isPresent());
    direct.getWriter(source).get().write(StreamElement.upsert(proxied, source, UUID.randomUUID().toString(), "key", "event.abc", System.currentTimeMillis(), "test".getBytes(StandardCharsets.UTF_8)), (s, exc) -> {
        assertTrue(s);
    });
    direct.getWriter(source).get().write(StreamElement.upsert(proxied, source, UUID.randomUUID().toString(), "key", "event.def", System.currentTimeMillis(), "test2".getBytes(StandardCharsets.UTF_8)), (s, exc) -> {
        assertTrue(s);
    });
    assertEquals(2, read.size());
    assertEquals("test", new String((byte[]) read.get(0).getValue()));
    assertEquals(source, read.get(0).getAttributeDescriptor());
    assertEquals("event.abc", read.get(0).getAttribute());
    assertEquals("key", read.get(0).getKey());
    assertEquals("test2", new String((byte[]) read.get(1).getValue()));
    assertEquals(source, read.get(1).getAttributeDescriptor());
    assertEquals("event.def", read.get(1).getAttribute());
    assertEquals("key", read.get(1).getKey());
}

Example 40 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class DirectDataOperatorTest method testReplicationWriteReadonlyObserveLocal.

@SuppressWarnings("unchecked")
@Test(timeout = 10000)
public void testReplicationWriteReadonlyObserveLocal() throws InterruptedException {
    repo.reloadConfig(true, ConfigFactory.load().withFallback(ConfigFactory.parseString("replications.gateway-replication.read-only = true")).withFallback(ConfigFactory.parseString("replications.gateway-replication.read = local")).withFallback(ConfigFactory.load("test-replication.conf")).withFallback(ConfigFactory.load("test-reference.conf")));
    EntityDescriptor gateway = repo.getEntity("gateway");
    AttributeDescriptor<Object> armed = gateway.getAttribute("armed");
    TimeUnit.MILLISECONDS.sleep(300);
    CommitLogReader reader = direct.getCommitLogReader(armed).orElseThrow(() -> new IllegalStateException("Missing random access reader for armed"));
    CountDownLatch latch = new CountDownLatch(1);
    reader.observe("dummy", new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            assertEquals(ingest.getAttributeDescriptor(), armed);
            latch.countDown();
            context.confirm();
            return true;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    assertTrue(direct.getWriter(armed).isPresent());
    OnlineAttributeWriter writer = direct.getWriter(armed).get();
    writer.write(StreamElement.upsert(gateway, armed, "uuid", "gw", armed.getName(), System.currentTimeMillis(), new byte[] { 1, 2 }), (succ, exc) -> {
        assertTrue(succ);
    });
    latch.await();
}