Examples with CommitLogReader cz.o2.proxima.direct.commitlog.CommitLogReader used on opensource projects

Example 61 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class ReplicationController method consumeLog.

private void consumeLog(DirectAttributeFamilyDescriptor primaryFamily, DirectAttributeFamilyDescriptor replicaFamily) {
    final CommitLogReader commitLog = primaryFamily.getCommitLogReader().orElseThrow(() -> new IllegalStateException(String.format("Failed to find commit-log reader in family %s.", primaryFamily)));
    final AttributeWriterBase writer = replicaFamily.getWriter().orElseThrow(() -> new IllegalStateException(String.format("Unable to get writer for family %s.", replicaFamily.getDesc().getName())));
    final StorageFilter filter = replicaFamily.getDesc().getFilter();
    final Set<AttributeDescriptor<?>> allowedAttributes = new HashSet<>(replicaFamily.getAttributes());
    final String name = replicaFamily.getDesc().getReplicationConsumerNameFactory().apply();
    log.info("Using consumer name {} to replicate family {}", name, replicaFamily.getDesc().getName());
    registerWriterTo(name, commitLog, allowedAttributes, filter, writer);
    log.info("Started consumer {} consuming from log {} with URI {} into {} attributes {}", name, commitLog, commitLog.getUri(), writer.getUri(), allowedAttributes);
}

Example 62 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class ReplicationControllerTest method testSimpleEventReplicationWithFilter.

@Test
public void testSimpleEventReplicationWithFilter() {
    final List<StreamElement> written = new ArrayList<>();
    final CommitLogReader reader = Optionals.get(direct.getCommitLogReader(data));
    final CommitLogObserver observer = controller.createOnlineObserver("consumer", direct.getCommitLogReader(data).orElseThrow(() -> new IllegalArgumentException("Missing commit log reader for data")), Sets.newHashSet(data), ingest -> false, fakeOnlineWriter(written));
    try (ObserveHandle ignored = reader.observe("consumer", observer)) {
        writeEvent();
        assertEquals(0, written.size());
    }
}

Example 63 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class SingleTopicMultipleReplicationsTest method testProxyReadWildcardReplicated.

@Test(timeout = 5000)
public void testProxyReadWildcardReplicated() throws InterruptedException {
    Optional<AttributeFamilyDescriptor> wildcardPrimary = repo.getFamiliesForAttribute(wildcard).stream().filter(af -> af.getType() == StorageType.PRIMARY).findFirst();
    assertTrue(wildcardPrimary.isPresent());
    AttributeFamilyDescriptor family = wildcardPrimary.get();
    assertEquals("proxy::proxy::replication_wildcard-replication_replicated::" + "replication_wildcard-replication_write::proxy::" + "replication_wildcard-replication_replicated::" + "replication_wildcard-replication_write", family.getName());
    assertTrue(family.getAccess().canReadCommitLog());
    CommitLogReader reader = direct.resolveRequired(family).getCommitLogReader().orElseThrow(() -> new IllegalStateException("Missing commit-log in " + family));
    CountDownLatch latch = new CountDownLatch(1);
    reader.observe("dummy", new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            assertEquals(wildcard, ingest.getAttributeDescriptor());
            assertEquals("wildcard.1", ingest.getAttribute());
            latch.countDown();
            context.confirm();
            return false;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    }).waitUntilReady();
    OnlineAttributeWriter writer = direct.getWriter(wildcardInput).orElseThrow(() -> new IllegalStateException("Missing writer for " + wildcardInput));
    writer.write(StreamElement.upsert(entity, wildcard, uuid(), "key", wildcard.toAttributePrefix() + "1", now, value()), (succ, exc) -> {
    });
    assertTrue(latch.await(1, TimeUnit.SECONDS));
}

Example 64 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class SingleTopicMultipleReplicationsTest method testProxyObserveBulkPartitionsWildcardReplicated.

@Test(timeout = 5000)
public void testProxyObserveBulkPartitionsWildcardReplicated() throws InterruptedException {
    Optional<AttributeFamilyDescriptor> wildcardPrimary = repo.getFamiliesForAttribute(wildcard).stream().filter(af -> af.getType() == StorageType.PRIMARY).findFirst();
    assertTrue(wildcardPrimary.isPresent());
    AttributeFamilyDescriptor family = wildcardPrimary.get();
    assertTrue(family.getAccess().canReadCommitLog());
    CommitLogReader reader = direct.resolveRequired(family).getCommitLogReader().orElseThrow(() -> new IllegalStateException("Missing commit-log in " + family));
    CountDownLatch latch = new CountDownLatch(1);
    reader.observeBulkPartitions(reader.getPartitions(), Position.CURRENT, new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            assertEquals(wildcard, ingest.getAttributeDescriptor());
            assertEquals("wildcard.1", ingest.getAttribute());
            latch.countDown();
            return false;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    }).waitUntilReady();
    OnlineAttributeWriter writer = direct.getWriter(wildcardInput).orElseThrow(() -> new IllegalStateException("Missing writer for " + wildcardInput));
    writer.write(StreamElement.upsert(entity, wildcard, uuid(), "key", wildcard.toAttributePrefix() + "1", now, value()), (succ, exc) -> {
    });
    assertTrue(latch.await(1, TimeUnit.SECONDS));
}

Example 65 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class ReplicationRunner method runAttributeReplicas.

/**
 * Run replications of attributes to replica attribute families.
 *
 * @param direct {@link DirectDataOperator} direct data operator
 * @param onReplicated callback called for each replicated element
 */
public static void runAttributeReplicas(DirectDataOperator direct, Consumer<StreamElement> onReplicated) {
    direct.getAllFamilies().filter(af -> af.getDesc().getType() == StorageType.REPLICA).filter(af -> !af.getDesc().getAccess().isReadonly() && !af.getDesc().isProxy()).forEach(af -> {
        List<AttributeDescriptor<?>> attributes = af.getAttributes();
        final AttributeWriterBase writer = Optionals.get(af.getWriter());
        final CommitLogReader primaryCommitLogReader = Optionals.get(attributes.stream().flatMap(a -> direct.getFamiliesForAttribute(a).stream()).filter(f -> f.getDesc().getType() == StorageType.PRIMARY).distinct().findFirst().flatMap(DirectAttributeFamilyDescriptor::getCommitLogReader));
        final ObserveHandle handle;
        if (writer instanceof OnlineAttributeWriter) {
            final OnlineAttributeWriter onlineWriter = writer.online();
            handle = primaryCommitLogReader.observe(af.getDesc().getName(), (CommitLogObserver) (ingest, context) -> {
                log.debug("Replicating input {} to {}", ingest, writer);
                onlineWriter.write(ingest, (succ, exc) -> {
                    context.commit(succ, exc);
                    onReplicated.accept(ingest);
                });
                return true;
            });
        } else {
            final BulkAttributeWriter bulkWriter = writer.bulk();
            handle = primaryCommitLogReader.observe(af.getDesc().getName(), (CommitLogObserver) (ingest, context) -> {
                log.debug("Replicating input {} to {}", ingest, writer);
                bulkWriter.write(ingest, context.getWatermark(), (succ, exc) -> {
                    context.commit(succ, exc);
                    onReplicated.accept(ingest);
                });
                return true;
            });
        }
        ExceptionUtils.unchecked(handle::waitUntilReady);
        log.info("Started attribute replica {}", af.getDesc().getName());
    });
}