Examples with CommitLogReader cz.o2.proxima.direct.commitlog.CommitLogReader used on opensource projects

Example 41 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class ReplicationControllerTest method testBulkReplication.

@Test
public void testBulkReplication() throws InterruptedException {
    final List<StreamElement> written = new ArrayList<>();
    final CommitLogReader reader = Optionals.get(direct.getCommitLogReader(data));
    final CommitLogObserver observer = controller.createBulkObserver("consumer", reader, Sets.newHashSet(data), new PassthroughFilter(), fakeBulkWriter(written, stamp -> stamp == now - InMemStorage.getBoundedOutOfOrderness() + 100));
    try (ObserveHandle ignored = reader.observeBulk("consumer", observer)) {
        for (int i = 0; i < 10; i++) {
            writeEvent(now + 20 * i);
        }
        assertEquals(6, written.size());
    }
    livenessLatch.await();
    assertEquals(1.0, Metrics.LIVENESS.getValue(), 0.0001);
}

Example 42 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class ReplicationControllerTest method testSimpleEventReplication.

@Test(timeout = 5000)
public void testSimpleEventReplication() throws InterruptedException {
    List<StreamElement> written = new ArrayList<>();
    final CommitLogReader reader = Optionals.get(direct.getCommitLogReader(data));
    final CommitLogObserver observer = controller.createOnlineObserver("consumer", direct.getCommitLogReader(data).orElseThrow(() -> new IllegalArgumentException("Missing commit log reader for data")), Sets.newHashSet(data), new PassthroughFilter(), fakeOnlineWriter(written));
    try (ObserveHandle ignored = reader.observe("consumer", observer)) {
        writeEvent();
        assertEquals(1, written.size());
    }
    livenessLatch.await();
    assertEquals(1.0, Metrics.LIVENESS.getValue(), 0.0001);
}

Example 43 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class InMemStorage method createAccessor.

@Override
public DataAccessor createAccessor(DirectDataOperator op, AttributeFamilyDescriptor familyDescriptor) {
    final EntityDescriptor entity = familyDescriptor.getEntity();
    final URI uri = familyDescriptor.getStorageUri();
    final Map<String, Object> cfg = familyDescriptor.getCfg();
    log.info("Creating accessor {} for URI {}", getClass(), uri);
    holder().observers.computeIfAbsent(uri, k -> Collections.synchronizedNavigableMap(new TreeMap<>()));
    final int numPartitions = Optional.ofNullable(cfg.get(NUM_PARTITIONS)).map(v -> Integer.parseInt(v.toString())).orElse(1);
    final Partitioner partitioner = Optional.ofNullable(cfg.get(ConfigConstants.PARTITIONER)).map(name -> Classpath.newInstance(name.toString(), Partitioner.class)).orElseGet(KeyAttributePartitioner::new);
    final Repository opRepo = op.getRepository();
    final RepositoryFactory repositoryFactory = opRepo.asFactory();
    final OnlineAttributeWriter.Factory<?> writerFactory = new Writer(entity, uri, numPartitions, partitioner).asFactory();
    final CommitLogReader.Factory<?> commitLogReaderFactory = new InMemCommitLogReader(entity, uri, op.getContext().getExecutorFactory(), partitioner, numPartitions).asFactory();
    final RandomAccessReader.Factory<Reader> randomAccessReaderFactory;
    final BatchLogReader.Factory<Reader> batchLogReaderFactory;
    final CachedView.Factory cachedViewFactory;
    if (numPartitions > 1) {
        randomAccessReaderFactory = null;
        batchLogReaderFactory = null;
        cachedViewFactory = null;
    } else {
        final ReaderFactory readerFactory = new Reader(entity, uri, op.getContext().getExecutorFactory()).asFactory();
        randomAccessReaderFactory = readerFactory;
        batchLogReaderFactory = readerFactory;
        cachedViewFactory = new LocalCachedPartitionedView(entity, commitLogReaderFactory.apply(opRepo), writerFactory.apply(opRepo)).asFactory();
    }
    return new DataAccessor() {

        private static final long serialVersionUID = 1L;

        @Nullable
        private transient Repository repo = opRepo;

        @Override
        public URI getUri() {
            return uri;
        }

        @Override
        public Optional<AttributeWriterBase> getWriter(Context context) {
            Objects.requireNonNull(context);
            return Optional.of(writerFactory.apply(repo()));
        }

        @Override
        public Optional<CommitLogReader> getCommitLogReader(Context context) {
            Objects.requireNonNull(context);
            return Optional.of(commitLogReaderFactory.apply(repo()));
        }

        @Override
        public Optional<RandomAccessReader> getRandomAccessReader(Context context) {
            Objects.requireNonNull(context);
            return Optional.ofNullable(randomAccessReaderFactory).map(item -> item.apply(repo()));
        }

        @Override
        public Optional<CachedView> getCachedView(Context context) {
            Objects.requireNonNull(context);
            return Optional.ofNullable(cachedViewFactory).map(item -> item.apply(repo()));
        }

        @Override
        public Optional<BatchLogReader> getBatchLogReader(Context context) {
            Objects.requireNonNull(context);
            return Optional.ofNullable(batchLogReaderFactory).map(item -> item.apply(repo()));
        }

        private Repository repo() {
            if (this.repo == null) {
                this.repo = repositoryFactory.apply();
            }
            return this.repo;
        }
    };
}

Example 44 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class ReplicationController method runTransform.

private void runTransform(String name, TransformationDescriptor transform, DirectAttributeFamilyDescriptor family) {
    final StorageFilter filter = transform.getFilter();
    final String consumer = transform.getConsumerNameFactory().apply();
    final CommitLogReader reader = family.getCommitLogReader().orElseThrow(() -> new IllegalStateException("Unable to get reader for family " + family.getDesc().getName() + "."));
    final TransformationObserver observer;
    if (transform.getTransformation().isContextual()) {
        DirectElementWiseTransform transformation = transform.getTransformation().as(DirectElementWiseTransform.class);
        observer = contextualObserver(dataOperator, name, transformation, transform.getOutputTransactionMode() == OutputTransactionMode.ENABLED, filter);
    } else {
        ElementWiseTransformation transformation = transform.getTransformation().asElementWiseTransform();
        observer = nonContextualObserver(dataOperator, name, transformation, transform.getOutputTransactionMode() == OutputTransactionMode.ENABLED, filter);
    }
    startTransformationObserverUsing(consumer, reader, observer);
    log.info("Started transformer {} reading from {} using {}", consumer, reader.getUri(), transform.getTransformation().getClass());
}

Example 45 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveNonExternalizableWatermark.

@Test
public void testObserveNonExternalizableWatermark() throws InterruptedException {
    int numElements = 10;
    CommitLogReader reader = ListCommitLog.ofNonExternalizable(data(numElements), direct.getContext());
    CountDownLatch first = new CountDownLatch(1);
    List<Long> watermarks = new ArrayList<>();
    ObserveHandle handle = reader.observeBulk(null, new CommitLogObserver() {

        int received = 0;

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            watermarks.add(context.getWatermark());
            if (++received == numElements) {
                context.confirm();
            }
            return true;
        }

        @Override
        public void onCompleted() {
            first.countDown();
        }
    });
    first.await();
    assertEquals(numElements, watermarks.size());
    long min = watermarks.get(0);
    for (int i = 1; i < numElements; i++) {
        assertEquals(min, (long) watermarks.get(i));
    }
}