Search in sources :

Example 1 with LocalCachedPartitionedView

use of cz.o2.proxima.direct.view.LocalCachedPartitionedView in project proxima-platform by O2-Czech-Republic.

the class LocalCachedPartitionedView method asFactory.

@Override
public Factory asFactory() {
    final CommitLogReader.Factory<?> readerFactory = reader.asFactory();
    final OnlineAttributeWriter.Factory<?> writerFactory = writer.asFactory();
    final EntityDescriptor entity = this.entity;
    return repo -> new LocalCachedPartitionedView(entity, readerFactory.apply(repo), writerFactory.apply(repo));
}
Also used : Consumer(cz.o2.proxima.functional.Consumer) Partition(cz.o2.proxima.storage.Partition) EntityDescriptor(cz.o2.proxima.repository.EntityDescriptor) OnlineAttributeWriter(cz.o2.proxima.direct.core.OnlineAttributeWriter) AtomicReference(java.util.concurrent.atomic.AtomicReference) Value(lombok.Value) ExceptionUtils(cz.o2.proxima.util.ExceptionUtils) StreamElement(cz.o2.proxima.storage.StreamElement) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Pair(cz.o2.proxima.util.Pair) RawOffset(cz.o2.proxima.direct.randomaccess.RawOffset) Duration(java.time.Duration) URI(java.net.URI) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader) Nullable(javax.annotation.Nullable) Payload(cz.o2.proxima.direct.view.TimeBoundedVersionedCache.Payload) SynchronousQueue(java.util.concurrent.SynchronousQueue) Collection(java.util.Collection) AttributeDescriptor(cz.o2.proxima.repository.AttributeDescriptor) CommitLogObserver(cz.o2.proxima.direct.commitlog.CommitLogObserver) BlockingQueue(java.util.concurrent.BlockingQueue) ObserveHandle(cz.o2.proxima.direct.commitlog.ObserveHandle) CommitCallback(cz.o2.proxima.direct.core.CommitCallback) Collectors(java.util.stream.Collectors) BiConsumer(cz.o2.proxima.functional.BiConsumer) Offset(cz.o2.proxima.direct.commitlog.Offset) Objects(java.util.Objects) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) KeyValue(cz.o2.proxima.direct.randomaccess.KeyValue) Optional(java.util.Optional) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) RandomOffset(cz.o2.proxima.direct.randomaccess.RandomOffset) Position(cz.o2.proxima.storage.commitlog.Position) EntityDescriptor(cz.o2.proxima.repository.EntityDescriptor) OnlineAttributeWriter(cz.o2.proxima.direct.core.OnlineAttributeWriter) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader)

Example 2 with LocalCachedPartitionedView

use of cz.o2.proxima.direct.view.LocalCachedPartitionedView in project proxima-platform by O2-Czech-Republic.

the class DirectAttributeFamilyProxyDescriptor method getPartitionedCachedView.

private static Optional<CachedView> getPartitionedCachedView(AttrLookup lookup, Context context, AttributeFamilyProxyDescriptor desc) {
    if (desc.getTargetFamilyRead().getAccess().canReadCommitLog() && !desc.getTargetFamilyWrite().getAccess().isReadonly() && desc.getTargetFamilyRead().getAccess().canCreateCachedView()) {
        Optional<CommitLogReader> maybeReader = getCommitLogReader(lookup, context, desc);
        Optional<OnlineAttributeWriter> maybeWriter = getWriter(lookup, context, desc).map(AttributeWriterBase::online);
        if (maybeReader.isPresent() && maybeWriter.isPresent()) {
            return Optional.of(new LocalCachedPartitionedView(desc.getTargetFamilyRead().getEntity(), maybeReader.get(), maybeWriter.get()));
        }
    }
    return Optional.empty();
}
Also used : LocalCachedPartitionedView(cz.o2.proxima.direct.view.LocalCachedPartitionedView) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader)

Example 3 with LocalCachedPartitionedView

use of cz.o2.proxima.direct.view.LocalCachedPartitionedView in project proxima-platform by O2-Czech-Republic.

the class InMemStorage method createAccessor.

@Override
public DataAccessor createAccessor(DirectDataOperator op, AttributeFamilyDescriptor familyDescriptor) {
    final EntityDescriptor entity = familyDescriptor.getEntity();
    final URI uri = familyDescriptor.getStorageUri();
    final Map<String, Object> cfg = familyDescriptor.getCfg();
    log.info("Creating accessor {} for URI {}", getClass(), uri);
    holder().observers.computeIfAbsent(uri, k -> Collections.synchronizedNavigableMap(new TreeMap<>()));
    final int numPartitions = Optional.ofNullable(cfg.get(NUM_PARTITIONS)).map(v -> Integer.parseInt(v.toString())).orElse(1);
    final Partitioner partitioner = Optional.ofNullable(cfg.get(ConfigConstants.PARTITIONER)).map(name -> Classpath.newInstance(name.toString(), Partitioner.class)).orElseGet(KeyAttributePartitioner::new);
    final Repository opRepo = op.getRepository();
    final RepositoryFactory repositoryFactory = opRepo.asFactory();
    final OnlineAttributeWriter.Factory<?> writerFactory = new Writer(entity, uri, numPartitions, partitioner).asFactory();
    final CommitLogReader.Factory<?> commitLogReaderFactory = new InMemCommitLogReader(entity, uri, op.getContext().getExecutorFactory(), partitioner, numPartitions).asFactory();
    final RandomAccessReader.Factory<Reader> randomAccessReaderFactory;
    final BatchLogReader.Factory<Reader> batchLogReaderFactory;
    final CachedView.Factory cachedViewFactory;
    if (numPartitions > 1) {
        randomAccessReaderFactory = null;
        batchLogReaderFactory = null;
        cachedViewFactory = null;
    } else {
        final ReaderFactory readerFactory = new Reader(entity, uri, op.getContext().getExecutorFactory()).asFactory();
        randomAccessReaderFactory = readerFactory;
        batchLogReaderFactory = readerFactory;
        cachedViewFactory = new LocalCachedPartitionedView(entity, commitLogReaderFactory.apply(opRepo), writerFactory.apply(opRepo)).asFactory();
    }
    return new DataAccessor() {

        private static final long serialVersionUID = 1L;

        @Nullable
        private transient Repository repo = opRepo;

        @Override
        public URI getUri() {
            return uri;
        }

        @Override
        public Optional<AttributeWriterBase> getWriter(Context context) {
            Objects.requireNonNull(context);
            return Optional.of(writerFactory.apply(repo()));
        }

        @Override
        public Optional<CommitLogReader> getCommitLogReader(Context context) {
            Objects.requireNonNull(context);
            return Optional.of(commitLogReaderFactory.apply(repo()));
        }

        @Override
        public Optional<RandomAccessReader> getRandomAccessReader(Context context) {
            Objects.requireNonNull(context);
            return Optional.ofNullable(randomAccessReaderFactory).map(item -> item.apply(repo()));
        }

        @Override
        public Optional<CachedView> getCachedView(Context context) {
            Objects.requireNonNull(context);
            return Optional.ofNullable(cachedViewFactory).map(item -> item.apply(repo()));
        }

        @Override
        public Optional<BatchLogReader> getBatchLogReader(Context context) {
            Objects.requireNonNull(context);
            return Optional.ofNullable(batchLogReaderFactory).map(item -> item.apply(repo()));
        }

        private Repository repo() {
            if (this.repo == null) {
                this.repo = repositoryFactory.apply();
            }
            return this.repo;
        }
    };
}
Also used : Partitioners(cz.o2.proxima.storage.commitlog.Partitioners) BatchLogReader(cz.o2.proxima.direct.batch.BatchLogReader) ConfigConstants(cz.o2.proxima.repository.ConfigConstants) RepositoryFactory(cz.o2.proxima.repository.RepositoryFactory) KeyAttributePartitioner(cz.o2.proxima.storage.commitlog.KeyAttributePartitioner) Future(java.util.concurrent.Future) WatermarkEstimator(cz.o2.proxima.time.WatermarkEstimator) Pair(cz.o2.proxima.util.Pair) Map(java.util.Map) Optionals(cz.o2.proxima.util.Optionals) ReadWriteLock(java.util.concurrent.locks.ReadWriteLock) Set(java.util.Set) ObserveHandle(cz.o2.proxima.direct.commitlog.ObserveHandle) Serializable(java.io.Serializable) CountDownLatch(java.util.concurrent.CountDownLatch) Slf4j(lombok.extern.slf4j.Slf4j) KeyValue(cz.o2.proxima.direct.randomaccess.KeyValue) DirectDataOperator(cz.o2.proxima.direct.core.DirectDataOperator) AbstractOnlineAttributeWriter(cz.o2.proxima.direct.core.AbstractOnlineAttributeWriter) Consumer(cz.o2.proxima.functional.Consumer) OffsetCommitter(cz.o2.proxima.direct.commitlog.CommitLogObserver.OffsetCommitter) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) Supplier(java.util.function.Supplier) ObserverUtils(cz.o2.proxima.direct.commitlog.ObserverUtils) ArrayList(java.util.ArrayList) Lists(com.google.common.collect.Lists) DataAccessor(cz.o2.proxima.direct.core.DataAccessor) RawOffset(cz.o2.proxima.direct.randomaccess.RawOffset) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Watermarks(cz.o2.proxima.time.Watermarks) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader) Nullable(javax.annotation.Nullable) WatermarkIdlePolicy(cz.o2.proxima.time.WatermarkIdlePolicy) AbstractStorage(cz.o2.proxima.storage.AbstractStorage) MoreObjects(com.google.common.base.MoreObjects) CommitLogObserver(cz.o2.proxima.direct.commitlog.CommitLogObserver) AtomicLong(java.util.concurrent.atomic.AtomicLong) Partitioner(cz.o2.proxima.storage.commitlog.Partitioner) Lock(java.util.concurrent.locks.Lock) TreeMap(java.util.TreeMap) AutoService(com.google.auto.service.AutoService) Preconditions(com.google.common.base.Preconditions) Position(cz.o2.proxima.storage.commitlog.Position) ScheduledFuture(java.util.concurrent.ScheduledFuture) AttributeWriterBase(cz.o2.proxima.direct.core.AttributeWriterBase) Partition(cz.o2.proxima.storage.Partition) EntityDescriptor(cz.o2.proxima.repository.EntityDescriptor) TerminationContext(cz.o2.proxima.direct.batch.TerminationContext) CachedView(cz.o2.proxima.direct.view.CachedView) StreamElement(cz.o2.proxima.storage.StreamElement) PartitionedWatermarkEstimator(cz.o2.proxima.time.PartitionedWatermarkEstimator) SerializationException(cz.o2.proxima.scheme.SerializationException) URI(java.net.URI) TypeReference(com.fasterxml.jackson.core.type.TypeReference) OffsetExternalizer(cz.o2.proxima.direct.commitlog.OffsetExternalizer) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ScheduledThreadPoolExecutor(java.util.concurrent.ScheduledThreadPoolExecutor) UUID(java.util.UUID) NavigableMap(java.util.NavigableMap) CommitCallback(cz.o2.proxima.direct.core.CommitCallback) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) OnNextContext(cz.o2.proxima.direct.batch.BatchLogObserver.OnNextContext) Entry(java.util.Map.Entry) Optional(java.util.Optional) MinimalPartitionWatermarkEstimator(cz.o2.proxima.direct.time.MinimalPartitionWatermarkEstimator) RandomOffset(cz.o2.proxima.direct.randomaccess.RandomOffset) SortedMap(java.util.SortedMap) ObserverUtils.asRepartitionContext(cz.o2.proxima.direct.commitlog.ObserverUtils.asRepartitionContext) Context(cz.o2.proxima.direct.core.Context) LocalCachedPartitionedView(cz.o2.proxima.direct.view.LocalCachedPartitionedView) Getter(lombok.Getter) BoundedOutOfOrdernessWatermarkEstimator(cz.o2.proxima.direct.time.BoundedOutOfOrdernessWatermarkEstimator) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) OnlineAttributeWriter(cz.o2.proxima.direct.core.OnlineAttributeWriter) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) BatchLogObservers(cz.o2.proxima.direct.batch.BatchLogObservers) HashSet(java.util.HashSet) Factory(cz.o2.proxima.functional.Factory) RandomAccessReader(cz.o2.proxima.direct.randomaccess.RandomAccessReader) ExecutorService(java.util.concurrent.ExecutorService) Classpath(cz.o2.proxima.util.Classpath) Repository(cz.o2.proxima.repository.Repository) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) AttributeDescriptor(cz.o2.proxima.repository.AttributeDescriptor) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) BatchLogObserver(cz.o2.proxima.direct.batch.BatchLogObserver) AttributeFamilyDescriptor(cz.o2.proxima.repository.AttributeFamilyDescriptor) Offset(cz.o2.proxima.direct.commitlog.Offset) TimeUnit(java.util.concurrent.TimeUnit) DataAccessorFactory(cz.o2.proxima.direct.core.DataAccessorFactory) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) CachedView(cz.o2.proxima.direct.view.CachedView) DataAccessor(cz.o2.proxima.direct.core.DataAccessor) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader) BatchLogReader(cz.o2.proxima.direct.batch.BatchLogReader) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader) RandomAccessReader(cz.o2.proxima.direct.randomaccess.RandomAccessReader) KeyAttributePartitioner(cz.o2.proxima.storage.commitlog.KeyAttributePartitioner) URI(java.net.URI) RandomAccessReader(cz.o2.proxima.direct.randomaccess.RandomAccessReader) AbstractOnlineAttributeWriter(cz.o2.proxima.direct.core.AbstractOnlineAttributeWriter) OnlineAttributeWriter(cz.o2.proxima.direct.core.OnlineAttributeWriter) RepositoryFactory(cz.o2.proxima.repository.RepositoryFactory) KeyAttributePartitioner(cz.o2.proxima.storage.commitlog.KeyAttributePartitioner) Partitioner(cz.o2.proxima.storage.commitlog.Partitioner) TerminationContext(cz.o2.proxima.direct.batch.TerminationContext) OnNextContext(cz.o2.proxima.direct.batch.BatchLogObserver.OnNextContext) ObserverUtils.asRepartitionContext(cz.o2.proxima.direct.commitlog.ObserverUtils.asRepartitionContext) Context(cz.o2.proxima.direct.core.Context) AttributeWriterBase(cz.o2.proxima.direct.core.AttributeWriterBase) TreeMap(java.util.TreeMap) EntityDescriptor(cz.o2.proxima.repository.EntityDescriptor) Repository(cz.o2.proxima.repository.Repository) LocalCachedPartitionedView(cz.o2.proxima.direct.view.LocalCachedPartitionedView) BatchLogReader(cz.o2.proxima.direct.batch.BatchLogReader) AbstractOnlineAttributeWriter(cz.o2.proxima.direct.core.AbstractOnlineAttributeWriter) OnlineAttributeWriter(cz.o2.proxima.direct.core.OnlineAttributeWriter)

Aggregations

CommitLogReader (cz.o2.proxima.direct.commitlog.CommitLogReader)3 VisibleForTesting (com.google.common.annotations.VisibleForTesting)2 CommitLogObserver (cz.o2.proxima.direct.commitlog.CommitLogObserver)2 ObserveHandle (cz.o2.proxima.direct.commitlog.ObserveHandle)2 Offset (cz.o2.proxima.direct.commitlog.Offset)2 CommitCallback (cz.o2.proxima.direct.core.CommitCallback)2 OnlineAttributeWriter (cz.o2.proxima.direct.core.OnlineAttributeWriter)2 KeyValue (cz.o2.proxima.direct.randomaccess.KeyValue)2 RandomOffset (cz.o2.proxima.direct.randomaccess.RandomOffset)2 RawOffset (cz.o2.proxima.direct.randomaccess.RawOffset)2 LocalCachedPartitionedView (cz.o2.proxima.direct.view.LocalCachedPartitionedView)2 Consumer (cz.o2.proxima.functional.Consumer)2 AttributeDescriptor (cz.o2.proxima.repository.AttributeDescriptor)2 EntityDescriptor (cz.o2.proxima.repository.EntityDescriptor)2 Partition (cz.o2.proxima.storage.Partition)2 StreamElement (cz.o2.proxima.storage.StreamElement)2 Position (cz.o2.proxima.storage.commitlog.Position)2 Pair (cz.o2.proxima.util.Pair)2 URI (java.net.URI)2 Collection (java.util.Collection)2