Search in sources :

Example 1 with DataOperator

use of cz.o2.proxima.repository.DataOperator in project proxima-platform by O2-Czech-Republic.

the class ReplicationController method runTransform.

private void runTransform(String name, TransformationDescriptor transform, DirectAttributeFamilyDescriptor family) {
    final StorageFilter filter = transform.getFilter();
    final String consumer = transform.getConsumerNameFactory().apply();
    final CommitLogReader reader = family.getCommitLogReader().orElseThrow(() -> new IllegalStateException("Unable to get reader for family " + family.getDesc().getName() + "."));
    final TransformationObserver observer;
    if (transform.getTransformation().isContextual()) {
        DirectElementWiseTransform transformation = transform.getTransformation().as(DirectElementWiseTransform.class);
        observer = contextualObserver(dataOperator, name, transformation, transform.getOutputTransactionMode() == OutputTransactionMode.ENABLED, filter);
    } else {
        ElementWiseTransformation transformation = transform.getTransformation().asElementWiseTransform();
        observer = nonContextualObserver(dataOperator, name, transformation, transform.getOutputTransactionMode() == OutputTransactionMode.ENABLED, filter);
    }
    startTransformationObserverUsing(consumer, reader, observer);
    log.info("Started transformer {} reading from {} using {}", consumer, reader.getUri(), transform.getTransformation().getClass());
}
Also used : ElementWiseTransformation(cz.o2.proxima.transform.ElementWiseTransformation) StorageFilter(cz.o2.proxima.storage.StorageFilter) DirectElementWiseTransform(cz.o2.proxima.direct.transform.DirectElementWiseTransform) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader) TransformationObserver(cz.o2.proxima.direct.transform.TransformationObserver)

Example 2 with DataOperator

use of cz.o2.proxima.repository.DataOperator in project proxima-platform by O2-Czech-Republic.

the class ReplicationController method runTransformer.

private void runTransformer(String name, TransformationDescriptor transform) {
    if (transform.getInputTransactionMode() == InputTransactionMode.TRANSACTIONAL) {
        log.info("Skipping run of transformation {} which read from transactional attributes {}. " + "Will be executed during transaction commit.", name, transform.getAttributes());
        return;
    }
    DirectAttributeFamilyDescriptor family = transform.getAttributes().stream().map(attr -> getAttributeDescriptorStreamFor(dataOperator, attr).collect(Collectors.toSet())).reduce(Sets::intersection).filter(s -> !s.isEmpty()).flatMap(s -> s.stream().filter(f -> f.getCommitLogReader().isPresent()).findAny()).orElseThrow(() -> new IllegalArgumentException("Cannot obtain attribute family for " + transform.getAttributes()));
    runTransform(name, transform, family);
}
Also used : AttributeWriterBase(cz.o2.proxima.direct.core.AttributeWriterBase) Getter(lombok.Getter) InputTransactionMode(cz.o2.proxima.repository.TransformationDescriptor.InputTransactionMode) EntityDescriptor(cz.o2.proxima.repository.EntityDescriptor) StorageType(cz.o2.proxima.storage.StorageType) CompletableFuture(java.util.concurrent.CompletableFuture) OnlineAttributeWriter(cz.o2.proxima.direct.core.OnlineAttributeWriter) DirectElementWiseTransform(cz.o2.proxima.direct.transform.DirectElementWiseTransform) HashSet(java.util.HashSet) StreamElement(cz.o2.proxima.storage.StreamElement) Pair(cz.o2.proxima.util.Pair) Map(java.util.Map) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ConfigFactory(com.typesafe.config.ConfigFactory) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader) TransformationDescriptor(cz.o2.proxima.repository.TransformationDescriptor) Metrics(cz.o2.proxima.server.metrics.Metrics) BulkAttributeWriter(cz.o2.proxima.direct.core.BulkAttributeWriter) Repository(cz.o2.proxima.repository.Repository) AttributeDescriptor(cz.o2.proxima.repository.AttributeDescriptor) CommitLogObserver(cz.o2.proxima.direct.commitlog.CommitLogObserver) Set(java.util.Set) StorageFilter(cz.o2.proxima.storage.StorageFilter) ScheduledThreadPoolExecutor(java.util.concurrent.ScheduledThreadPoolExecutor) RetryPolicy(net.jodah.failsafe.RetryPolicy) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) File(java.io.File) Failsafe(net.jodah.failsafe.Failsafe) CommitLogObservers(cz.o2.proxima.direct.commitlog.CommitLogObservers) Objects(java.util.Objects) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) Stream(java.util.stream.Stream) DirectAttributeFamilyDescriptor(cz.o2.proxima.direct.core.DirectAttributeFamilyDescriptor) TransformationObserver(cz.o2.proxima.direct.transform.TransformationObserver) Optional(java.util.Optional) TerminationStrategy(cz.o2.proxima.direct.commitlog.CommitLogObservers.TerminationStrategy) ElementWiseTransformation(cz.o2.proxima.transform.ElementWiseTransformation) VisibleForTesting(com.google.common.annotations.VisibleForTesting) DirectDataOperator(cz.o2.proxima.direct.core.DirectDataOperator) AllArgsConstructor(lombok.AllArgsConstructor) Collections(java.util.Collections) OutputTransactionMode(cz.o2.proxima.repository.TransformationDescriptor.OutputTransactionMode) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) DirectAttributeFamilyDescriptor(cz.o2.proxima.direct.core.DirectAttributeFamilyDescriptor)

Aggregations

CommitLogReader (cz.o2.proxima.direct.commitlog.CommitLogReader)2 DirectElementWiseTransform (cz.o2.proxima.direct.transform.DirectElementWiseTransform)2 TransformationObserver (cz.o2.proxima.direct.transform.TransformationObserver)2 StorageFilter (cz.o2.proxima.storage.StorageFilter)2 ElementWiseTransformation (cz.o2.proxima.transform.ElementWiseTransformation)2 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Sets (com.google.common.collect.Sets)1 ConfigFactory (com.typesafe.config.ConfigFactory)1 CommitLogObserver (cz.o2.proxima.direct.commitlog.CommitLogObserver)1 CommitLogObservers (cz.o2.proxima.direct.commitlog.CommitLogObservers)1 TerminationStrategy (cz.o2.proxima.direct.commitlog.CommitLogObservers.TerminationStrategy)1 AttributeWriterBase (cz.o2.proxima.direct.core.AttributeWriterBase)1 BulkAttributeWriter (cz.o2.proxima.direct.core.BulkAttributeWriter)1 DirectAttributeFamilyDescriptor (cz.o2.proxima.direct.core.DirectAttributeFamilyDescriptor)1 DirectDataOperator (cz.o2.proxima.direct.core.DirectDataOperator)1 OnlineAttributeWriter (cz.o2.proxima.direct.core.OnlineAttributeWriter)1 AttributeDescriptor (cz.o2.proxima.repository.AttributeDescriptor)1 EntityDescriptor (cz.o2.proxima.repository.EntityDescriptor)1 Repository (cz.o2.proxima.repository.Repository)1 TransformationDescriptor (cz.o2.proxima.repository.TransformationDescriptor)1