Examples with TransformationDescriptor cz.o2.proxima.repository.TransformationDescriptor used on opensource projects

Example 6 with TransformationDescriptor

use of cz.o2.proxima.repository.TransformationDescriptor in project proxima-platform by O2-Czech-Republic.

the class ReplicationController method runTransform.

private void runTransform(String name, TransformationDescriptor transform, DirectAttributeFamilyDescriptor family) {
    final StorageFilter filter = transform.getFilter();
    final String consumer = transform.getConsumerNameFactory().apply();
    final CommitLogReader reader = family.getCommitLogReader().orElseThrow(() -> new IllegalStateException("Unable to get reader for family " + family.getDesc().getName() + "."));
    final TransformationObserver observer;
    if (transform.getTransformation().isContextual()) {
        DirectElementWiseTransform transformation = transform.getTransformation().as(DirectElementWiseTransform.class);
        observer = contextualObserver(dataOperator, name, transformation, transform.getOutputTransactionMode() == OutputTransactionMode.ENABLED, filter);
    } else {
        ElementWiseTransformation transformation = transform.getTransformation().asElementWiseTransform();
        observer = nonContextualObserver(dataOperator, name, transformation, transform.getOutputTransactionMode() == OutputTransactionMode.ENABLED, filter);
    }
    startTransformationObserverUsing(consumer, reader, observer);
    log.info("Started transformer {} reading from {} using {}", consumer, reader.getUri(), transform.getTransformation().getClass());
}

Example 7 with TransformationDescriptor

use of cz.o2.proxima.repository.TransformationDescriptor in project proxima-platform by O2-Czech-Republic.

the class ReplicationController method runTransformer.

private void runTransformer(String name, TransformationDescriptor transform) {
    if (transform.getInputTransactionMode() == InputTransactionMode.TRANSACTIONAL) {
        log.info("Skipping run of transformation {} which read from transactional attributes {}. " + "Will be executed during transaction commit.", name, transform.getAttributes());
        return;
    }
    DirectAttributeFamilyDescriptor family = transform.getAttributes().stream().map(attr -> getAttributeDescriptorStreamFor(dataOperator, attr).collect(Collectors.toSet())).reduce(Sets::intersection).filter(s -> !s.isEmpty()).flatMap(s -> s.stream().filter(f -> f.getCommitLogReader().isPresent()).findAny()).orElseThrow(() -> new IllegalArgumentException("Cannot obtain attribute family for " + transform.getAttributes()));
    runTransform(name, transform, family);
}

Example 8 with TransformationDescriptor

use of cz.o2.proxima.repository.TransformationDescriptor in project proxima-platform by O2-Czech-Republic.

the class SingleTopicMultipleReplicationsTest method testTransformationWildcardInput.

@Test
public void testTransformationWildcardInput() {
    TransformationDescriptor wildcardInputTransform = repo.getTransformations().get("_wildcardReplication_read");
    assertNotNull(wildcardInputTransform);
    assertEquals(1, wildcardInputTransform.getAttributes().size());
    assertEquals(wildcardInput, wildcardInputTransform.getAttributes().get(0));
    List<StreamElement> transformed = new ArrayList<>();
    int transformedCount = wildcardInputTransform.getTransformation().asElementWiseTransform().apply(StreamElement.upsert(entity, wildcard, uuid(), "key", "wildcard.1", now, value()), transformed::add);
    assertEquals(1, transformedCount);
    assertEquals(1, transformed.size());
    assertEquals("_wildcardReplication_replicated$_raw.*", transformed.get(0).getAttributeDescriptor().getName());
    assertEquals("_raw.2", transformed.get(0).getAttribute());
}

Example 9 with TransformationDescriptor

use of cz.o2.proxima.repository.TransformationDescriptor in project proxima-platform by O2-Czech-Republic.

the class TransformationRunner method runTransformation.

/**
 * Run given transformation in local JVM.
 *
 * @param direct the operator to run transformations with
 * @param name name of the transformation
 * @param desc the transformation to run
 * @param onReplicated callback to be called before write to replicated target
 * @return {@link ObserveHandle} of the transformation
 */
public static ObserveHandle runTransformation(DirectDataOperator direct, String name, TransformationDescriptor desc, Consumer<StreamElement> onReplicated) {
    final CommitLogObserver observer;
    if (desc.getTransformation().isContextual()) {
        observer = new TransformationObserver.Contextual(direct, name, desc.getTransformation().as(DirectElementWiseTransform.class), desc.getOutputTransactionMode() == OutputTransactionMode.ENABLED, desc.getFilter()) {

            @Override
            protected void onReplicated(StreamElement element) {
                onReplicated.accept(element);
            }
        };
    } else {
        observer = new TransformationObserver.NonContextual(direct, name, desc.getTransformation().asElementWiseTransform(), desc.getOutputTransactionMode() == OutputTransactionMode.ENABLED, desc.getFilter()) {

            @Override
            protected void onReplicated(StreamElement element) {
                onReplicated.accept(element);
            }
        };
    }
    CommitLogReader reader = desc.getAttributes().stream().flatMap(attr -> findFamilyDescriptorForAttribute(direct, attr)).findAny().flatMap(DirectAttributeFamilyDescriptor::getCommitLogReader).orElseThrow(() -> new IllegalStateException("No commit log reader for attributes of transformation " + desc));
    log.debug("Starting to observe reader {} with observer {} as {}", reader, observer, name);
    return reader.observe(name, observer);
}

Example 10 with TransformationDescriptor

use of cz.o2.proxima.repository.TransformationDescriptor in project proxima-platform by O2-Czech-Republic.

the class SingleTopicMultipleReplicationsTest method testTransformationWildcardWriteRead.

@Test
public void testTransformationWildcardWriteRead() {
    TransformationDescriptor wildcardReplicatedTransform = repo.getTransformations().get("_wildcardReplication_replicated");
    assertNotNull(wildcardReplicatedTransform);
    assertEquals(1, wildcardReplicatedTransform.getAttributes().size());
    assertEquals(rawWrite, wildcardReplicatedTransform.getAttributes().get(0));
    List<StreamElement> transformed = new ArrayList<>();
    int transformedCount = wildcardReplicatedTransform.getTransformation().asElementWiseTransform().apply(StreamElement.upsert(entity, raw, uuid(), "key", "_raw.2", now, value()), transformed::add);
    assertEquals(1, transformedCount);
    assertEquals(1, transformed.size());
    assertEquals("_wildcardReplication_replicated$_raw.*", transformed.get(0).getAttributeDescriptor().getName());
    assertEquals("_raw.2", transformed.get(0).getAttribute());
}