Examples with Transformation cz.o2.proxima.transform.Transformation used on opensource projects

Example 1 with Transformation

use of cz.o2.proxima.transform.Transformation in project proxima-platform by O2-Czech-Republic.

the class DirectDataOperatorTest method testReplicationTransformations.

@Test
public void testReplicationTransformations() {
    repo.reloadConfig(true, ConfigFactory.load().withFallback(ConfigFactory.load("test-replication.conf")).withFallback(ConfigFactory.load("test-reference.conf")).resolve());
    final EntityDescriptor dummy = repo.getEntity("dummy");
    Map<String, TransformationDescriptor> transformations = repo.getTransformations();
    assertNotNull(transformations.get("_dummyReplicationMasterSlave_slave"));
    assertNotNull(transformations.get("_dummyReplicationMasterSlave_replicated"));
    assertNotNull(transformations.get("_dummyReplicationProxiedSlave_read"));
    // transformation from local writes to slave
    checkTransformation(dummy, transformations.get("_dummyReplicationMasterSlave_slave"), "wildcard.*", "_dummyReplicationMasterSlave_write$wildcard.*", "wildcard.*", "_dummyReplicationMasterSlave_slave$wildcard.*");
    // transformation from local writes to replicated result
    checkTransformation(dummy, transformations.get("_dummyReplicationMasterSlave_replicated"), "wildcard.*", "_dummyReplicationMasterSlave_write$wildcard.*", "wildcard.*", "_dummyReplicationMasterSlave_replicated$wildcard.*");
    // transformation from remote writes to local replicated result
    // with proxy
    checkTransformation(dummy, transformations.get("_dummyReplicationProxiedSlave_read"), "data", "data", "_d", "_dummyReplicationProxiedSlave_replicated$_d");
}

Example 2 with Transformation

use of cz.o2.proxima.transform.Transformation in project proxima-platform by O2-Czech-Republic.

the class ConfigRepository method readTransformations.

private void readTransformations(Config cfg) {
    if (entitiesByName.isEmpty()) {
        // no loaded entities, no more stuff to read
        return;
    }
    Map<String, Object> cfgTransforms = Optional.ofNullable(cfg.root().get("transformations")).map(v -> toMap("transformations", v.unwrapped())).orElse(null);
    if (cfgTransforms == null) {
        log.info("Skipping empty transformations configuration.");
        return;
    }
    cfgTransforms.forEach((name, v) -> {
        Map<String, Object> transformation = toMap(name, v);
        boolean disabled = Optional.ofNullable(transformation.get(DISABLED)).map(d -> Boolean.valueOf(d.toString())).orElse(false);
        if (disabled) {
            log.info("Skipping load of disabled transformation {}", name);
            return;
        }
        EntityDescriptor entity = findEntity(readStr(ENTITY, transformation, name)).orElseThrow(() -> new IllegalArgumentException(String.format("Entity `%s` doesn't exist", transformation.get(ENTITY))));
        Transformation t = Classpath.newInstance(readStr("using", transformation, name), Transformation.class);
        List<AttributeDescriptor<?>> attrs = readList(ATTRIBUTES, transformation, name).stream().flatMap(a -> searchAttributesMatching(a, entity, true, true).stream()).collect(Collectors.toList());
        TransformationDescriptor.Builder desc = TransformationDescriptor.newBuilder().setName(name).addAttributes(attrs).setTransformation(t);
        Optional.ofNullable(transformation.get(FILTER)).map(Object::toString).map(s -> Classpath.newInstance(s, StorageFilter.class)).ifPresent(desc::setFilter);
        TransformationDescriptor transformationDescriptor = desc.build();
        setupTransform(transformationDescriptor.getTransformation(), transformation);
        this.transformations.put(name, transformationDescriptor);
    });
}

Example 3 with Transformation

use of cz.o2.proxima.transform.Transformation in project proxima-platform by O2-Czech-Republic.

the class DirectDataOperatorTest method testReplicationTransformationsNonProxied.

@Test
public void testReplicationTransformationsNonProxied() {
    repo.reloadConfig(true, ConfigFactory.load().withFallback(ConfigFactory.load("test-replication.conf")).withFallback(ConfigFactory.load("test-reference.conf")).resolve());
    EntityDescriptor gateway = repo.getEntity("gateway");
    Map<String, TransformationDescriptor> transformations = repo.getTransformations();
    assertNotNull(transformations.get("_gatewayReplication_read"));
    assertNotNull(transformations.get("_gatewayReplication_inmemSecond"));
    // transformation from remote writes to local replicated result
    // without proxy
    checkTransformation(gateway, transformations.get("_gatewayReplication_read"), "armed", "armed");
    // transformation from local writes to slave
    checkTransformation(gateway, transformations.get("_gatewayReplication_inmemSecond"), "armed", "_gatewayReplication_write$armed", "armed", "_gatewayReplication_inmemSecond$armed");
}

Example 4 with Transformation

use of cz.o2.proxima.transform.Transformation in project proxima-platform by O2-Czech-Republic.

the class ReplicationController method runTransform.

private void runTransform(String name, TransformationDescriptor transform, DirectAttributeFamilyDescriptor family) {
    final StorageFilter filter = transform.getFilter();
    final String consumer = transform.getConsumerNameFactory().apply();
    final CommitLogReader reader = family.getCommitLogReader().orElseThrow(() -> new IllegalStateException("Unable to get reader for family " + family.getDesc().getName() + "."));
    final TransformationObserver observer;
    if (transform.getTransformation().isContextual()) {
        DirectElementWiseTransform transformation = transform.getTransformation().as(DirectElementWiseTransform.class);
        observer = contextualObserver(dataOperator, name, transformation, transform.getOutputTransactionMode() == OutputTransactionMode.ENABLED, filter);
    } else {
        ElementWiseTransformation transformation = transform.getTransformation().asElementWiseTransform();
        observer = nonContextualObserver(dataOperator, name, transformation, transform.getOutputTransactionMode() == OutputTransactionMode.ENABLED, filter);
    }
    startTransformationObserverUsing(consumer, reader, observer);
    log.info("Started transformer {} reading from {} using {}", consumer, reader.getUri(), transform.getTransformation().getClass());
}

Example 5 with Transformation

use of cz.o2.proxima.transform.Transformation in project proxima-platform by O2-Czech-Republic.

the class ReplicationController method runTransformer.

private void runTransformer(String name, TransformationDescriptor transform) {
    if (transform.getInputTransactionMode() == InputTransactionMode.TRANSACTIONAL) {
        log.info("Skipping run of transformation {} which read from transactional attributes {}. " + "Will be executed during transaction commit.", name, transform.getAttributes());
        return;
    }
    DirectAttributeFamilyDescriptor family = transform.getAttributes().stream().map(attr -> getAttributeDescriptorStreamFor(dataOperator, attr).collect(Collectors.toSet())).reduce(Sets::intersection).filter(s -> !s.isEmpty()).flatMap(s -> s.stream().filter(f -> f.getCommitLogReader().isPresent()).findAny()).orElseThrow(() -> new IllegalArgumentException("Cannot obtain attribute family for " + transform.getAttributes()));
    runTransform(name, transform, family);
}