Examples with DirectAttributeFamilyDescriptor cz.o2.proxima.direct.core.DirectAttributeFamilyDescriptor used on opensource projects

Example 26 with DirectAttributeFamilyDescriptor

use of cz.o2.proxima.direct.core.DirectAttributeFamilyDescriptor in project proxima-platform by O2-Czech-Republic.

the class TransformationRunner method runTransformation.

/**
 * Run given transformation in local JVM.
 *
 * @param direct the operator to run transformations with
 * @param name name of the transformation
 * @param desc the transformation to run
 * @param onReplicated callback to be called before write to replicated target
 * @return {@link ObserveHandle} of the transformation
 */
public static ObserveHandle runTransformation(DirectDataOperator direct, String name, TransformationDescriptor desc, Consumer<StreamElement> onReplicated) {
    final CommitLogObserver observer;
    if (desc.getTransformation().isContextual()) {
        observer = new TransformationObserver.Contextual(direct, name, desc.getTransformation().as(DirectElementWiseTransform.class), desc.getOutputTransactionMode() == OutputTransactionMode.ENABLED, desc.getFilter()) {

            @Override
            protected void onReplicated(StreamElement element) {
                onReplicated.accept(element);
            }
        };
    } else {
        observer = new TransformationObserver.NonContextual(direct, name, desc.getTransformation().asElementWiseTransform(), desc.getOutputTransactionMode() == OutputTransactionMode.ENABLED, desc.getFilter()) {

            @Override
            protected void onReplicated(StreamElement element) {
                onReplicated.accept(element);
            }
        };
    }
    CommitLogReader reader = desc.getAttributes().stream().flatMap(attr -> findFamilyDescriptorForAttribute(direct, attr)).findAny().flatMap(DirectAttributeFamilyDescriptor::getCommitLogReader).orElseThrow(() -> new IllegalStateException("No commit log reader for attributes of transformation " + desc));
    log.debug("Starting to observe reader {} with observer {} as {}", reader, observer, name);
    return reader.observe(name, observer);
}

Example 27 with DirectAttributeFamilyDescriptor

use of cz.o2.proxima.direct.core.DirectAttributeFamilyDescriptor in project proxima-platform by O2-Czech-Republic.

the class BeamStream method persistIntoTargetFamily.

@Override
public void persistIntoTargetFamily(RepositoryProvider repoProvider, String targetFamilyname, int parallelism) {
    DirectAttributeFamilyDescriptor familyDescriptor = repoProvider.getDirect().getAllFamilies().filter(af -> af.getDesc().getName().equals(targetFamilyname)).findAny().orElseThrow(() -> new IllegalArgumentException(String.format("Family [%s] does not exist", targetFamilyname)));
    Preconditions.checkArgument(!familyDescriptor.getDesc().getAccess().isReadonly());
    AttributeWriterBase rawWriter = familyDescriptor.getWriter().orElseThrow(() -> new IllegalArgumentException(String.format("Family [%s] does not have writer", targetFamilyname)));
    RepositoryFactory repositoryFactory = repoProvider.getRepo().asFactory();
    AttributeWriterBase.Factory<?> writerFactory = rawWriter.asFactory();
    SerializableScopedValue<Integer, AttributeWriterBase> writer = new SerializableScopedValue<>(() -> writerFactory.apply(repositoryFactory.apply()));
    Set<String> allowedAttributes = familyDescriptor.getAttributes().stream().map(AttributeDescriptor::getName).collect(Collectors.toSet());
    switch(rawWriter.getType()) {
        case ONLINE:
            writeUsingOnlineWriterFactory("write-to-" + targetFamilyname, el -> {
                Preconditions.checkArgument(el == null || allowedAttributes.contains(el.getAttributeDescriptor().getName()));
                return writer.get(0).online();
            });
            break;
        case BULK:
            writeUsingBulkWriterFactory("write-bulk-to-" + targetFamilyname, parallelism, BulkWriterFactory.wrap(writer, allowedAttributes));
            break;
        default:
            throw new IllegalArgumentException("Unknonw type " + rawWriter.getType());
    }
}

Example 28 with DirectAttributeFamilyDescriptor

use of cz.o2.proxima.direct.core.DirectAttributeFamilyDescriptor in project proxima-platform by O2-Czech-Republic.

the class MultiAccessBuilderTest method testMultiAttributes.

@Test
public void testMultiAttributes() {
    EntityDescriptor gateway = repo.findEntity("gateway").orElseThrow(() -> new IllegalStateException("Missing entity gateway"));
    AttributeDescriptor<?> armed = gateway.findAttribute("armed").orElseThrow(() -> new IllegalStateException("Missing attribute armed in gateway"));
    AttributeDescriptor<?> device = gateway.findAttribute("device.*").orElseThrow(() -> new IllegalStateException("Missing attribute device.* in gateway"));
    RandomAccessReader base = repo.getAllFamilies().filter(af -> af.getName().equals("gateway-storage-stream")).findAny().map(direct::resolveRequired).flatMap(DirectAttributeFamilyDescriptor::getRandomAccessReader).orElseThrow(() -> new IllegalStateException("Cannot get random access reader"));
    reader = RandomAccessReader.newBuilder(repo, direct).addAttributes(base, armed, device).build();
    // write some data
    direct.getWriter(armed).get().write(StreamElement.upsert(gateway, armed, UUID.randomUUID().toString(), "gw", armed.getName(), now, new byte[] { 1, 2 }), (succ, exc) -> {
    });
    direct.getWriter(device).get().write(StreamElement.upsert(gateway, device, UUID.randomUUID().toString(), "gw", device.toAttributePrefix() + "1", now, new byte[] { 2, 3 }), (succ, exc) -> {
    });
    Optional<? extends KeyValue<?>> kv = reader.get("gw", armed);
    assertTrue(kv.isPresent());
    assertArrayEquals(new byte[] { 1, 2 }, kv.get().getValue());
    kv = reader.get("gw", device.toAttributePrefix() + "1", device);
    assertTrue(kv.isPresent());
    assertArrayEquals(new byte[] { 2, 3 }, kv.get().getValue());
    kv = reader.get("gw", device.toAttributePrefix() + "2", device);
    assertFalse(kv.isPresent());
}

Example 29 with DirectAttributeFamilyDescriptor

use of cz.o2.proxima.direct.core.DirectAttributeFamilyDescriptor in project proxima-platform by O2-Czech-Republic.

the class ReplicationController method indexFamilyToCommitLogs.

/**
 * Retrieve attribute family and it's associated commit log(s). The families returned are only
 * those which are not used as commit log themselves.
 */
private Map<DirectAttributeFamilyDescriptor, Set<DirectAttributeFamilyDescriptor>> indexFamilyToCommitLogs() {
    // each attribute and its associated primary family
    final Map<AttributeDescriptor<?>, DirectAttributeFamilyDescriptor> primaryFamilies = dataOperator.getAllFamilies().filter(family -> family.getDesc().getType() == StorageType.PRIMARY).flatMap(primaryFamily -> primaryFamily.getAttributes().stream().map(attribute -> Pair.of(attribute, primaryFamily))).collect(Collectors.toMap(Pair::getFirst, Pair::getSecond));
    return dataOperator.getAllFamilies().filter(family -> family.getDesc().getType() == StorageType.REPLICA).map(replicaFamily -> {
        if (replicaFamily.getSource().isPresent()) {
            final String source = replicaFamily.getSource().get();
            return Pair.of(replicaFamily, Collections.singleton(dataOperator.getAllFamilies().filter(af2 -> af2.getDesc().getName().equals(source)).findAny().orElseThrow(() -> new IllegalArgumentException(String.format("Unknown family %s.", source)))));
        }
        return Pair.of(replicaFamily, replicaFamily.getAttributes().stream().map(attr -> {
            final DirectAttributeFamilyDescriptor primaryFamily = primaryFamilies.get(attr);
            final Optional<OnlineAttributeWriter> maybeWriter = dataOperator.getWriter(attr);
            if (primaryFamily == null && maybeWriter.isPresent()) {
                throw new IllegalStateException(String.format("Missing source commit log family for %s.", attr));
            }
            return primaryFamily;
        }).filter(Objects::nonNull).collect(Collectors.toSet()));
    }).collect(Collectors.toMap(Pair::getFirst, Pair::getSecond));
}

Example 30 with DirectAttributeFamilyDescriptor

use of cz.o2.proxima.direct.core.DirectAttributeFamilyDescriptor in project proxima-platform by O2-Czech-Republic.

the class ReplicationController method runReplicationThreads.

public CompletableFuture<Void> runReplicationThreads() {
    final CompletableFuture<Void> completed = new CompletableFuture<>();
    replications.add(completed);
    // index the repository
    Map<DirectAttributeFamilyDescriptor, Set<DirectAttributeFamilyDescriptor>> familyToCommitLog;
    familyToCommitLog = indexFamilyToCommitLogs();
    log.info("Starting consumer threads for familyToCommitLog {}", familyToCommitLog);
    // execute threads to consume the commit log
    familyToCommitLog.forEach((replicaFamily, primaryFamilies) -> {
        for (DirectAttributeFamilyDescriptor primaryFamily : primaryFamilies) {
            if (!replicaFamily.getDesc().getAccess().isReadonly()) {
                consumeLog(primaryFamily, replicaFamily);
            } else {
                log.debug("Not starting thread for read-only family {}", replicaFamily);
            }
        }
    });
    // execute transformer threads
    repository.getTransformations().forEach(this::runTransformer);
    scheduler.scheduleAtFixedRate(this::checkLiveness, 0, 1, TimeUnit.SECONDS);
    return completed;
}