Examples with Operator com.datatorrent.api.Operator used on opensource projects

Example 1 with Operator

use of com.datatorrent.api.Operator in project apex-core by apache.

the class GenericNodeTest method testReservoirPortMapping.

@Test
public void testReservoirPortMapping() throws InterruptedException {
    long maxSleep = 5000;
    long sleeptime = 25L;
    GenericOperator go = new GenericOperator();
    final GenericNode gn = new GenericNode(go, new com.datatorrent.stram.engine.OperatorContext(0, "operator", new DefaultAttributeMap(), null));
    gn.setId(1);
    AbstractReservoir reservoir1 = AbstractReservoir.newReservoir("ip1Res", 1024);
    AbstractReservoir reservoir2 = AbstractReservoir.newReservoir("ip2Res", 1024);
    gn.connectInputPort("ip1", reservoir1);
    gn.connectInputPort("ip2", reservoir2);
    gn.connectOutputPort("op", Sink.BLACKHOLE);
    gn.firstWindowMillis = 0;
    gn.windowWidthMillis = 100;
    final AtomicBoolean ab = new AtomicBoolean(false);
    Thread t = new Thread() {

        @Override
        public void run() {
            ab.set(true);
            gn.activate();
            gn.run();
            gn.deactivate();
        }
    };
    t.start();
    long interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((ab.get() == false) && (interval < maxSleep));
    gn.populateReservoirInputPortMap();
    gn.shutdown();
    t.join();
    Assert.assertTrue("Port Mapping Size", gn.reservoirPortMap.size() == 2);
    Assert.assertTrue("Sink 1 is not a port", gn.reservoirPortMap.get(reservoir1) instanceof Operator.InputPort);
    Assert.assertTrue("Sink 2 is not a port", gn.reservoirPortMap.get(reservoir2) instanceof Operator.InputPort);
}

Example 2 with Operator

use of com.datatorrent.api.Operator in project apex-core by apache.

the class Operators method describe.

public static void describe(GenericOperator operator, OperatorDescriptor descriptor) {
    for (Class<?> c = operator.getClass(); c != Object.class; c = c.getSuperclass()) {
        Field[] fields = c.getDeclaredFields();
        for (Field field : fields) {
            field.setAccessible(true);
            InputPortFieldAnnotation inputAnnotation = field.getAnnotation(InputPortFieldAnnotation.class);
            OutputPortFieldAnnotation outputAnnotation = field.getAnnotation(OutputPortFieldAnnotation.class);
            AppData.QueryPort adqAnnotation = field.getAnnotation(AppData.QueryPort.class);
            AppData.ResultPort adrAnnotation = field.getAnnotation(AppData.ResultPort.class);
            try {
                Object portObject = field.get(operator);
                if (portObject instanceof InputPort) {
                    descriptor.addInputPort((Operator.InputPort<?>) portObject, field, inputAnnotation, adqAnnotation);
                } else {
                    if (inputAnnotation != null) {
                        throw new IllegalArgumentException("port is not of type " + InputPort.class.getName() + ": " + field);
                    }
                }
                if (portObject instanceof OutputPort) {
                    descriptor.addOutputPort((Operator.OutputPort<?>) portObject, field, outputAnnotation, adrAnnotation);
                } else {
                    if (outputAnnotation != null) {
                        throw new IllegalArgumentException("port is not of type " + OutputPort.class.getName() + ": " + field);
                    }
                }
            } catch (IllegalAccessException e) {
                throw new RuntimeException(e);
            }
        }
    }
}

Example 3 with Operator

use of com.datatorrent.api.Operator in project apex-core by apache.

the class PhysicalPlan method assignContainers.

private void assignContainers(Set<PTContainer> newContainers, Set<PTContainer> releaseContainers) {
    Set<PTOperator> mxnUnifiers = Sets.newHashSet();
    for (PTOperator o : this.newOpers.keySet()) {
        mxnUnifiers.addAll(o.upstreamMerge.values());
    }
    Set<PTContainer> updatedContainers = Sets.newHashSet();
    HashMap<PTOperator, PTContainer> operatorContainerMap = Maps.newHashMap();
    for (Map.Entry<PTOperator, Operator> operEntry : this.newOpers.entrySet()) {
        PTOperator oper = operEntry.getKey();
        Checkpoint checkpoint = getActivationCheckpoint(operEntry.getKey());
        initCheckpoint(oper, operEntry.getValue(), checkpoint);
        if (mxnUnifiers.contains(operEntry.getKey())) {
            // MxN unifiers are assigned with the downstream operator
            continue;
        }
        PTContainer newContainer = null;
        int memoryMB = 0;
        // handle container locality
        for (PTOperator inlineOper : oper.getGrouping(Locality.CONTAINER_LOCAL).getOperatorSet()) {
            if (inlineOper.container != null) {
                newContainer = inlineOper.container;
                break;
            }
            memoryMB += inlineOper.operatorMeta.getValue(OperatorContext.MEMORY_MB);
            memoryMB += inlineOper.getBufferServerMemory();
        }
        if (newContainer == null) {
            int vCores = getVCores(oper.getGrouping(Locality.CONTAINER_LOCAL).getOperatorSet());
            // attempt to find empty container with required size
            for (PTContainer c : this.containers) {
                if (c.operators.isEmpty() && c.getState() == PTContainer.State.ACTIVE && c.getAllocatedMemoryMB() == memoryMB && c.getAllocatedVCores() == vCores) {
                    LOG.debug("Reusing existing container {} for {}", c, oper);
                    c.setRequiredMemoryMB(0);
                    c.setRequiredVCores(0);
                    newContainer = c;
                    break;
                }
            }
            if (newContainer == null) {
                // get new container
                LOG.debug("New container for: " + oper);
                newContainer = new PTContainer(this);
                newContainers.add(newContainer);
                containers.add(newContainer);
            }
            updatedContainers.add(newContainer);
        }
        setContainer(oper, newContainer);
    }
    // release containers that are no longer used and update operator to container map for applying anti-affinity
    for (PTContainer c : this.containers) {
        if (c.operators.isEmpty()) {
            LOG.debug("Container {} to be released", c);
            releaseContainers.add(c);
            containers.remove(c);
        } else {
            for (PTOperator oper : c.operators) {
                operatorContainerMap.put(oper, c);
            }
            c.getStrictAntiPrefs().clear();
            c.getPreferredAntiPrefs().clear();
        }
    }
    for (PTContainer c : updatedContainers) {
        updateContainerMemoryWithBufferServer(c);
        c.setRequiredVCores(getVCores(c.getOperators()));
    }
    AffinityRulesSet affinityRuleSet = dag.getAttributes().get(DAGContext.AFFINITY_RULES_SET);
    // Add anti-affinity restrictions in Containers
    if (affinityRuleSet != null && affinityRuleSet.getAffinityRules() != null) {
        setAntiAffinityForContainers(dag, affinityRuleSet.getAffinityRules(), operatorContainerMap);
    }
}

Example 4 with Operator

use of com.datatorrent.api.Operator in project apex-core by apache.

the class LogicalPlan method validateProcessingMode.

private void validateProcessingMode(OperatorMeta om, Set<OperatorMeta> visited) {
    for (StreamMeta is : om.getInputStreams().values()) {
        if (!visited.contains(is.getSource().getOperatorMeta())) {
            // process all inputs first
            return;
        }
    }
    visited.add(om);
    Operator.ProcessingMode pm = om.getValue(OperatorContext.PROCESSING_MODE);
    for (StreamMeta os : om.outputStreams.values()) {
        for (InputPortMeta sink : os.sinks) {
            OperatorMeta sinkOm = sink.getOperatorMeta();
            Operator.ProcessingMode sinkPm = sinkOm.attributes == null ? null : sinkOm.attributes.get(OperatorContext.PROCESSING_MODE);
            if (sinkPm == null) {
                // set it to AT_MOST_ONCE as well
                if (Operator.ProcessingMode.AT_MOST_ONCE.equals(pm)) {
                    LOG.warn("Setting processing mode for operator {} to {}", sinkOm.getName(), pm);
                    sinkOm.getAttributes().put(OperatorContext.PROCESSING_MODE, pm);
                } else if (Operator.ProcessingMode.EXACTLY_ONCE.equals(pm)) {
                    // If the source processing mode is EXACTLY_ONCE and a processing mode is not specified for the sink then
                    // throw a validation error
                    String msg = String.format("Processing mode for %s should be AT_MOST_ONCE for source %s/%s", sinkOm.getName(), om.getName(), pm);
                    throw new ValidationException(msg);
                }
            } else {
                /*
           * If the source processing mode is AT_MOST_ONCE and the processing mode for the sink is not AT_MOST_ONCE throw a validation error
           * If the source processing mode is EXACTLY_ONCE and the processing mode for the sink is not AT_MOST_ONCE throw a validation error
           */
                if ((Operator.ProcessingMode.AT_MOST_ONCE.equals(pm) && (sinkPm != pm)) || (Operator.ProcessingMode.EXACTLY_ONCE.equals(pm) && !Operator.ProcessingMode.AT_MOST_ONCE.equals(sinkPm))) {
                    String msg = String.format("Processing mode %s/%s not valid for source %s/%s", sinkOm.getName(), sinkPm, om.getName(), pm);
                    throw new ValidationException(msg);
                }
            }
            validateProcessingMode(sinkOm, visited);
        }
    }
}

Example 5 with Operator

use of com.datatorrent.api.Operator in project apex-core by apache.

the class LogicalPlan method validate.

/**
   * Validate the plan. Includes checks that required ports are connected,
   * required configuration parameters specified, graph free of cycles etc.
   *
   * @throws ConstraintViolationException
   */
public void validate() throws ConstraintViolationException {
    ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
    Validator validator = factory.getValidator();
    checkAttributeValueSerializable(this.getAttributes(), DAG.class.getName());
    // clear oioRoot values in all operators
    for (OperatorMeta n : operators.values()) {
        n.oioRoot = null;
    }
    // clear visited on all operators
    for (OperatorMeta n : operators.values()) {
        n.nindex = null;
        n.lowlink = null;
        // validate configuration
        Set<ConstraintViolation<Operator>> constraintViolations = validator.validate(n.getOperator());
        if (!constraintViolations.isEmpty()) {
            Set<ConstraintViolation<?>> copySet = new HashSet<>(constraintViolations.size());
            // (should be public <T> ConstraintViolationException(String message, Set<ConstraintViolation<T>> constraintViolations) { ... })
            for (ConstraintViolation<Operator> cv : constraintViolations) {
                copySet.add(cv);
            }
            throw new ConstraintViolationException("Operator " + n.getName() + " violates constraints " + copySet, copySet);
        }
        OperatorMeta.PortMapping portMapping = n.getPortMapping();
        checkAttributeValueSerializable(n.getAttributes(), n.getName());
        // Check operator annotation
        if (n.operatorAnnotation != null) {
            // Check if partition property of the operator is being honored
            if (!n.operatorAnnotation.partitionable()) {
                // Check if any of the input ports have partition attributes set
                for (InputPortMeta pm : portMapping.inPortMap.values()) {
                    Boolean paralellPartition = pm.getValue(PortContext.PARTITION_PARALLEL);
                    if (paralellPartition) {
                        throw new ValidationException("Operator " + n.getName() + " is not partitionable but PARTITION_PARALLEL attribute is set");
                    }
                }
                // Check if the operator implements Partitioner
                if (n.getValue(OperatorContext.PARTITIONER) != null || n.attributes != null && !n.attributes.contains(OperatorContext.PARTITIONER) && Partitioner.class.isAssignableFrom(n.getOperator().getClass())) {
                    throw new ValidationException("Operator " + n.getName() + " provides partitioning capabilities but the annotation on the operator class declares it non partitionable!");
                }
            }
            // a multiple of application window count
            if (!n.operatorAnnotation.checkpointableWithinAppWindow()) {
                if (n.getValue(OperatorContext.CHECKPOINT_WINDOW_COUNT) % n.getValue(OperatorContext.APPLICATION_WINDOW_COUNT) != 0) {
                    throw new ValidationException("Operator " + n.getName() + " cannot be check-pointed between an application window " + "but the checkpoint-window-count " + n.getValue(OperatorContext.CHECKPOINT_WINDOW_COUNT) + " is not a multiple application-window-count " + n.getValue(OperatorContext.APPLICATION_WINDOW_COUNT));
                }
            }
        }
        // check that non-optional ports are connected
        for (InputPortMeta pm : portMapping.inPortMap.values()) {
            checkAttributeValueSerializable(pm.getAttributes(), n.getName() + "." + pm.getPortName());
            StreamMeta sm = n.inputStreams.get(pm);
            if (sm == null) {
                if ((pm.portAnnotation == null || !pm.portAnnotation.optional()) && pm.classDeclaringHiddenPort == null) {
                    throw new ValidationException("Input port connection required: " + n.name + "." + pm.getPortName());
                }
            } else {
                if (pm.classDeclaringHiddenPort != null) {
                    throw new ValidationException(String.format("Invalid port connected: %s.%s is hidden by %s.%s", pm.classDeclaringHiddenPort.getName(), pm.getPortName(), pm.operatorMeta.getOperator().getClass().getName(), pm.getPortName()));
                }
                // check locality constraints
                DAG.Locality locality = sm.getLocality();
                if (locality == DAG.Locality.THREAD_LOCAL) {
                    if (n.inputStreams.size() > 1) {
                        validateThreadLocal(n);
                    }
                }
                if (pm.portAnnotation != null && pm.portAnnotation.schemaRequired()) {
                    //since schema is required, the port attribute TUPLE_CLASS should be present
                    if (pm.attributes.get(PortContext.TUPLE_CLASS) == null) {
                        throw new ValidationException("Attribute " + PortContext.TUPLE_CLASS.getName() + " missing on port : " + n.name + "." + pm.getPortName());
                    }
                }
            }
        }
        for (OutputPortMeta pm : portMapping.outPortMap.values()) {
            checkAttributeValueSerializable(pm.getAttributes(), n.getName() + "." + pm.getPortName());
            if (!n.outputStreams.containsKey(pm)) {
                if ((pm.portAnnotation != null && !pm.portAnnotation.optional()) && pm.classDeclaringHiddenPort == null) {
                    throw new ValidationException("Output port connection required: " + n.name + "." + pm.getPortName());
                }
            } else {
                //port is connected
                if (pm.classDeclaringHiddenPort != null) {
                    throw new ValidationException(String.format("Invalid port connected: %s.%s is hidden by %s.%s", pm.classDeclaringHiddenPort.getName(), pm.getPortName(), pm.operatorMeta.getOperator().getClass().getName(), pm.getPortName()));
                }
                if (pm.portAnnotation != null && pm.portAnnotation.schemaRequired()) {
                    //since schema is required, the port attribute TUPLE_CLASS should be present
                    if (pm.attributes.get(PortContext.TUPLE_CLASS) == null) {
                        throw new ValidationException("Attribute " + PortContext.TUPLE_CLASS.getName() + " missing on port : " + n.name + "." + pm.getPortName());
                    }
                }
            }
        }
    }
    ValidationContext validatonContext = new ValidationContext();
    for (OperatorMeta n : operators.values()) {
        if (n.nindex == null) {
            findStronglyConnected(n, validatonContext);
        }
    }
    if (!validatonContext.invalidCycles.isEmpty()) {
        throw new ValidationException("Loops in graph: " + validatonContext.invalidCycles);
    }
    List<List<String>> invalidDelays = new ArrayList<>();
    for (OperatorMeta n : rootOperators) {
        findInvalidDelays(n, invalidDelays, new Stack<OperatorMeta>());
    }
    if (!invalidDelays.isEmpty()) {
        throw new ValidationException("Invalid delays in graph: " + invalidDelays);
    }
    for (StreamMeta s : streams.values()) {
        if (s.source == null) {
            throw new ValidationException("Stream source not connected: " + s.getName());
        }
        if (s.sinks.isEmpty()) {
            throw new ValidationException("Stream sink not connected: " + s.getName());
        }
    }
    // Validate root operators are input operators
    for (OperatorMeta om : this.rootOperators) {
        if (!(om.getOperator() instanceof InputOperator)) {
            throw new ValidationException(String.format("Root operator: %s is not a Input operator", om.getName()));
        }
    }
    // processing mode
    Set<OperatorMeta> visited = Sets.newHashSet();
    for (OperatorMeta om : this.rootOperators) {
        validateProcessingMode(om, visited);
    }
    validateAffinityRules();
}