Examples with InputOperator com.datatorrent.api.InputOperator used on opensource projects

Example 1 with InputOperator

use of com.datatorrent.api.InputOperator in project apex-core by apache.

the class LogicalPlan method validate.

/**
   * Validate the plan. Includes checks that required ports are connected,
   * required configuration parameters specified, graph free of cycles etc.
   *
   * @throws ConstraintViolationException
   */
public void validate() throws ConstraintViolationException {
    ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
    Validator validator = factory.getValidator();
    checkAttributeValueSerializable(this.getAttributes(), DAG.class.getName());
    // clear oioRoot values in all operators
    for (OperatorMeta n : operators.values()) {
        n.oioRoot = null;
    }
    // clear visited on all operators
    for (OperatorMeta n : operators.values()) {
        n.nindex = null;
        n.lowlink = null;
        // validate configuration
        Set<ConstraintViolation<Operator>> constraintViolations = validator.validate(n.getOperator());
        if (!constraintViolations.isEmpty()) {
            Set<ConstraintViolation<?>> copySet = new HashSet<>(constraintViolations.size());
            // (should be public <T> ConstraintViolationException(String message, Set<ConstraintViolation<T>> constraintViolations) { ... })
            for (ConstraintViolation<Operator> cv : constraintViolations) {
                copySet.add(cv);
            }
            throw new ConstraintViolationException("Operator " + n.getName() + " violates constraints " + copySet, copySet);
        }
        OperatorMeta.PortMapping portMapping = n.getPortMapping();
        checkAttributeValueSerializable(n.getAttributes(), n.getName());
        // Check operator annotation
        if (n.operatorAnnotation != null) {
            // Check if partition property of the operator is being honored
            if (!n.operatorAnnotation.partitionable()) {
                // Check if any of the input ports have partition attributes set
                for (InputPortMeta pm : portMapping.inPortMap.values()) {
                    Boolean paralellPartition = pm.getValue(PortContext.PARTITION_PARALLEL);
                    if (paralellPartition) {
                        throw new ValidationException("Operator " + n.getName() + " is not partitionable but PARTITION_PARALLEL attribute is set");
                    }
                }
                // Check if the operator implements Partitioner
                if (n.getValue(OperatorContext.PARTITIONER) != null || n.attributes != null && !n.attributes.contains(OperatorContext.PARTITIONER) && Partitioner.class.isAssignableFrom(n.getOperator().getClass())) {
                    throw new ValidationException("Operator " + n.getName() + " provides partitioning capabilities but the annotation on the operator class declares it non partitionable!");
                }
            }
            // a multiple of application window count
            if (!n.operatorAnnotation.checkpointableWithinAppWindow()) {
                if (n.getValue(OperatorContext.CHECKPOINT_WINDOW_COUNT) % n.getValue(OperatorContext.APPLICATION_WINDOW_COUNT) != 0) {
                    throw new ValidationException("Operator " + n.getName() + " cannot be check-pointed between an application window " + "but the checkpoint-window-count " + n.getValue(OperatorContext.CHECKPOINT_WINDOW_COUNT) + " is not a multiple application-window-count " + n.getValue(OperatorContext.APPLICATION_WINDOW_COUNT));
                }
            }
        }
        // check that non-optional ports are connected
        for (InputPortMeta pm : portMapping.inPortMap.values()) {
            checkAttributeValueSerializable(pm.getAttributes(), n.getName() + "." + pm.getPortName());
            StreamMeta sm = n.inputStreams.get(pm);
            if (sm == null) {
                if ((pm.portAnnotation == null || !pm.portAnnotation.optional()) && pm.classDeclaringHiddenPort == null) {
                    throw new ValidationException("Input port connection required: " + n.name + "." + pm.getPortName());
                }
            } else {
                if (pm.classDeclaringHiddenPort != null) {
                    throw new ValidationException(String.format("Invalid port connected: %s.%s is hidden by %s.%s", pm.classDeclaringHiddenPort.getName(), pm.getPortName(), pm.operatorMeta.getOperator().getClass().getName(), pm.getPortName()));
                }
                // check locality constraints
                DAG.Locality locality = sm.getLocality();
                if (locality == DAG.Locality.THREAD_LOCAL) {
                    if (n.inputStreams.size() > 1) {
                        validateThreadLocal(n);
                    }
                }
                if (pm.portAnnotation != null && pm.portAnnotation.schemaRequired()) {
                    //since schema is required, the port attribute TUPLE_CLASS should be present
                    if (pm.attributes.get(PortContext.TUPLE_CLASS) == null) {
                        throw new ValidationException("Attribute " + PortContext.TUPLE_CLASS.getName() + " missing on port : " + n.name + "." + pm.getPortName());
                    }
                }
            }
        }
        for (OutputPortMeta pm : portMapping.outPortMap.values()) {
            checkAttributeValueSerializable(pm.getAttributes(), n.getName() + "." + pm.getPortName());
            if (!n.outputStreams.containsKey(pm)) {
                if ((pm.portAnnotation != null && !pm.portAnnotation.optional()) && pm.classDeclaringHiddenPort == null) {
                    throw new ValidationException("Output port connection required: " + n.name + "." + pm.getPortName());
                }
            } else {
                //port is connected
                if (pm.classDeclaringHiddenPort != null) {
                    throw new ValidationException(String.format("Invalid port connected: %s.%s is hidden by %s.%s", pm.classDeclaringHiddenPort.getName(), pm.getPortName(), pm.operatorMeta.getOperator().getClass().getName(), pm.getPortName()));
                }
                if (pm.portAnnotation != null && pm.portAnnotation.schemaRequired()) {
                    //since schema is required, the port attribute TUPLE_CLASS should be present
                    if (pm.attributes.get(PortContext.TUPLE_CLASS) == null) {
                        throw new ValidationException("Attribute " + PortContext.TUPLE_CLASS.getName() + " missing on port : " + n.name + "." + pm.getPortName());
                    }
                }
            }
        }
    }
    ValidationContext validatonContext = new ValidationContext();
    for (OperatorMeta n : operators.values()) {
        if (n.nindex == null) {
            findStronglyConnected(n, validatonContext);
        }
    }
    if (!validatonContext.invalidCycles.isEmpty()) {
        throw new ValidationException("Loops in graph: " + validatonContext.invalidCycles);
    }
    List<List<String>> invalidDelays = new ArrayList<>();
    for (OperatorMeta n : rootOperators) {
        findInvalidDelays(n, invalidDelays, new Stack<OperatorMeta>());
    }
    if (!invalidDelays.isEmpty()) {
        throw new ValidationException("Invalid delays in graph: " + invalidDelays);
    }
    for (StreamMeta s : streams.values()) {
        if (s.source == null) {
            throw new ValidationException("Stream source not connected: " + s.getName());
        }
        if (s.sinks.isEmpty()) {
            throw new ValidationException("Stream sink not connected: " + s.getName());
        }
    }
    // Validate root operators are input operators
    for (OperatorMeta om : this.rootOperators) {
        if (!(om.getOperator() instanceof InputOperator)) {
            throw new ValidationException(String.format("Root operator: %s is not a Input operator", om.getName()));
        }
    }
    // processing mode
    Set<OperatorMeta> visited = Sets.newHashSet();
    for (OperatorMeta om : this.rootOperators) {
        validateProcessingMode(om, visited);
    }
    validateAffinityRules();
}

Example 2 with InputOperator

use of com.datatorrent.api.InputOperator in project apex-core by apache.

the class StreamingContainerAgent method createOperatorDeployInfo.

/**
   * Create deploy info for operator.
   * <p>
   *
   * @return {@link com.datatorrent.stram.api.OperatorDeployInfo}
   */
private OperatorDeployInfo createOperatorDeployInfo(PTOperator oper) {
    OperatorDeployInfo ndi;
    if (oper.isUnifier()) {
        UnifierDeployInfo udi = new UnifierDeployInfo();
        /* the constructor auto sets the type */
        try {
            udi.operatorAttributes = oper.getUnifiedOperatorMeta().getAttributes().clone();
        } catch (CloneNotSupportedException ex) {
            throw new RuntimeException("Cannot clone unifier attributes", ex);
        }
        ndi = udi;
    } else {
        ndi = new OperatorDeployInfo();
        Operator operator = oper.getOperatorMeta().getOperator();
        if (operator instanceof InputOperator) {
            ndi.type = OperatorType.INPUT;
            if (!oper.getInputs().isEmpty()) {
                //we check if any input port is connected which would make it a Generic operator.
                for (PTOperator.PTInput ptInput : oper.getInputs()) {
                    if (ptInput.logicalStream != null && ptInput.logicalStream.getSource() != null) {
                        ndi.type = OperatorType.GENERIC;
                        break;
                    }
                }
            }
        } else {
            ndi.type = OperatorType.GENERIC;
        }
    }
    Checkpoint checkpoint = oper.getRecoveryCheckpoint();
    ProcessingMode pm = oper.getOperatorMeta().getValue(OperatorContext.PROCESSING_MODE);
    if (pm == ProcessingMode.AT_MOST_ONCE || pm == ProcessingMode.EXACTLY_ONCE) {
        // TODO: following should be handled in the container at deploy time
        // for exactly once container should also purge previous checkpoint
        // whenever new checkpoint is written.
        StorageAgent agent = oper.getOperatorMeta().getAttributes().get(OperatorContext.STORAGE_AGENT);
        if (agent == null) {
            agent = initCtx.getValue(OperatorContext.STORAGE_AGENT);
        }
        // pick checkpoint most recently written
        try {
            long[] windowIds = agent.getWindowIds(oper.getId());
            long checkpointId = Stateless.WINDOW_ID;
            for (long windowId : windowIds) {
                if (windowId > checkpointId) {
                    checkpointId = windowId;
                }
            }
            if (checkpoint == null || checkpoint.windowId != checkpointId) {
                checkpoint = new Checkpoint(checkpointId, 0, 0);
            }
        } catch (Exception e) {
            throw new RuntimeException("Failed to determine checkpoint window id " + oper, e);
        }
    }
    LOG.debug("{} recovery checkpoint {}", oper, checkpoint);
    ndi.checkpoint = checkpoint;
    ndi.name = oper.getOperatorMeta().getName();
    ndi.id = oper.getId();
    try {
        // clone map before modifying it
        ndi.contextAttributes = oper.getOperatorMeta().getAttributes().clone();
    } catch (CloneNotSupportedException ex) {
        throw new RuntimeException("Cannot clone operator attributes", ex);
    }
    if (oper.isOperatorStateLess()) {
        ndi.contextAttributes.put(OperatorContext.STATELESS, true);
    }
    return ndi;
}