Examples with OperatorMeta com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta used on opensource projects

Example 66 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class PhysicalPlan method removeLogicalStream.

/**
   * Remove physical representation of given stream. Operators that are affected
   * in the execution layer will be added to the set. This method does not
   * automatically remove operators from the plan.
   *
   * @param sm
   */
public void removeLogicalStream(StreamMeta sm) {
    // remove incoming connections for logical stream
    for (InputPortMeta ipm : sm.getSinks()) {
        OperatorMeta om = ipm.getOperatorMeta();
        PMapping m = this.logicalToPTOperator.get(om);
        if (m == null) {
            throw new AssertionError("Unknown operator " + om);
        }
        for (PTOperator oper : m.partitions) {
            List<PTInput> inputsCopy = Lists.newArrayList(oper.inputs);
            for (PTInput input : oper.inputs) {
                if (input.logicalStream == sm) {
                    input.source.sinks.remove(input);
                    inputsCopy.remove(input);
                    undeployOpers.add(oper);
                    deployOpers.add(oper);
                }
            }
            oper.inputs = inputsCopy;
        }
    }
    // remove outgoing connections for logical stream
    PMapping m = this.logicalToPTOperator.get(sm.getSource().getOperatorMeta());
    for (PTOperator oper : m.partitions) {
        List<PTOutput> outputsCopy = Lists.newArrayList(oper.outputs);
        for (PTOutput out : oper.outputs) {
            if (out.logicalStream == sm) {
                for (PTInput input : out.sinks) {
                    PTOperator downstreamOper = input.source.source;
                    downstreamOper.inputs.remove(input);
                    Set<PTOperator> deps = this.getDependents(Collections.singletonList(downstreamOper));
                    undeployOpers.addAll(deps);
                    deployOpers.addAll(deps);
                }
                outputsCopy.remove(out);
                undeployOpers.add(oper);
                deployOpers.add(oper);
            }
        }
        oper.outputs = outputsCopy;
    }
}

Example 67 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class PlanModifier method setOperatorProperty.

/**
   * Set the property on a new operator. Since this is only intended to modify
   * previously added operators, no change to the physical plan is required.
   *
   * @param operatorName
   * @param propertyName
   * @param propertyValue
   */
public void setOperatorProperty(String operatorName, String propertyName, String propertyValue) {
    OperatorMeta om = assertGetOperator(operatorName);
    if (physicalPlan != null) {
        for (PTOperator oper : physicalPlan.getOperators(om)) {
            if (!physicalPlan.newOpers.containsKey(oper)) {
                throw new ValidationException("Properties can only be set on new operators: " + om + " " + propertyName + " " + propertyValue);
            }
        }
    }
    Map<String, String> props = Collections.singletonMap(propertyName, propertyValue);
    LogicalPlanConfiguration.setOperatorProperties(om.getOperator(), props);
}

Example 68 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class StreamMapping method addSlidingUnifiers.

private void addSlidingUnifiers() {
    OperatorMeta sourceOM = streamMeta.getSource().getOperatorMeta();
    if (sourceOM.getAttributes().contains(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT)) {
        if (sourceOM.getValue(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT) < sourceOM.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT)) {
            plan.undeployOpers.addAll(slidingUnifiers);
            slidingUnifiers.clear();
            List<PTOutput> newUpstream = Lists.newArrayList();
            PTOperator slidingUnifier;
            for (PTOutput source : upstream) {
                slidingUnifier = StreamMapping.createSlidingUnifier(streamMeta, plan, sourceOM.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT), sourceOM.getValue(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT));
                addInput(slidingUnifier, source, null);
                this.slidingUnifiers.add(slidingUnifier);
                newUpstream.add(slidingUnifier.outputs.get(0));
            }
            upstream.clear();
            upstream.addAll(newUpstream);
        } else {
            LOG.warn("Sliding Window Count {} should be less than APPLICATION WINDOW COUNT {}", sourceOM.getValue(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT), sourceOM.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT));
        }
    }
}

Example 69 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class StreamMapping method createUnifier.

public static PTOperator createUnifier(StreamMeta streamMeta, PhysicalPlan plan) {
    OperatorMeta um = streamMeta.getSource().getUnifierMeta();
    PTOperator pu = plan.newOperator(um, um.getName());
    Operator unifier = um.getOperator();
    PortMappingDescriptor mergeDesc = new PortMappingDescriptor();
    Operators.describe(unifier, mergeDesc);
    if (mergeDesc.outputPorts.size() != 1) {
        throw new AssertionError("Unifier must have a single output port, instead found : " + mergeDesc.outputPorts);
    }
    pu.unifiedOperatorMeta = streamMeta.getSource().getOperatorMeta();
    pu.outputs.add(new PTOutput(mergeDesc.outputPorts.keySet().iterator().next(), streamMeta, pu));
    plan.newOpers.put(pu, unifier);
    return pu;
}

Example 70 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class StramWebServices method getLogicalOperator.

@GET
@Path(PATH_LOGICAL_PLAN_OPERATORS + "/{operatorName}")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getLogicalOperator(@PathParam("operatorName") String operatorName) throws Exception {
    init();
    OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName);
    if (logicalOperator == null) {
        throw new NotFoundException();
    }
    LogicalOperatorInfo logicalOperatorInfo = dagManager.getLogicalOperatorInfo(operatorName);
    return new JSONObject(objectMapper.writeValueAsString(logicalOperatorInfo));
}