Examples with OperatorMeta com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta used on opensource projects

Example 16 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class PlanModifier method addOperator.

/**
   * Add operator to logical plan.
   * @param name
   */
public void addOperator(String name, Operator operator) {
    logicalPlan.addOperator(name, operator);
    // add to physical plan after all changes are done
    if (physicalPlan != null) {
        OperatorMeta om = logicalPlan.getMeta(operator);
        physicalPlan.addLogicalOperator(om);
    }
}

Example 17 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class PlanModifier method getInputPort.

private InputPort<?> getInputPort(String operName, String portName) {
    OperatorMeta om = assertGetOperator(operName);
    Operators.PortMappingDescriptor portMap = new Operators.PortMappingDescriptor();
    Operators.describe(om.getOperator(), portMap);
    PortContextPair<InputPort<?>> port = portMap.inputPorts.get(portName);
    if (port == null) {
        throw new AssertionError(String.format("Invalid port %s (%s)", portName, om));
    }
    return port.component;
}

Example 18 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class PhysicalPlan method onStatusUpdate.

public void onStatusUpdate(PTOperator oper) {
    for (StatsListener l : oper.statsListeners) {
        final StatsListener.Response rsp = l.processStats(oper.stats);
        if (rsp != null) {
            //LOG.debug("Response to processStats = {}", rsp.repartitionRequired);
            oper.loadIndicator = rsp.loadIndicator;
            if (rsp.repartitionRequired) {
                final OperatorMeta om = oper.getOperatorMeta();
                // concurrent heartbeat processing
                if (this.pendingRepartition.putIfAbsent(om, om) != null) {
                    LOG.debug("Skipping repartitioning for {} load {}", oper, oper.loadIndicator);
                } else {
                    LOG.debug("Scheduling repartitioning for {} load {}", oper, oper.loadIndicator);
                    // hand over to monitor thread
                    Runnable r = new Runnable() {

                        @Override
                        public void run() {
                            redoPartitions(logicalToPTOperator.get(om), rsp.repartitionNote);
                            pendingRepartition.remove(om);
                        }
                    };
                    ctx.dispatch(r);
                }
            }
            if (rsp.operatorRequests != null) {
                for (OperatorRequest cmd : rsp.operatorRequests) {
                    StramToNodeRequest request = new StramToNodeRequest();
                    request.operatorId = oper.getId();
                    request.requestType = StramToNodeRequest.RequestType.CUSTOM;
                    request.cmd = cmd;
                    ctx.addOperatorRequest(oper, request);
                }
            }
            // for backward compatibility
            if (rsp.operatorCommands != null) {
                for (@SuppressWarnings("deprecation") com.datatorrent.api.StatsListener.OperatorCommand cmd : rsp.operatorCommands) {
                    StramToNodeRequest request = new StramToNodeRequest();
                    request.operatorId = oper.getId();
                    request.requestType = StramToNodeRequest.RequestType.CUSTOM;
                    OperatorCommandConverter converter = new OperatorCommandConverter();
                    converter.cmd = cmd;
                    request.cmd = converter;
                    ctx.addOperatorRequest(oper, request);
                }
            }
        }
    }
}

Example 19 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class PhysicalPlan method setAntiAffinityForContainers.

public void setAntiAffinityForContainers(LogicalPlan dag, Collection<AffinityRule> affinityRules, Map<PTOperator, PTContainer> operatorContainerMap) {
    for (AffinityRule rule : affinityRules) {
        if (rule.getOperatorsList() != null && rule.getType() == Type.ANTI_AFFINITY) {
            for (int i = 0; i < rule.getOperatorsList().size() - 1; i++) {
                for (int j = i + 1; j < rule.getOperatorsList().size(); j++) {
                    OperatorPair operators = new OperatorPair(rule.getOperatorsList().get(i), rule.getOperatorsList().get(j));
                    PMapping firstPMapping = logicalToPTOperator.get(dag.getOperatorMeta(operators.first));
                    OperatorMeta opMeta = dag.getOperatorMeta(operators.second);
                    PMapping secondMapping = logicalToPTOperator.get(opMeta);
                    for (PTOperator firstPtOperator : firstPMapping.partitions) {
                        PTContainer firstContainer = operatorContainerMap.get(firstPtOperator);
                        for (PTOperator secondPtOperator : secondMapping.partitions) {
                            PTContainer secondContainer = operatorContainerMap.get(secondPtOperator);
                            if (firstContainer == secondContainer || firstContainer.getStrictAntiPrefs().contains(secondContainer)) {
                                continue;
                            }
                            if (rule.isRelaxLocality()) {
                                firstContainer.getPreferredAntiPrefs().add(secondContainer);
                                secondContainer.getPreferredAntiPrefs().add(firstContainer);
                            } else {
                                firstContainer.getStrictAntiPrefs().add(secondContainer);
                                secondContainer.getStrictAntiPrefs().add(firstContainer);
                            }
                        }
                    }
                }
            }
        }
    }
}

Example 20 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class PhysicalPlan method updatePartitionsInfoForPersistOperator.

private void updatePartitionsInfoForPersistOperator(LogicalPlan dag) {
    // StreamCodec for persist operator
    try {
        for (OperatorMeta n : dag.getAllOperators()) {
            for (StreamMeta s : n.getOutputStreams().values()) {
                if (s.getPersistOperator() != null) {
                    InputPortMeta persistInputPort = s.getPersistOperatorInputPort();
                    StreamCodecWrapperForPersistance<?> persistCodec = (StreamCodecWrapperForPersistance<?>) persistInputPort.getStreamCodec();
                    if (persistCodec == null) {
                        continue;
                    }
                    // Logging is enabled for the stream
                    for (InputPortMeta portMeta : s.getSinksToPersist()) {
                        updatePersistOperatorWithSinkPartitions(persistInputPort, s.getPersistOperator(), persistCodec, portMeta);
                    }
                }
                // Check partitioning for persist operators per sink too
                for (Map.Entry<InputPortMeta, InputPortMeta> entry : s.sinkSpecificPersistInputPortMap.entrySet()) {
                    InputPortMeta persistInputPort = entry.getValue();
                    StreamCodec<?> streamCodec = persistInputPort.getStreamCodec();
                    if (streamCodec != null && streamCodec instanceof StreamCodecWrapperForPersistance) {
                        updatePersistOperatorWithSinkPartitions(persistInputPort, s.sinkSpecificPersistOperatorMap.get(entry.getKey()), (StreamCodecWrapperForPersistance<?>) streamCodec, entry.getKey());
                    }
                }
            }
        }
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}