Examples with OperatorMeta com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta used on opensource projects

Example 21 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class PlanModifier method removeOperator.

/**
   * Remove named operator and any outgoing streams.
   * @param name
   */
public void removeOperator(String name) {
    OperatorMeta om = logicalPlan.getOperatorMeta(name);
    if (om == null) {
        return;
    }
    if (!om.getInputStreams().isEmpty()) {
        for (Map.Entry<InputPortMeta, StreamMeta> input : om.getInputStreams().entrySet()) {
            if (input.getValue().getSinks().size() == 1) {
                // would result in dangling stream
                String msg = String.format("Operator %s connected to input streams %s", om.getName(), om.getInputStreams());
                throw new ValidationException(msg);
            }
        }
    }
    if (!om.getOutputStreams().isEmpty()) {
        String msg = String.format("Operator %s connected to output streams %s", om.getName(), om.getOutputStreams());
        throw new ValidationException(msg);
    }
    /*
    // remove associated sinks
    Map<InputPortMeta, StreamMeta> inputStreams = om.getInputStreams();
    for (Map.Entry<InputPortMeta, StreamMeta> e : inputStreams.entrySet()) {
      if (e.getKey().getOperatorMeta() == om) {
        if (e.getValue().getSinks().size() == 1) {
          // drop stream
          e.getValue().remove();
        }
      }
    }
    */
    logicalPlan.removeOperator(om.getOperator());
    if (physicalPlan != null) {
        physicalPlan.removeLogicalOperator(om);
    }
}

Example 22 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class StreamMapping method createSlidingUnifier.

public static PTOperator createSlidingUnifier(StreamMeta streamMeta, PhysicalPlan plan, int operatorApplicationWindowCount, int slidingWindowCount) {
    int gcd = IntMath.gcd(operatorApplicationWindowCount, slidingWindowCount);
    OperatorMeta um = streamMeta.getSource().getSlidingUnifier(operatorApplicationWindowCount / gcd, gcd, slidingWindowCount / gcd);
    PTOperator pu = plan.newOperator(um, um.getName());
    Operator unifier = um.getOperator();
    PortMappingDescriptor mergeDesc = new PortMappingDescriptor();
    Operators.describe(unifier, mergeDesc);
    if (mergeDesc.outputPorts.size() != 1) {
        throw new AssertionError("Unifier must have a single output port, instead found : " + mergeDesc.outputPorts);
    }
    pu.unifiedOperatorMeta = streamMeta.getSource().getOperatorMeta();
    pu.outputs.add(new PTOutput(mergeDesc.outputPorts.keySet().iterator().next(), streamMeta, pu));
    plan.newOpers.put(pu, unifier);
    return pu;
}

Example 23 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class StreamingContainerManager method updateRecoveryCheckpoints.

/**
   * Compute checkpoints required for a given operator instance to be recovered.
   * This is done by looking at checkpoints available for downstream dependencies first,
   * and then selecting the most recent available checkpoint that is smaller than downstream.
   *
   * @param operator Operator instance for which to find recovery checkpoint
   * @param ctx      Context into which to collect traversal info
   */
public void updateRecoveryCheckpoints(PTOperator operator, UpdateCheckpointsContext ctx, boolean recovery) {
    if (operator.getRecoveryCheckpoint().windowId < ctx.committedWindowId.longValue()) {
        ctx.committedWindowId.setValue(operator.getRecoveryCheckpoint().windowId);
    }
    if (operator.getState() == PTOperator.State.ACTIVE && (ctx.currentTms - operator.stats.lastWindowIdChangeTms) > operator.stats.windowProcessingTimeoutMillis) {
        // if the checkpoint is ahead, then it is not blocked but waiting for activation (state-less recovery, at-most-once)
        if (ctx.committedWindowId.longValue() >= operator.getRecoveryCheckpoint().windowId && !recovery) {
            LOG.warn("Marking operator {} blocked committed window {}, recovery window {}, current time {}, last window id change time {}, window processing timeout millis {}", operator, Codec.getStringWindowId(ctx.committedWindowId.longValue()), Codec.getStringWindowId(operator.getRecoveryCheckpoint().windowId), ctx.currentTms, operator.stats.lastWindowIdChangeTms, operator.stats.windowProcessingTimeoutMillis);
            ctx.blocked.add(operator);
        }
    }
    // the most recent checkpoint eligible for recovery based on downstream state
    Checkpoint maxCheckpoint = Checkpoint.INITIAL_CHECKPOINT;
    Set<OperatorMeta> checkpointGroup = ctx.checkpointGroups.get(operator.getOperatorMeta());
    if (checkpointGroup == null) {
        checkpointGroup = Collections.singleton(operator.getOperatorMeta());
    }
    // find intersection of checkpoints that group can collectively move to
    TreeSet<Checkpoint> commonCheckpoints = new TreeSet<>(new Checkpoint.CheckpointComparator());
    synchronized (operator.checkpoints) {
        commonCheckpoints.addAll(operator.checkpoints);
    }
    Set<PTOperator> groupOpers = new HashSet<>(checkpointGroup.size());
    boolean pendingDeploy = operator.getState() == PTOperator.State.PENDING_DEPLOY;
    if (checkpointGroup.size() > 1) {
        for (OperatorMeta om : checkpointGroup) {
            Collection<PTOperator> operators = plan.getAllOperators(om);
            Collection<PTOperator> unifiers = getUnifiersInCheckpointGroup(operators);
            operators.addAll(unifiers);
            for (PTOperator groupOper : operators) {
                synchronized (groupOper.checkpoints) {
                    commonCheckpoints.retainAll(groupOper.checkpoints);
                }
                // visit all downstream operators of the group
                ctx.visited.add(groupOper);
                groupOpers.add(groupOper);
                pendingDeploy |= groupOper.getState() == PTOperator.State.PENDING_DEPLOY;
            }
        }
        // highest common checkpoint
        if (!commonCheckpoints.isEmpty()) {
            maxCheckpoint = commonCheckpoints.last();
        }
    } else {
        // without logical grouping, treat partitions as independent
        // this is especially important for parallel partitioning
        ctx.visited.add(operator);
        groupOpers.add(operator);
        maxCheckpoint = operator.getRecentCheckpoint();
        if (ctx.recovery && maxCheckpoint.windowId == Stateless.WINDOW_ID && operator.isOperatorStateLess()) {
            long currentWindowId = WindowGenerator.getWindowId(ctx.currentTms, this.vars.windowStartMillis, this.getLogicalPlan().getValue(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS));
            maxCheckpoint = new Checkpoint(currentWindowId, 0, 0);
        }
    }
    // DFS downstream operators
    for (PTOperator groupOper : groupOpers) {
        for (PTOperator.PTOutput out : groupOper.getOutputs()) {
            for (PTOperator.PTInput sink : out.sinks) {
                PTOperator sinkOperator = sink.target;
                if (groupOpers.contains(sinkOperator)) {
                    // downstream operator within group
                    continue;
                }
                if (!ctx.visited.contains(sinkOperator)) {
                    // downstream traversal
                    updateRecoveryCheckpoints(sinkOperator, ctx, recovery);
                }
                // when dynamically adding new operators
                if (sinkOperator.getRecoveryCheckpoint().windowId >= operator.getRecoveryCheckpoint().windowId) {
                    maxCheckpoint = Checkpoint.min(maxCheckpoint, sinkOperator.getRecoveryCheckpoint());
                }
                if (ctx.blocked.contains(sinkOperator)) {
                    if (sinkOperator.stats.getCurrentWindowId() == operator.stats.getCurrentWindowId()) {
                        // downstream operator is blocked by this operator
                        ctx.blocked.remove(sinkOperator);
                    }
                }
            }
        }
    }
    // find the common checkpoint that is <= downstream recovery checkpoint
    if (!commonCheckpoints.contains(maxCheckpoint)) {
        if (!commonCheckpoints.isEmpty()) {
            maxCheckpoint = Objects.firstNonNull(commonCheckpoints.floor(maxCheckpoint), maxCheckpoint);
        }
    }
    for (PTOperator groupOper : groupOpers) {
        // checkpoint frozen during deployment
        if (!pendingDeploy || ctx.recovery) {
            // remove previous checkpoints
            Checkpoint c1 = Checkpoint.INITIAL_CHECKPOINT;
            LinkedList<Checkpoint> checkpoints = groupOper.checkpoints;
            synchronized (checkpoints) {
                if (!checkpoints.isEmpty() && (checkpoints.getFirst()).windowId <= maxCheckpoint.windowId) {
                    c1 = checkpoints.getFirst();
                    Checkpoint c2;
                    while (checkpoints.size() > 1 && ((c2 = checkpoints.get(1)).windowId) <= maxCheckpoint.windowId) {
                        checkpoints.removeFirst();
                        //LOG.debug("Checkpoint to delete: operator={} windowId={}", operator.getName(), c1);
                        this.purgeCheckpoints.add(new Pair<>(groupOper, c1.windowId));
                        c1 = c2;
                    }
                } else {
                    if (ctx.recovery && checkpoints.isEmpty() && groupOper.isOperatorStateLess()) {
                        LOG.debug("Adding checkpoint for stateless operator {} {}", groupOper, Codec.getStringWindowId(maxCheckpoint.windowId));
                        c1 = groupOper.addCheckpoint(maxCheckpoint.windowId, this.vars.windowStartMillis);
                    }
                }
            }
            //LOG.debug("Operator {} checkpoints: commit {} recent {}", new Object[] {operator.getName(), c1, operator.checkpoints});
            groupOper.setRecoveryCheckpoint(c1);
        } else {
            LOG.debug("Skipping checkpoint update {} during {}", groupOper, groupOper.getState());
        }
    }
}

Example 24 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class DelayOperatorTest method testInvalidDelayDetection.

@Test
public void testInvalidDelayDetection() {
    LogicalPlan dag = new LogicalPlan();
    GenericTestOperator opB = dag.addOperator("B", GenericTestOperator.class);
    GenericTestOperator opC = dag.addOperator("C", GenericTestOperator.class);
    GenericTestOperator opD = dag.addOperator("D", GenericTestOperator.class);
    DefaultDelayOperator<Object> opDelay = dag.addOperator("opDelay", DefaultDelayOperator.class);
    dag.addStream("BtoC", opB.outport1, opC.inport1);
    dag.addStream("CtoD", opC.outport1, opD.inport1);
    dag.addStream("CtoDelay", opC.outport2, opDelay.input);
    dag.addStream("DelayToD", opDelay.output, opD.inport2);
    List<List<String>> invalidDelays = new ArrayList<>();
    dag.findInvalidDelays(dag.getMeta(opB), invalidDelays, new Stack<OperatorMeta>());
    assertEquals("operator invalid delay", 1, invalidDelays.size());
    try {
        dag.validate();
        fail("validation should fail");
    } catch (ValidationException e) {
    // expected
    }
    dag = new LogicalPlan();
    opB = dag.addOperator("B", GenericTestOperator.class);
    opC = dag.addOperator("C", GenericTestOperator.class);
    opD = dag.addOperator("D", GenericTestOperator.class);
    opDelay = dag.addOperator("opDelay", DefaultDelayOperator.class);
    dag.setOperatorAttribute(opDelay, Context.OperatorContext.APPLICATION_WINDOW_COUNT, 2);
    dag.addStream("BtoC", opB.outport1, opC.inport1);
    dag.addStream("CtoD", opC.outport1, opD.inport1);
    dag.addStream("CtoDelay", opC.outport2, opDelay.input);
    dag.addStream("DelayToC", opDelay.output, opC.inport2);
    invalidDelays = new ArrayList<>();
    dag.findInvalidDelays(dag.getMeta(opB), invalidDelays, new Stack<OperatorMeta>());
    assertEquals("operator invalid delay", 1, invalidDelays.size());
    try {
        dag.validate();
        fail("validation should fail");
    } catch (ValidationException e) {
    // expected
    }
    dag = new LogicalPlan();
    opB = dag.addOperator("B", GenericTestOperator.class);
    opC = dag.addOperator("C", GenericTestOperator.class);
    opD = dag.addOperator("D", GenericTestOperator.class);
    opDelay = dag.addOperator("opDelay", DefaultDelayOperator.class);
    dag.addStream("BtoC", opB.outport1, opC.inport1);
    dag.addStream("CtoD", opC.outport1, opD.inport1);
    dag.addStream("CtoDelay", opC.outport2, opDelay.input).setLocality(DAG.Locality.THREAD_LOCAL);
    dag.addStream("DelayToC", opDelay.output, opC.inport2).setLocality(DAG.Locality.THREAD_LOCAL);
    try {
        dag.validate();
        fail("validation should fail");
    } catch (ValidationException e) {
    // expected
    }
}

Example 25 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class LogicalPlanConfigurationTest method testModuleUnifierLevelAttributes.

@Test
@SuppressWarnings({ "UnnecessaryBoxing", "AssertEqualsBetweenInconvertibleTypes" })
public void testModuleUnifierLevelAttributes() {
    class DummyOperator extends BaseOperator {

        int prop;

        public transient DefaultInputPort<Integer> input = new DefaultInputPort<Integer>() {

            @Override
            public void process(Integer tuple) {
                LOG.debug(tuple.intValue() + " processed");
                output.emit(tuple);
            }
        };

        public transient DefaultOutputPort<Integer> output = new DefaultOutputPort<>();
    }
    class DummyOutputOperator extends BaseOperator {

        int prop;

        public transient DefaultInputPort<Integer> input = new DefaultInputPort<Integer>() {

            @Override
            public void process(Integer tuple) {
                LOG.debug(tuple.intValue() + " processed");
            }
        };
    }
    class TestUnifierAttributeModule implements Module {

        public transient ProxyInputPort<Integer> moduleInput = new ProxyInputPort<>();

        public transient ProxyOutputPort<Integer> moduleOutput = new Module.ProxyOutputPort<>();

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            DummyOperator dummyOperator = dag.addOperator("DummyOperator", new DummyOperator());
            dag.setOperatorAttribute(dummyOperator, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<DummyOperator>(3));
            dag.setUnifierAttribute(dummyOperator.output, OperatorContext.TIMEOUT_WINDOW_COUNT, 2);
            moduleInput.set(dummyOperator.input);
            moduleOutput.set(dummyOperator.output);
        }
    }
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            Module m1 = dag.addModule("TestModule", new TestUnifierAttributeModule());
            DummyOutputOperator dummyOutputOperator = dag.addOperator("DummyOutputOperator", new DummyOutputOperator());
            dag.addStream("Module To Operator", ((TestUnifierAttributeModule) m1).moduleOutput, dummyOutputOperator.input);
        }
    };
    String appName = "UnifierApp";
    LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new MockStorageAgent());
    dagBuilder.prepareDAG(dag, app, appName);
    LogicalPlan.OperatorMeta ometa = dag.getOperatorMeta("TestModule$DummyOperator");
    LogicalPlan.OperatorMeta om = null;
    for (Map.Entry<LogicalPlan.OutputPortMeta, LogicalPlan.StreamMeta> entry : ometa.getOutputStreams().entrySet()) {
        if (entry.getKey().getPortName().equals("output")) {
            om = entry.getKey().getUnifierMeta();
        }
    }
    /*
     * Verify the attribute value after preparing DAG.
     */
    Assert.assertNotNull(om);
    Assert.assertEquals("", Integer.valueOf(2), om.getValue(Context.OperatorContext.TIMEOUT_WINDOW_COUNT));
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    List<PTContainer> containers = plan.getContainers();
    LogicalPlan.OperatorMeta operatorMeta = null;
    for (PTContainer container : containers) {
        List<PTOperator> operators = container.getOperators();
        for (PTOperator operator : operators) {
            if (operator.isUnifier()) {
                operatorMeta = operator.getOperatorMeta();
            }
        }
    }
    /*
     * Verify attribute after physical plan creation with partitioned operators.
     */
    Assert.assertEquals("", Integer.valueOf(2), operatorMeta.getValue(OperatorContext.TIMEOUT_WINDOW_COUNT));
}