Examples with InputPortMeta com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta used on opensource projects

Example 6 with InputPortMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta in project apex-core by apache.

the class StreamMapping method redoMapping.

/**
   * rebuild the tree, which may cause more changes to execution layer than need be
   * TODO: investigate incremental logic
   */
private void redoMapping() {
    Set<Pair<PTOperator, InputPortMeta>> downstreamOpers = Sets.newHashSet();
    // figure out the downstream consumers
    for (InputPortMeta ipm : streamMeta.getSinks()) {
        // skipped for parallel partitions - those are handled elsewhere
        if (!ipm.getValue(PortContext.PARTITION_PARALLEL) && plan.hasMapping(ipm.getOperatorMeta())) {
            List<PTOperator> partitions = plan.getOperators(ipm.getOperatorMeta());
            for (PTOperator doper : partitions) {
                downstreamOpers.add(new Pair<>(doper, ipm));
            }
        }
    }
    if (!downstreamOpers.isEmpty()) {
        // unifiers are required
        for (PTOperator unifier : this.cascadingUnifiers) {
            detachUnifier(unifier);
        }
        if (this.finalUnifier != null) {
            detachUnifier(finalUnifier);
        }
        List<PTOperator> currentUnifiers = Lists.newArrayList(this.cascadingUnifiers);
        this.cascadingUnifiers.clear();
        plan.undeployOpers.addAll(currentUnifiers);
        addSlidingUnifiers();
        int limit = streamMeta.getSource().getValue(PortContext.UNIFIER_LIMIT);
        boolean separateUnifiers = false;
        Integer lastId = null;
        for (InputPortMeta ipm : streamMeta.getSinks()) {
            Integer id = plan.getStreamCodecIdentifier(ipm.getStreamCodec());
            if (lastId == null) {
                lastId = id;
            } else if (!id.equals(lastId)) {
                separateUnifiers = true;
                break;
            }
        }
        List<PTOutput> unifierSources = this.upstream;
        Map<StreamCodec<?>, List<PTOutput>> cascadeUnifierSourcesMap = Maps.newHashMap();
        if (limit > 1 && this.upstream.size() > limit) {
            // cascading unifier
            if (!separateUnifiers) {
                unifierSources = setupCascadingUnifiers(this.upstream, currentUnifiers, limit, 0);
            } else {
                for (InputPortMeta ipm : streamMeta.getSinks()) {
                    StreamCodec<?> streamCodec = ipm.getStreamCodec();
                    if (!cascadeUnifierSourcesMap.containsKey(streamCodec)) {
                        unifierSources = setupCascadingUnifiers(this.upstream, currentUnifiers, limit, 0);
                        cascadeUnifierSourcesMap.put(streamCodec, unifierSources);
                    }
                }
            }
        }
        // remove remaining unifiers
        for (PTOperator oper : currentUnifiers) {
            plan.removePTOperator(oper);
        }
        // Directly getting attribute from map to know if it is set or not as it can be overriden by the input
        Boolean sourceSingleFinal = streamMeta.getSource().getAttributes().get(PortContext.UNIFIER_SINGLE_FINAL);
        // link the downstream operators with the unifiers
        for (Pair<PTOperator, InputPortMeta> doperEntry : downstreamOpers) {
            Map<LogicalPlan.InputPortMeta, PartitionKeys> partKeys = doperEntry.first.partitionKeys;
            PartitionKeys pks = partKeys != null ? partKeys.get(doperEntry.second) : null;
            Boolean sinkSingleFinal = doperEntry.second.getAttributes().get(PortContext.UNIFIER_SINGLE_FINAL);
            boolean lastSingle = (sinkSingleFinal != null) ? sinkSingleFinal : (sourceSingleFinal != null ? sourceSingleFinal.booleanValue() : PortContext.UNIFIER_SINGLE_FINAL.defaultValue);
            if (upstream.size() > 1) {
                // detach downstream from upstream operator for the case where no unifier existed previously
                for (PTOutput source : upstream) {
                    Iterator<PTInput> sinks = source.sinks.iterator();
                    while (sinks.hasNext()) {
                        PTInput sink = sinks.next();
                        if (sink.target == doperEntry.first) {
                            doperEntry.first.inputs.remove(sink);
                            sinks.remove();
                        }
                    }
                }
                if (!separateUnifiers && lastSingle) {
                    if (finalUnifier == null) {
                        finalUnifier = createUnifier(streamMeta, plan);
                    }
                    setInput(doperEntry.first, doperEntry.second, finalUnifier, (pks == null) || (pks.mask == 0) ? null : pks);
                    if (finalUnifier.inputs.isEmpty()) {
                        // set unifier inputs once, regardless how many downstream operators there are
                        for (PTOutput out : unifierSources) {
                            addInput(this.finalUnifier, out, null);
                        }
                    }
                } else {
                    // MxN partitioning: unifier per downstream partition
                    LOG.debug("MxN unifier for {} {} {}", new Object[] { doperEntry.first, doperEntry.second.getPortName(), pks });
                    PTOperator unifier = doperEntry.first.upstreamMerge.get(doperEntry.second);
                    if (unifier == null) {
                        unifier = createUnifier(streamMeta, plan);
                        doperEntry.first.upstreamMerge.put(doperEntry.second, unifier);
                        setInput(doperEntry.first, doperEntry.second, unifier, null);
                    }
                    // sources may change dynamically, rebuild inputs (as for cascading unifiers)
                    for (PTInput in : unifier.inputs) {
                        in.source.sinks.remove(in);
                    }
                    unifier.inputs.clear();
                    List<PTOutput> doperUnifierSources = unifierSources;
                    if (separateUnifiers) {
                        List<PTOutput> cascadeSources = cascadeUnifierSourcesMap.get(doperEntry.second.getStreamCodec());
                        if (cascadeSources != null) {
                            doperUnifierSources = cascadeSources;
                        }
                    }
                    // add new inputs
                    for (PTOutput out : doperUnifierSources) {
                        addInput(unifier, out, (pks == null) || (pks.mask == 0) ? null : pks);
                    }
                }
            } else {
                // no partitioning
                PTOperator unifier = doperEntry.first.upstreamMerge.remove(doperEntry.second);
                if (unifier != null) {
                    plan.removePTOperator(unifier);
                }
                setInput(doperEntry.first, doperEntry.second, upstream.get(0).source, pks);
            }
        }
        // 2) Downstream operators partitions are scaled up from one to multiple. (replaced by merged unifier)
        if (finalUnifier != null && finalUnifier.inputs.isEmpty()) {
            plan.removePTOperator(finalUnifier);
            finalUnifier = null;
        }
    }
}

Example 7 with InputPortMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta in project apex-core by apache.

the class PhysicalPlan method updatePartitionsInfoForPersistOperator.

private void updatePartitionsInfoForPersistOperator(LogicalPlan dag) {
    // StreamCodec for persist operator
    try {
        for (OperatorMeta n : dag.getAllOperators()) {
            for (StreamMeta s : n.getOutputStreams().values()) {
                if (s.getPersistOperator() != null) {
                    InputPortMeta persistInputPort = s.getPersistOperatorInputPort();
                    StreamCodecWrapperForPersistance<?> persistCodec = (StreamCodecWrapperForPersistance<?>) persistInputPort.getStreamCodec();
                    if (persistCodec == null) {
                        continue;
                    }
                    // Logging is enabled for the stream
                    for (InputPortMeta portMeta : s.getSinksToPersist()) {
                        updatePersistOperatorWithSinkPartitions(persistInputPort, s.getPersistOperator(), persistCodec, portMeta);
                    }
                }
                // Check partitioning for persist operators per sink too
                for (Map.Entry<InputPortMeta, InputPortMeta> entry : s.sinkSpecificPersistInputPortMap.entrySet()) {
                    InputPortMeta persistInputPort = entry.getValue();
                    StreamCodec<?> streamCodec = persistInputPort.getStreamCodec();
                    if (streamCodec != null && streamCodec instanceof StreamCodecWrapperForPersistance) {
                        updatePersistOperatorWithSinkPartitions(persistInputPort, s.sinkSpecificPersistOperatorMap.get(entry.getKey()), (StreamCodecWrapperForPersistance<?>) streamCodec, entry.getKey());
                    }
                }
            }
        }
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 8 with InputPortMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta in project apex-core by apache.

the class PhysicalPlan method setContainer.

private void setContainer(PTOperator pOperator, PTContainer container) {
    LOG.debug("Setting container {} for {}", container, pOperator);
    assert (pOperator.container == null) : "Container already assigned for " + pOperator;
    pOperator.container = container;
    container.operators.add(pOperator);
    int upStreamUnifierMemory = 0;
    if (!pOperator.upstreamMerge.isEmpty()) {
        for (Map.Entry<InputPortMeta, PTOperator> mEntry : pOperator.upstreamMerge.entrySet()) {
            assert (mEntry.getValue().container == null) : "Container already assigned for " + mEntry.getValue();
            mEntry.getValue().container = container;
            container.operators.add(mEntry.getValue());
            upStreamUnifierMemory += mEntry.getValue().getOperatorMeta().getValue(OperatorContext.MEMORY_MB);
        }
    }
    int memoryMB = pOperator.getOperatorMeta().getValue(OperatorContext.MEMORY_MB) + upStreamUnifierMemory;
    container.setRequiredMemoryMB(container.getRequiredMemoryMB() + memoryMB);
}

Example 9 with InputPortMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta in project apex-core by apache.

the class PlanModifier method removeOperator.

/**
   * Remove named operator and any outgoing streams.
   * @param name
   */
public void removeOperator(String name) {
    OperatorMeta om = logicalPlan.getOperatorMeta(name);
    if (om == null) {
        return;
    }
    if (!om.getInputStreams().isEmpty()) {
        for (Map.Entry<InputPortMeta, StreamMeta> input : om.getInputStreams().entrySet()) {
            if (input.getValue().getSinks().size() == 1) {
                // would result in dangling stream
                String msg = String.format("Operator %s connected to input streams %s", om.getName(), om.getInputStreams());
                throw new ValidationException(msg);
            }
        }
    }
    if (!om.getOutputStreams().isEmpty()) {
        String msg = String.format("Operator %s connected to output streams %s", om.getName(), om.getOutputStreams());
        throw new ValidationException(msg);
    }
    /*
    // remove associated sinks
    Map<InputPortMeta, StreamMeta> inputStreams = om.getInputStreams();
    for (Map.Entry<InputPortMeta, StreamMeta> e : inputStreams.entrySet()) {
      if (e.getKey().getOperatorMeta() == om) {
        if (e.getValue().getSinks().size() == 1) {
          // drop stream
          e.getValue().remove();
        }
      }
    }
    */
    logicalPlan.removeOperator(om.getOperator());
    if (physicalPlan != null) {
        physicalPlan.removeLogicalOperator(om);
    }
}

Example 10 with InputPortMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta in project apex-malhar by apache.

the class ApexStreamImplTest method testAddOperator.

@Test
public void testAddOperator() {
    LogicalPlan dag = new LogicalPlan();
    TestOperator<String, Integer> firstOperator = new TestOperator<>();
    TestOperator<Integer, Date> secondOperator = new TestOperator<>();
    new ApexStreamImpl<String>().addOperator(firstOperator, null, firstOperator.output, name("first")).endWith(secondOperator, secondOperator.input, name("second")).with(DAG.Locality.THREAD_LOCAL).with(Context.OperatorContext.AUTO_RECORD, true).with("prop", "TestProp").populateDag(dag);
    Assert.assertTrue(dag.getAllOperators().size() == 2);
    Set<String> opNames = new HashSet<>();
    opNames.add("first");
    opNames.add("second");
    for (LogicalPlan.OperatorMeta operatorMeta : dag.getAllOperators()) {
        Assert.assertTrue(operatorMeta.getOperator() instanceof TestOperator);
        Assert.assertTrue(opNames.contains(operatorMeta.getName()));
        if (operatorMeta.getName().equals("second")) {
            // Assert the Context.OperatorContext.AUTO_RECORD attribute has been set to true for second operator
            Assert.assertTrue(operatorMeta.getAttributes().get(Context.OperatorContext.AUTO_RECORD));
            // Assert the prop has been set to TestProp for second operator
            Assert.assertEquals("TestProp", ((TestOperator) operatorMeta.getOperator()).prop);
        } else {
            // Assert the Context.OperatorContext.AUTO_RECORD attribute keeps as default false for first operator
            Assert.assertNull(operatorMeta.getAttributes().get(Context.OperatorContext.AUTO_RECORD));
            // Assert the prop has not been set for first operator
            Assert.assertNull(((TestOperator) operatorMeta.getOperator()).prop);
        }
    }
    Collection<LogicalPlan.StreamMeta> streams = dag.getAllStreams();
    // Assert there is only one stream
    Assert.assertTrue(streams.size() == 1);
    for (LogicalPlan.StreamMeta stream : streams) {
        // Assert the stream is from first operator to second operator
        Assert.assertEquals("first", stream.getSource().getOperatorMeta().getName());
        Collection<InputPortMeta> portMetaCollection = stream.getSinks();
        Assert.assertTrue(1 == portMetaCollection.size());
        for (InputPortMeta inputPortMeta : portMetaCollection) {
            Assert.assertEquals("second", inputPortMeta.getOperatorMeta().getName());
        }
        // Assert the stream is thread local
        Assert.assertTrue(stream.getLocality() == DAG.Locality.THREAD_LOCAL);
    }
}