Examples with OperatorMeta com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta used on opensource projects

Example 61 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class PhysicalPlanTest method testParallelPartitioning.

@Test
public void testParallelPartitioning() {
    LogicalPlan dag = new LogicalPlan();
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    dag.addStream("o1Output1", o1.outport1, o2.inport1, o3.inport1).setLocality(null);
    dag.addStream("o2Output1", o2.outport1, o3.inport2).setLocality(Locality.CONTAINER_LOCAL);
    dag.setInputPortAttribute(o3.inport2, PortContext.PARTITION_PARALLEL, true);
    // parallel partition two downstream operators
    PartitioningTestOperator o3_1 = dag.addOperator("o3_1", PartitioningTestOperator.class);
    o3_1.fixedCapacity = false;
    dag.setInputPortAttribute(o3_1.inport1, PortContext.PARTITION_PARALLEL, true);
    OperatorMeta o3_1Meta = dag.getMeta(o3_1);
    GenericTestOperator o3_2 = dag.addOperator("o3_2", GenericTestOperator.class);
    dag.setInputPortAttribute(o3_2.inport1, PortContext.PARTITION_PARALLEL, true);
    OperatorMeta o3_2Meta = dag.getMeta(o3_2);
    dag.addStream("o3outport1", o3.outport1, o3_1.inport1, o3_2.inport1).setLocality(Locality.CONTAINER_LOCAL);
    // join within parallel partition
    GenericTestOperator o4 = dag.addOperator("o4", GenericTestOperator.class);
    dag.setInputPortAttribute(o4.inport1, PortContext.PARTITION_PARALLEL, true);
    dag.setInputPortAttribute(o4.inport2, PortContext.PARTITION_PARALLEL, true);
    OperatorMeta o4Meta = dag.getMeta(o4);
    dag.addStream("o3_1.outport1", o3_1.outport1, o4.inport1).setLocality(Locality.CONTAINER_LOCAL);
    dag.addStream("o3_2.outport1", o3_2.outport1, o4.inport2).setLocality(Locality.CONTAINER_LOCAL);
    // non inline
    GenericTestOperator o5single = dag.addOperator("o5single", GenericTestOperator.class);
    dag.addStream("o4outport1", o4.outport1, o5single.inport1);
    int maxContainers = 4;
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, maxContainers);
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new TestPlanContext());
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    Assert.assertEquals("number of containers", 4, plan.getContainers().size());
    PTContainer container1 = plan.getContainers().get(0);
    Assert.assertEquals("number operators " + container1, 1, container1.getOperators().size());
    Assert.assertEquals("operators " + container1, "o1", container1.getOperators().get(0).getOperatorMeta().getName());
    for (int i = 1; i < 3; i++) {
        PTContainer container2 = plan.getContainers().get(i);
        Assert.assertEquals("number operators " + container2, 5, container2.getOperators().size());
        Set<String> expectedLogicalNames = Sets.newHashSet("o2", "o3", o3_1Meta.getName(), o3_2Meta.getName(), o4Meta.getName());
        Set<String> actualLogicalNames = Sets.newHashSet();
        for (PTOperator p : container2.getOperators()) {
            actualLogicalNames.add(p.getOperatorMeta().getName());
        }
        Assert.assertEquals("operator names " + container2, expectedLogicalNames, actualLogicalNames);
    }
    List<OperatorMeta> inlineOperators = Lists.newArrayList(dag.getMeta(o2), o3_1Meta, o3_2Meta);
    for (OperatorMeta ow : inlineOperators) {
        List<PTOperator> partitionedInstances = plan.getOperators(ow);
        Assert.assertEquals("" + partitionedInstances, 2, partitionedInstances.size());
        for (PTOperator p : partitionedInstances) {
            Assert.assertEquals("outputs " + p, 1, p.getOutputs().size());
            Assert.assertTrue("downstream inline " + p.getOutputs().get(0), p.getOutputs().get(0).isDownStreamInline());
        }
    }
    // container 4: Unifier for o4 & O5
    PTContainer container4 = plan.getContainers().get(3);
    PTOperator ptOperatorO5 = plan.getOperators(dag.getMeta(o5single)).get(0);
    PTOperator unifier = ptOperatorO5.upstreamMerge.values().iterator().next();
    Assert.assertEquals("number operators " + container4, 2, container4.getOperators().size());
    Assert.assertEquals("operators " + container4, o4Meta.getMeta(o4.outport1).getUnifierMeta(), unifier.getOperatorMeta());
    Assert.assertEquals("unifier inputs" + unifier.getInputs(), 2, unifier.getInputs().size());
    Assert.assertEquals("unifier outputs" + unifier.getOutputs(), 1, unifier.getOutputs().size());
    OperatorMeta o5Meta = dag.getMeta(o5single);
    Assert.assertEquals("operators " + container4, o5Meta, ptOperatorO5.getOperatorMeta());
    List<PTOperator> o5Instances = plan.getOperators(o5Meta);
    Assert.assertEquals("" + o5Instances, 1, o5Instances.size());
    Assert.assertEquals("inputs" + ptOperatorO5.getInputs(), 1, ptOperatorO5.getInputs().size());
    Assert.assertEquals("inputs" + ptOperatorO5.getInputs(), unifier, ptOperatorO5.getInputs().get(0).source.source);
    // verify partitioner was called for parallel partition
    Assert.assertNotNull("partitioner called " + o3_1, o3_1.partitions);
    for (PTOperator p : plan.getOperators(o3_1Meta)) {
        Assert.assertEquals("inputs " + p, 1, p.getInputs().size());
        for (PTInput pti : p.getInputs()) {
            Assert.assertNull("partition keys " + pti, pti.partitions);
        }
    }
}

Example 62 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class StreamingContainerManager method processHeartbeat.

/**
   * process the heartbeat from each container.
   * called by the RPC thread for each container. (i.e. called by multiple threads)
   *
   * @param heartbeat
   * @return heartbeat response
   */
@SuppressWarnings("StatementWithEmptyBody")
public ContainerHeartbeatResponse processHeartbeat(ContainerHeartbeat heartbeat) {
    long currentTimeMillis = clock.getTime();
    final StreamingContainerAgent sca = this.containers.get(heartbeat.getContainerId());
    if (sca == null || sca.container.getState() == PTContainer.State.KILLED) {
        // could be orphaned container that was replaced and needs to terminate
        LOG.error("Unknown container {}", heartbeat.getContainerId());
        ContainerHeartbeatResponse response = new ContainerHeartbeatResponse();
        response.shutdown = ShutdownType.ABORT;
        return response;
    }
    //LOG.debug("{} {} {}", new Object[]{sca.container.containerId, sca.container.bufferServerAddress, sca.container.getState()});
    if (sca.container.getState() == PTContainer.State.ALLOCATED) {
        // capture dynamically assigned address from container
        if (sca.container.bufferServerAddress == null && heartbeat.bufferServerHost != null) {
            sca.container.bufferServerAddress = InetSocketAddress.createUnresolved(heartbeat.bufferServerHost, heartbeat.bufferServerPort);
            LOG.info("Container {} buffer server: {}", sca.container.getExternalId(), sca.container.bufferServerAddress);
        }
        final long containerStartTime = System.currentTimeMillis();
        sca.container.setState(PTContainer.State.ACTIVE);
        sca.container.setStartedTime(containerStartTime);
        sca.container.setFinishedTime(-1);
        sca.jvmName = heartbeat.jvmName;
        poolExecutor.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    containerFile.append(sca.getContainerInfo());
                } catch (IOException ex) {
                    LOG.warn("Cannot write to container file");
                }
                for (PTOperator ptOp : sca.container.getOperators()) {
                    try {
                        JSONObject operatorInfo = new JSONObject();
                        operatorInfo.put("name", ptOp.getName());
                        operatorInfo.put("id", ptOp.getId());
                        operatorInfo.put("container", sca.container.getExternalId());
                        operatorInfo.put("startTime", containerStartTime);
                        operatorFile.append(operatorInfo);
                    } catch (IOException | JSONException ex) {
                        LOG.warn("Cannot write to operator file: ", ex);
                    }
                }
            }
        });
    }
    sca.containerStackTrace = heartbeat.stackTrace;
    if (heartbeat.restartRequested) {
        LOG.error("Container {} restart request", sca.container.getExternalId());
        containerStopRequests.put(sca.container.getExternalId(), sca.container.getExternalId());
    }
    sca.memoryMBFree = heartbeat.memoryMBFree;
    sca.gcCollectionCount = heartbeat.gcCollectionCount;
    sca.gcCollectionTime = heartbeat.gcCollectionTime;
    sca.undeployOpers.clear();
    sca.deployOpers.clear();
    if (!this.deployChangeInProgress.get()) {
        sca.deployCnt = this.deployChangeCnt;
    }
    Set<Integer> reportedOperators = Sets.newHashSetWithExpectedSize(sca.container.getOperators().size());
    for (OperatorHeartbeat shb : heartbeat.getContainerStats().operators) {
        long maxEndWindowTimestamp = 0;
        reportedOperators.add(shb.nodeId);
        PTOperator oper = this.plan.getAllOperators().get(shb.getNodeId());
        if (oper == null) {
            LOG.info("Heartbeat for unknown operator {} (container {})", shb.getNodeId(), heartbeat.getContainerId());
            sca.undeployOpers.add(shb.nodeId);
            continue;
        }
        if (shb.requestResponse != null) {
            for (StatsListener.OperatorResponse obj : shb.requestResponse) {
                if (obj instanceof OperatorResponse) {
                    // This is to identify platform requests
                    commandResponse.put((Long) obj.getResponseId(), obj.getResponse());
                    LOG.debug(" Got back the response {} for the request {}", obj, obj.getResponseId());
                } else {
                    // This is to identify user requests
                    oper.stats.responses.add(obj);
                }
            }
        }
        //LOG.debug("heartbeat {} {}/{} {}", oper, oper.getState(), shb.getState(), oper.getContainer().getExternalId());
        if (!(oper.getState() == PTOperator.State.ACTIVE && shb.getState() == OperatorHeartbeat.DeployState.ACTIVE)) {
            // deploy state may require synchronization
            processOperatorDeployStatus(oper, shb, sca);
        }
        oper.stats.lastHeartbeat = shb;
        List<ContainerStats.OperatorStats> statsList = shb.getOperatorStatsContainer();
        if (!statsList.isEmpty()) {
            long tuplesProcessed = 0;
            long tuplesEmitted = 0;
            long totalCpuTimeUsed = 0;
            int statCount = 0;
            long maxDequeueTimestamp = -1;
            oper.stats.recordingId = null;
            final OperatorStatus status = oper.stats;
            status.statsRevs.checkout();
            for (Map.Entry<String, PortStatus> entry : status.inputPortStatusList.entrySet()) {
                entry.getValue().recordingId = null;
            }
            for (Map.Entry<String, PortStatus> entry : status.outputPortStatusList.entrySet()) {
                entry.getValue().recordingId = null;
            }
            for (ContainerStats.OperatorStats stats : statsList) {
                if (stats == null) {
                    LOG.warn("Operator {} statistics list contains null element", shb.getNodeId());
                    continue;
                }
                /* report checkpoint-ed WindowId status of the operator */
                if (stats.checkpoint instanceof Checkpoint) {
                    if (oper.getRecentCheckpoint() == null || oper.getRecentCheckpoint().windowId < stats.checkpoint.getWindowId()) {
                        addCheckpoint(oper, (Checkpoint) stats.checkpoint);
                        if (stats.checkpointStats != null) {
                            status.checkpointStats = stats.checkpointStats;
                            status.checkpointTimeMA.add(stats.checkpointStats.checkpointTime);
                        }
                        oper.failureCount = 0;
                    }
                }
                oper.stats.recordingId = stats.recordingId;
                /* report all the other stuff */
                // calculate the stats related to end window
                // end window stats for a particular window id for a particular node
                EndWindowStats endWindowStats = new EndWindowStats();
                Collection<ContainerStats.OperatorStats.PortStats> ports = stats.inputPorts;
                if (ports != null) {
                    Set<String> currentInputPortSet = Sets.newHashSetWithExpectedSize(ports.size());
                    for (ContainerStats.OperatorStats.PortStats s : ports) {
                        currentInputPortSet.add(s.id);
                        PortStatus ps = status.inputPortStatusList.get(s.id);
                        if (ps == null) {
                            ps = status.new PortStatus();
                            ps.portName = s.id;
                            status.inputPortStatusList.put(s.id, ps);
                        }
                        ps.totalTuples += s.tupleCount;
                        ps.recordingId = s.recordingId;
                        tuplesProcessed += s.tupleCount;
                        endWindowStats.dequeueTimestamps.put(s.id, s.endWindowTimestamp);
                        Pair<Integer, String> operatorPortName = new Pair<>(oper.getId(), s.id);
                        Long lastEndWindowTimestamp = operatorPortLastEndWindowTimestamps.get(operatorPortName);
                        if (lastEndWindowTimestamp == null) {
                            lastEndWindowTimestamp = lastStatsTimestamp;
                        }
                        long portElapsedMillis = Math.max(s.endWindowTimestamp - lastEndWindowTimestamp, 0);
                        //LOG.debug("=== PROCESSED TUPLE COUNT for {}: {}, {}, {}, {}", operatorPortName, s.tupleCount, portElapsedMillis, operatorPortLastEndWindowTimestamps.get(operatorPortName), lastStatsTimestamp);
                        ps.tuplesPMSMA.add(s.tupleCount, portElapsedMillis);
                        ps.bufferServerBytesPMSMA.add(s.bufferServerBytes, portElapsedMillis);
                        ps.queueSizeMA.add(s.queueSize);
                        operatorPortLastEndWindowTimestamps.put(operatorPortName, s.endWindowTimestamp);
                        if (maxEndWindowTimestamp < s.endWindowTimestamp) {
                            maxEndWindowTimestamp = s.endWindowTimestamp;
                        }
                        if (s.endWindowTimestamp > maxDequeueTimestamp) {
                            maxDequeueTimestamp = s.endWindowTimestamp;
                        }
                    }
                    // need to remove dead ports, for unifiers
                    Iterator<Map.Entry<String, PortStatus>> it = status.inputPortStatusList.entrySet().iterator();
                    while (it.hasNext()) {
                        Map.Entry<String, PortStatus> entry = it.next();
                        if (!currentInputPortSet.contains(entry.getKey())) {
                            it.remove();
                        }
                    }
                }
                ports = stats.outputPorts;
                if (ports != null) {
                    Set<String> currentOutputPortSet = Sets.newHashSetWithExpectedSize(ports.size());
                    for (ContainerStats.OperatorStats.PortStats s : ports) {
                        currentOutputPortSet.add(s.id);
                        PortStatus ps = status.outputPortStatusList.get(s.id);
                        if (ps == null) {
                            ps = status.new PortStatus();
                            ps.portName = s.id;
                            status.outputPortStatusList.put(s.id, ps);
                        }
                        ps.totalTuples += s.tupleCount;
                        ps.recordingId = s.recordingId;
                        tuplesEmitted += s.tupleCount;
                        Pair<Integer, String> operatorPortName = new Pair<>(oper.getId(), s.id);
                        Long lastEndWindowTimestamp = operatorPortLastEndWindowTimestamps.get(operatorPortName);
                        if (lastEndWindowTimestamp == null) {
                            lastEndWindowTimestamp = lastStatsTimestamp;
                        }
                        long portElapsedMillis = Math.max(s.endWindowTimestamp - lastEndWindowTimestamp, 0);
                        //LOG.debug("=== EMITTED TUPLE COUNT for {}: {}, {}, {}, {}", operatorPortName, s.tupleCount, portElapsedMillis, operatorPortLastEndWindowTimestamps.get(operatorPortName), lastStatsTimestamp);
                        ps.tuplesPMSMA.add(s.tupleCount, portElapsedMillis);
                        ps.bufferServerBytesPMSMA.add(s.bufferServerBytes, portElapsedMillis);
                        operatorPortLastEndWindowTimestamps.put(operatorPortName, s.endWindowTimestamp);
                        if (maxEndWindowTimestamp < s.endWindowTimestamp) {
                            maxEndWindowTimestamp = s.endWindowTimestamp;
                        }
                    }
                    if (ports.size() > 0) {
                        endWindowStats.emitTimestamp = ports.iterator().next().endWindowTimestamp;
                    }
                    // need to remove dead ports, for unifiers
                    Iterator<Map.Entry<String, PortStatus>> it = status.outputPortStatusList.entrySet().iterator();
                    while (it.hasNext()) {
                        Map.Entry<String, PortStatus> entry = it.next();
                        if (!currentOutputPortSet.contains(entry.getKey())) {
                            it.remove();
                        }
                    }
                }
                // (we don't know the latency for output operators because they don't emit tuples)
                if (endWindowStats.emitTimestamp < 0) {
                    endWindowStats.emitTimestamp = maxDequeueTimestamp;
                }
                if (status.currentWindowId.get() != stats.windowId) {
                    status.lastWindowIdChangeTms = currentTimeMillis;
                    status.currentWindowId.set(stats.windowId);
                }
                totalCpuTimeUsed += stats.cpuTimeUsed;
                statCount++;
                if (oper.getOperatorMeta().getValue(OperatorContext.COUNTERS_AGGREGATOR) != null) {
                    endWindowStats.counters = stats.counters;
                }
                if (oper.getOperatorMeta().getMetricAggregatorMeta() != null && oper.getOperatorMeta().getMetricAggregatorMeta().getAggregator() != null) {
                    endWindowStats.metrics = stats.metrics;
                }
                if (stats.windowId > currentEndWindowStatsWindowId) {
                    Map<Integer, EndWindowStats> endWindowStatsMap = endWindowStatsOperatorMap.get(stats.windowId);
                    if (endWindowStatsMap == null) {
                        endWindowStatsMap = new ConcurrentSkipListMap<>();
                        Map<Integer, EndWindowStats> endWindowStatsMapPrevious = endWindowStatsOperatorMap.putIfAbsent(stats.windowId, endWindowStatsMap);
                        if (endWindowStatsMapPrevious != null) {
                            endWindowStatsMap = endWindowStatsMapPrevious;
                        }
                    }
                    endWindowStatsMap.put(shb.getNodeId(), endWindowStats);
                    Set<Integer> allCurrentOperators = plan.getAllOperators().keySet();
                    int numOperators = plan.getAllOperators().size();
                    if (allCurrentOperators.containsAll(endWindowStatsMap.keySet()) && endWindowStatsMap.size() == numOperators) {
                        completeEndWindowStatsWindowId = stats.windowId;
                    }
                }
            }
            status.totalTuplesProcessed.add(tuplesProcessed);
            status.totalTuplesEmitted.add(tuplesEmitted);
            OperatorMeta logicalOperator = oper.getOperatorMeta();
            LogicalOperatorStatus logicalStatus = logicalOperator.getStatus();
            if (!oper.isUnifier()) {
                logicalStatus.totalTuplesProcessed += tuplesProcessed;
                logicalStatus.totalTuplesEmitted += tuplesEmitted;
            }
            long lastMaxEndWindowTimestamp = operatorLastEndWindowTimestamps.containsKey(oper.getId()) ? operatorLastEndWindowTimestamps.get(oper.getId()) : lastStatsTimestamp;
            if (maxEndWindowTimestamp >= lastMaxEndWindowTimestamp) {
                double tuplesProcessedPMSMA = 0.0;
                double tuplesEmittedPMSMA = 0.0;
                if (statCount != 0) {
                    //LOG.debug("CPU for {}: {} / {} - {}", oper.getId(), totalCpuTimeUsed, maxEndWindowTimestamp, lastMaxEndWindowTimestamp);
                    status.cpuNanosPMSMA.add(totalCpuTimeUsed, maxEndWindowTimestamp - lastMaxEndWindowTimestamp);
                }
                for (PortStatus ps : status.inputPortStatusList.values()) {
                    tuplesProcessedPMSMA += ps.tuplesPMSMA.getAvg();
                }
                for (PortStatus ps : status.outputPortStatusList.values()) {
                    tuplesEmittedPMSMA += ps.tuplesPMSMA.getAvg();
                }
                status.tuplesProcessedPSMA.set(Math.round(tuplesProcessedPMSMA * 1000));
                status.tuplesEmittedPSMA.set(Math.round(tuplesEmittedPMSMA * 1000));
            } else {
            //LOG.warn("This timestamp for {} is lower than the previous!! {} < {}", oper.getId(),
            // maxEndWindowTimestamp, lastMaxEndWindowTimestamp);
            }
            operatorLastEndWindowTimestamps.put(oper.getId(), maxEndWindowTimestamp);
            status.listenerStats.add(statsList);
            this.reportStats.put(oper, oper);
            status.statsRevs.commit();
        }
        if (lastStatsTimestamp < maxEndWindowTimestamp) {
            lastStatsTimestamp = maxEndWindowTimestamp;
        }
    }
    sca.lastHeartbeatMillis = currentTimeMillis;
    for (PTOperator oper : sca.container.getOperators()) {
        if (!reportedOperators.contains(oper.getId())) {
            processOperatorDeployStatus(oper, null, sca);
        }
    }
    ContainerHeartbeatResponse rsp = getHeartbeatResponse(sca);
    if (heartbeat.getContainerStats().operators.isEmpty() && isApplicationIdle()) {
        LOG.info("requesting idle shutdown for container {}", heartbeat.getContainerId());
        rsp.shutdown = ShutdownType.ABORT;
    } else {
        if (sca.isShutdownRequested()) {
            LOG.info("requesting shutdown for container {}", heartbeat.getContainerId());
            rsp.shutdown = sca.shutdownRequest;
        }
    }
    List<StramToNodeRequest> requests = rsp.nodeRequests != null ? rsp.nodeRequests : new ArrayList<StramToNodeRequest>();
    ConcurrentLinkedQueue<StramToNodeRequest> operatorRequests = sca.getOperatorRequests();
    while (true) {
        StramToNodeRequest r = operatorRequests.poll();
        if (r == null) {
            break;
        }
        requests.add(r);
    }
    rsp.nodeRequests = requests;
    rsp.committedWindowId = committedWindowId;
    rsp.stackTraceRequired = sca.stackTraceRequested;
    sca.stackTraceRequested = false;
    apexPluginDispatcher.dispatch(new DAGExecutionEvent.HeartbeatExecutionEvent(heartbeat));
    return rsp;
}

Example 63 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class StreamingContainerManager method updateCheckpoints.

/**
   * Visit all operators to update current checkpoint based on updated downstream state.
   * Purge older checkpoints that are no longer needed.
   */
private long updateCheckpoints(boolean recovery) {
    int operatorCount = 0;
    UpdateCheckpointsContext ctx = new UpdateCheckpointsContext(clock, recovery, getCheckpointGroups());
    for (OperatorMeta logicalOperator : plan.getLogicalPlan().getRootOperators()) {
        List<PTOperator> operators = plan.getOperators(logicalOperator);
        if (operators != null) {
            for (PTOperator operator : operators) {
                operatorCount++;
                updateRecoveryCheckpoints(operator, ctx, recovery);
            }
        }
    }
    // if no physical operators are available, then don't update committedWindowId
    if (operatorCount == 0) {
        return committedWindowId;
    }
    purgeCheckpoints();
    for (PTOperator oper : ctx.blocked) {
        String containerId = oper.getContainer().getExternalId();
        if (containerId != null) {
            LOG.info("Blocked operator {} container {} time {}ms", oper, oper.getContainer().toIdStateString(), ctx.currentTms - oper.stats.lastWindowIdChangeTms);
            this.containerStopRequests.put(containerId, containerId);
        }
    }
    return ctx.committedWindowId.longValue();
}

Example 64 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class PhysicalPlan method redoPartitions.

private void redoPartitions(PMapping currentMapping, String note) {
    Partitioner<Operator> partitioner = getPartitioner(currentMapping);
    if (partitioner == null) {
        LOG.warn("No partitioner for {}", currentMapping.logicalOperator);
        return;
    }
    RepartitionContext mainPC = new RepartitionContext(partitioner, currentMapping, 0);
    if (mainPC.newPartitions.isEmpty()) {
        LOG.warn("Empty partition list after repartition: {}", currentMapping.logicalOperator);
        return;
    }
    int memoryPerPartition = currentMapping.logicalOperator.getValue(OperatorContext.MEMORY_MB);
    for (Map.Entry<OutputPortMeta, StreamMeta> stream : currentMapping.logicalOperator.getOutputStreams().entrySet()) {
        if (stream.getValue().getLocality() != Locality.THREAD_LOCAL && stream.getValue().getLocality() != Locality.CONTAINER_LOCAL) {
            memoryPerPartition += stream.getKey().getValue(PortContext.BUFFER_MEMORY_MB);
        }
    }
    for (OperatorMeta pp : currentMapping.parallelPartitions) {
        for (Map.Entry<OutputPortMeta, StreamMeta> stream : pp.getOutputStreams().entrySet()) {
            if (stream.getValue().getLocality() != Locality.THREAD_LOCAL && stream.getValue().getLocality() != Locality.CONTAINER_LOCAL) {
                memoryPerPartition += stream.getKey().getValue(PortContext.BUFFER_MEMORY_MB);
            }
        }
        memoryPerPartition += pp.getValue(OperatorContext.MEMORY_MB);
    }
    int requiredMemoryMB = (mainPC.newPartitions.size() - mainPC.currentPartitions.size()) * memoryPerPartition;
    if (requiredMemoryMB > availableMemoryMB) {
        LOG.warn("Insufficient headroom for repartitioning: available {}m required {}m", availableMemoryMB, requiredMemoryMB);
        return;
    }
    List<Partition<Operator>> addedPartitions = new ArrayList<>();
    // determine modifications of partition set, identify affected operator instance(s)
    for (Partition<Operator> newPartition : mainPC.newPartitions) {
        PTOperator op = mainPC.currentPartitionMap.remove(newPartition);
        if (op == null) {
            addedPartitions.add(newPartition);
        } else {
            // check whether mapping was changed
            for (DefaultPartition<Operator> pi : mainPC.currentPartitions) {
                if (pi == newPartition && pi.isModified()) {
                    // existing partition changed (operator or partition keys)
                    // remove/add to update subscribers and state
                    mainPC.currentPartitionMap.put(newPartition, op);
                    addedPartitions.add(newPartition);
                }
            }
        }
    }
    // remaining entries represent deprecated partitions
    this.undeployOpers.addAll(mainPC.currentPartitionMap.values());
    // downstream dependencies require redeploy, resolve prior to modifying plan
    Set<PTOperator> deps = this.getDependents(mainPC.currentPartitionMap.values());
    this.undeployOpers.addAll(deps);
    // dependencies need redeploy, except operators excluded in remove
    this.deployOpers.addAll(deps);
    // process parallel partitions before removing operators from the plan
    LinkedHashMap<PMapping, RepartitionContext> partitionContexts = Maps.newLinkedHashMap();
    Stack<OperatorMeta> parallelPartitions = new Stack<>();
    parallelPartitions.addAll(currentMapping.parallelPartitions);
    pendingLoop: while (!parallelPartitions.isEmpty()) {
        OperatorMeta ppMeta = parallelPartitions.pop();
        for (StreamMeta s : ppMeta.getInputStreams().values()) {
            if (currentMapping.parallelPartitions.contains(s.getSource().getOperatorMeta()) && parallelPartitions.contains(s.getSource().getOperatorMeta())) {
                parallelPartitions.push(ppMeta);
                parallelPartitions.remove(s.getSource().getOperatorMeta());
                parallelPartitions.push(s.getSource().getOperatorMeta());
                continue pendingLoop;
            }
        }
        LOG.debug("Processing parallel partition {}", ppMeta);
        PMapping ppm = this.logicalToPTOperator.get(ppMeta);
        Partitioner<Operator> ppp = getPartitioner(ppm);
        if (ppp == null) {
            partitionContexts.put(ppm, null);
        } else {
            RepartitionContext pc = new RepartitionContext(ppp, ppm, mainPC.newPartitions.size());
            if (pc.newPartitions == null) {
                throw new IllegalStateException("Partitioner returns null for parallel partition " + ppm.logicalOperator);
            }
            partitionContexts.put(ppm, pc);
        }
    }
    // plan updates start here, after all changes were identified
    // remove obsolete operators first, any freed resources
    // can subsequently be used for new/modified partitions
    List<PTOperator> copyPartitions = Lists.newArrayList(currentMapping.partitions);
    // remove deprecated partitions from plan
    for (PTOperator p : mainPC.currentPartitionMap.values()) {
        copyPartitions.remove(p);
        removePartition(p, currentMapping);
        mainPC.operatorIdToPartition.remove(p.getId());
    }
    currentMapping.partitions = copyPartitions;
    // add new operators
    for (Partition<Operator> newPartition : addedPartitions) {
        PTOperator p = addPTOperator(currentMapping, newPartition, mainPC.minCheckpoint);
        mainPC.operatorIdToPartition.put(p.getId(), newPartition);
    }
    // process parallel partition changes
    for (Map.Entry<PMapping, RepartitionContext> e : partitionContexts.entrySet()) {
        if (e.getValue() == null) {
            // no partitioner, add required operators
            for (int i = 0; i < addedPartitions.size(); i++) {
                LOG.debug("Automatically adding to parallel partition {}", e.getKey());
                // set activation windowId to confirm to upstream checkpoints
                addPTOperator(e.getKey(), null, mainPC.minCheckpoint);
            }
        } else {
            RepartitionContext pc = e.getValue();
            // track previous parallel partition mapping
            Map<Partition<Operator>, Partition<Operator>> prevMapping = Maps.newHashMap();
            for (int i = 0; i < mainPC.currentPartitions.size(); i++) {
                prevMapping.put(pc.currentPartitions.get(i), mainPC.currentPartitions.get(i));
            }
            // determine which new partitions match upstream, remaining to be treated as new operator
            Map<Partition<Operator>, Partition<Operator>> newMapping = Maps.newHashMap();
            Iterator<Partition<Operator>> itMain = mainPC.newPartitions.iterator();
            Iterator<Partition<Operator>> itParallel = pc.newPartitions.iterator();
            while (itMain.hasNext() && itParallel.hasNext()) {
                newMapping.put(itParallel.next(), itMain.next());
            }
            for (Partition<Operator> newPartition : pc.newPartitions) {
                PTOperator op = pc.currentPartitionMap.remove(newPartition);
                if (op == null) {
                    pc.addedPartitions.add(newPartition);
                } else if (prevMapping.get(newPartition) != newMapping.get(newPartition)) {
                    // upstream partitions don't match, remove/add to replace with new operator
                    pc.currentPartitionMap.put(newPartition, op);
                    pc.addedPartitions.add(newPartition);
                } else {
                    // check whether mapping was changed - based on DefaultPartition implementation
                    for (DefaultPartition<Operator> pi : pc.currentPartitions) {
                        if (pi == newPartition && pi.isModified()) {
                            // existing partition changed (operator or partition keys)
                            // remove/add to update subscribers and state
                            mainPC.currentPartitionMap.put(newPartition, op);
                            pc.addedPartitions.add(newPartition);
                        }
                    }
                }
            }
            if (!pc.currentPartitionMap.isEmpty()) {
                // remove obsolete partitions
                List<PTOperator> cowPartitions = Lists.newArrayList(e.getKey().partitions);
                for (PTOperator p : pc.currentPartitionMap.values()) {
                    cowPartitions.remove(p);
                    removePartition(p, e.getKey());
                    pc.operatorIdToPartition.remove(p.getId());
                }
                e.getKey().partitions = cowPartitions;
            }
            // add new partitions
            for (Partition<Operator> newPartition : pc.addedPartitions) {
                PTOperator oper = addPTOperator(e.getKey(), newPartition, mainPC.minCheckpoint);
                pc.operatorIdToPartition.put(oper.getId(), newPartition);
            }
            getPartitioner(e.getKey()).partitioned(pc.operatorIdToPartition);
        }
    }
    updateStreamMappings(currentMapping);
    for (PMapping pp : partitionContexts.keySet()) {
        updateStreamMappings(pp);
    }
    deployChanges();
    if (mainPC.currentPartitions.size() != mainPC.newPartitions.size()) {
        StramEvent ev = new StramEvent.PartitionEvent(currentMapping.logicalOperator.getName(), mainPC.currentPartitions.size(), mainPC.newPartitions.size());
        ev.setReason(note);
        this.ctx.recordEventAsync(ev);
    }
    partitioner.partitioned(mainPC.operatorIdToPartition);
}

Example 65 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class PlanModifier method addStream.

/**
   * Add stream to logical plan. If a stream with same name and source already
   * exists, the new downstream operator will be attached to it.
   *
   * @param streamName
   * @param sourceOperName
   * @param sourcePortName
   * @param targetOperName
   * @param targetPortName
   */
public void addStream(String streamName, String sourceOperName, String sourcePortName, String targetOperName, String targetPortName) {
    OperatorMeta om = logicalPlan.getOperatorMeta(sourceOperName);
    if (om == null) {
        throw new ValidationException("Invalid operator name " + sourceOperName);
    }
    Operators.PortMappingDescriptor portMap = new Operators.PortMappingDescriptor();
    Operators.describe(om.getOperator(), portMap);
    PortContextPair<OutputPort<?>> sourcePort = portMap.outputPorts.get(sourcePortName);
    if (sourcePort == null) {
        throw new AssertionError(String.format("Invalid port %s (%s)", sourcePortName, om));
    }
    addStream(streamName, sourcePort.component, getInputPort(targetOperName, targetPortName));
}