Examples with OperatorID org.apache.flink.runtime.jobgraph.OperatorID used on opensource projects

Example 56 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class StreamingJobGraphGenerator method createChain.

private List<StreamEdge> createChain(final Integer currentNodeId, final int chainIndex, final OperatorChainInfo chainInfo, final Map<Integer, OperatorChainInfo> chainEntryPoints) {
    Integer startNodeId = chainInfo.getStartNodeId();
    if (!builtVertices.contains(startNodeId)) {
        List<StreamEdge> transitiveOutEdges = new ArrayList<StreamEdge>();
        List<StreamEdge> chainableOutputs = new ArrayList<StreamEdge>();
        List<StreamEdge> nonChainableOutputs = new ArrayList<StreamEdge>();
        StreamNode currentNode = streamGraph.getStreamNode(currentNodeId);
        for (StreamEdge outEdge : currentNode.getOutEdges()) {
            if (isChainable(outEdge, streamGraph)) {
                chainableOutputs.add(outEdge);
            } else {
                nonChainableOutputs.add(outEdge);
            }
        }
        for (StreamEdge chainable : chainableOutputs) {
            transitiveOutEdges.addAll(createChain(chainable.getTargetId(), chainIndex + 1, chainInfo, chainEntryPoints));
        }
        for (StreamEdge nonChainable : nonChainableOutputs) {
            transitiveOutEdges.add(nonChainable);
            createChain(nonChainable.getTargetId(), // operators start at position 1 because 0 is for chained source inputs
            1, chainEntryPoints.computeIfAbsent(nonChainable.getTargetId(), (k) -> chainInfo.newChain(nonChainable.getTargetId())), chainEntryPoints);
        }
        chainedNames.put(currentNodeId, createChainedName(currentNodeId, chainableOutputs, Optional.ofNullable(chainEntryPoints.get(currentNodeId))));
        chainedMinResources.put(currentNodeId, createChainedMinResources(currentNodeId, chainableOutputs));
        chainedPreferredResources.put(currentNodeId, createChainedPreferredResources(currentNodeId, chainableOutputs));
        OperatorID currentOperatorId = chainInfo.addNodeToChain(currentNodeId, streamGraph.getStreamNode(currentNodeId).getOperatorName());
        if (currentNode.getInputFormat() != null) {
            getOrCreateFormatContainer(startNodeId).addInputFormat(currentOperatorId, currentNode.getInputFormat());
        }
        if (currentNode.getOutputFormat() != null) {
            getOrCreateFormatContainer(startNodeId).addOutputFormat(currentOperatorId, currentNode.getOutputFormat());
        }
        StreamConfig config = currentNodeId.equals(startNodeId) ? createJobVertex(startNodeId, chainInfo) : new StreamConfig(new Configuration());
        setVertexConfig(currentNodeId, config, chainableOutputs, nonChainableOutputs, chainInfo.getChainedSources());
        if (currentNodeId.equals(startNodeId)) {
            config.setChainStart();
            config.setChainIndex(chainIndex);
            config.setOperatorName(streamGraph.getStreamNode(currentNodeId).getOperatorName());
            for (StreamEdge edge : transitiveOutEdges) {
                connect(startNodeId, edge);
            }
            config.setOutEdgesInOrder(transitiveOutEdges);
            config.setTransitiveChainedTaskConfigs(chainedConfigs.get(startNodeId));
        } else {
            chainedConfigs.computeIfAbsent(startNodeId, k -> new HashMap<Integer, StreamConfig>());
            config.setChainIndex(chainIndex);
            StreamNode node = streamGraph.getStreamNode(currentNodeId);
            config.setOperatorName(node.getOperatorName());
            chainedConfigs.get(startNodeId).put(currentNodeId, config);
        }
        config.setOperatorID(currentOperatorId);
        if (chainableOutputs.isEmpty()) {
            config.setChainEnd();
        }
        return transitiveOutEdges;
    } else {
        return new ArrayList<>();
    }
}

Example 57 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class AbstractStreamOperator method setup.

// ------------------------------------------------------------------------
// Life Cycle
// ------------------------------------------------------------------------
@Override
public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) {
    final Environment environment = containingTask.getEnvironment();
    this.container = containingTask;
    this.config = config;
    try {
        InternalOperatorMetricGroup operatorMetricGroup = environment.getMetricGroup().getOrAddOperator(config.getOperatorID(), config.getOperatorName());
        this.output = new CountingOutput<>(output, operatorMetricGroup.getIOMetricGroup().getNumRecordsOutCounter());
        if (config.isChainEnd()) {
            operatorMetricGroup.getIOMetricGroup().reuseOutputMetricsForTask();
        }
        this.metrics = operatorMetricGroup;
    } catch (Exception e) {
        LOG.warn("An error occurred while instantiating task metrics.", e);
        this.metrics = UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup();
        this.output = output;
    }
    this.combinedWatermark = IndexedCombinedWatermarkStatus.forInputsCount(2);
    try {
        Configuration taskManagerConfig = environment.getTaskManagerInfo().getConfiguration();
        int historySize = taskManagerConfig.getInteger(MetricOptions.LATENCY_HISTORY_SIZE);
        if (historySize <= 0) {
            LOG.warn("{} has been set to a value equal or below 0: {}. Using default.", MetricOptions.LATENCY_HISTORY_SIZE, historySize);
            historySize = MetricOptions.LATENCY_HISTORY_SIZE.defaultValue();
        }
        final String configuredGranularity = taskManagerConfig.getString(MetricOptions.LATENCY_SOURCE_GRANULARITY);
        LatencyStats.Granularity granularity;
        try {
            granularity = LatencyStats.Granularity.valueOf(configuredGranularity.toUpperCase(Locale.ROOT));
        } catch (IllegalArgumentException iae) {
            granularity = LatencyStats.Granularity.OPERATOR;
            LOG.warn("Configured value {} option for {} is invalid. Defaulting to {}.", configuredGranularity, MetricOptions.LATENCY_SOURCE_GRANULARITY.key(), granularity);
        }
        MetricGroup jobMetricGroup = this.metrics.getJobMetricGroup();
        this.latencyStats = new LatencyStats(jobMetricGroup.addGroup("latency"), historySize, container.getIndexInSubtaskGroup(), getOperatorID(), granularity);
    } catch (Exception e) {
        LOG.warn("An error occurred while instantiating latency metrics.", e);
        this.latencyStats = new LatencyStats(UnregisteredMetricGroups.createUnregisteredTaskManagerJobMetricGroup().addGroup("latency"), 1, 0, new OperatorID(), LatencyStats.Granularity.SINGLE);
    }
    this.runtimeContext = new StreamingRuntimeContext(environment, environment.getAccumulatorRegistry().getUserMap(), getMetricGroup(), getOperatorID(), getProcessingTimeService(), null, environment.getExternalResourceInfoProvider());
    stateKeySelector1 = config.getStatePartitioner(0, getUserCodeClassloader());
    stateKeySelector2 = config.getStatePartitioner(1, getUserCodeClassloader());
}

Example 58 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class SourceOperatorFactory method createStreamOperator.

@Override
public <T extends StreamOperator<OUT>> T createStreamOperator(StreamOperatorParameters<OUT> parameters) {
    final OperatorID operatorId = parameters.getStreamConfig().getOperatorID();
    final OperatorEventGateway gateway = parameters.getOperatorEventDispatcher().getOperatorEventGateway(operatorId);
    final SourceOperator<OUT, ?> sourceOperator = instantiateSourceOperator(source::createReader, gateway, source.getSplitSerializer(), watermarkStrategy, parameters.getProcessingTimeService(), parameters.getContainingTask().getEnvironment().getTaskManagerInfo().getConfiguration(), parameters.getContainingTask().getEnvironment().getTaskManagerInfo().getTaskManagerExternalAddress(), emitProgressiveWatermarks);
    sourceOperator.setup(parameters.getContainingTask(), parameters.getStreamConfig(), parameters.getOutput());
    parameters.getOperatorEventDispatcher().registerEventHandler(operatorId, sourceOperator);
    // today's lunch is generics spaghetti
    @SuppressWarnings("unchecked") final T castedOperator = (T) sourceOperator;
    return castedOperator;
}

Example 59 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class TaskLocalStateStoreImplTest method storeStates.

private List<TestingTaskStateSnapshot> storeStates(int count) {
    List<TestingTaskStateSnapshot> taskStateSnapshots = new ArrayList<>(count);
    for (int i = 0; i < count; ++i) {
        OperatorID operatorID = new OperatorID();
        TestingTaskStateSnapshot taskStateSnapshot = new TestingTaskStateSnapshot();
        OperatorSubtaskState operatorSubtaskState = OperatorSubtaskState.builder().build();
        taskStateSnapshot.putSubtaskStateByOperatorID(operatorID, operatorSubtaskState);
        taskLocalStateStore.storeLocalState(i, taskStateSnapshot);
        taskStateSnapshots.add(taskStateSnapshot);
    }
    return taskStateSnapshots;
}

Example 60 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class StateChangelogStorageTest method testWriteAndRead.

@Test
public void testWriteAndRead() throws Exception {
    KeyGroupRange kgRange = KeyGroupRange.of(0, 5);
    Map<Integer, List<byte[]>> appendsByKeyGroup = generateAppends(kgRange, 10, 20);
    try (StateChangelogStorage<T> client = getFactory();
        StateChangelogWriter<T> writer = client.createWriter(new OperatorID().toString(), kgRange)) {
        SequenceNumber prev = writer.initialSequenceNumber();
        for (Map.Entry<Integer, List<byte[]>> entry : appendsByKeyGroup.entrySet()) {
            Integer group = entry.getKey();
            List<byte[]> appends = entry.getValue();
            for (byte[] bytes : appends) {
                writer.append(group, bytes);
            }
        }
        T handle = writer.persist(prev).get();
        StateChangelogHandleReader<T> reader = client.createReader();
        assertByteMapsEqual(appendsByKeyGroup, extract(handle, reader));
    }
}