Examples with Environment org.apache.flink.runtime.execution.Environment used on opensource projects

Example 11 with Environment

use of org.apache.flink.runtime.execution.Environment in project flink by apache.

the class AbstractStreamOperator method setup.

// ------------------------------------------------------------------------
// Life Cycle
// ------------------------------------------------------------------------
@Override
public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) {
    final Environment environment = containingTask.getEnvironment();
    this.container = containingTask;
    this.config = config;
    try {
        InternalOperatorMetricGroup operatorMetricGroup = environment.getMetricGroup().getOrAddOperator(config.getOperatorID(), config.getOperatorName());
        this.output = new CountingOutput<>(output, operatorMetricGroup.getIOMetricGroup().getNumRecordsOutCounter());
        if (config.isChainEnd()) {
            operatorMetricGroup.getIOMetricGroup().reuseOutputMetricsForTask();
        }
        this.metrics = operatorMetricGroup;
    } catch (Exception e) {
        LOG.warn("An error occurred while instantiating task metrics.", e);
        this.metrics = UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup();
        this.output = output;
    }
    this.combinedWatermark = IndexedCombinedWatermarkStatus.forInputsCount(2);
    try {
        Configuration taskManagerConfig = environment.getTaskManagerInfo().getConfiguration();
        int historySize = taskManagerConfig.getInteger(MetricOptions.LATENCY_HISTORY_SIZE);
        if (historySize <= 0) {
            LOG.warn("{} has been set to a value equal or below 0: {}. Using default.", MetricOptions.LATENCY_HISTORY_SIZE, historySize);
            historySize = MetricOptions.LATENCY_HISTORY_SIZE.defaultValue();
        }
        final String configuredGranularity = taskManagerConfig.getString(MetricOptions.LATENCY_SOURCE_GRANULARITY);
        LatencyStats.Granularity granularity;
        try {
            granularity = LatencyStats.Granularity.valueOf(configuredGranularity.toUpperCase(Locale.ROOT));
        } catch (IllegalArgumentException iae) {
            granularity = LatencyStats.Granularity.OPERATOR;
            LOG.warn("Configured value {} option for {} is invalid. Defaulting to {}.", configuredGranularity, MetricOptions.LATENCY_SOURCE_GRANULARITY.key(), granularity);
        }
        MetricGroup jobMetricGroup = this.metrics.getJobMetricGroup();
        this.latencyStats = new LatencyStats(jobMetricGroup.addGroup("latency"), historySize, container.getIndexInSubtaskGroup(), getOperatorID(), granularity);
    } catch (Exception e) {
        LOG.warn("An error occurred while instantiating latency metrics.", e);
        this.latencyStats = new LatencyStats(UnregisteredMetricGroups.createUnregisteredTaskManagerJobMetricGroup().addGroup("latency"), 1, 0, new OperatorID(), LatencyStats.Granularity.SINGLE);
    }
    this.runtimeContext = new StreamingRuntimeContext(environment, environment.getAccumulatorRegistry().getUserMap(), getMetricGroup(), getOperatorID(), getProcessingTimeService(), null, environment.getExternalResourceInfoProvider());
    stateKeySelector1 = config.getStatePartitioner(0, getUserCodeClassloader());
    stateKeySelector2 = config.getStatePartitioner(1, getUserCodeClassloader());
}

Example 12 with Environment

use of org.apache.flink.runtime.execution.Environment in project flink by apache.

the class OperatorStateBackendTest method testCorrectClassLoaderUsedOnSnapshot.

@SuppressWarnings("unchecked")
@Test
public void testCorrectClassLoaderUsedOnSnapshot() throws Exception {
    AbstractStateBackend abstractStateBackend = new MemoryStateBackend(4096);
    final Environment env = createMockEnvironment();
    CloseableRegistry cancelStreamRegistry = new CloseableRegistry();
    OperatorStateBackend operatorStateBackend = abstractStateBackend.createOperatorStateBackend(env, "test-op-name", emptyStateHandles, cancelStreamRegistry);
    AtomicInteger copyCounter = new AtomicInteger(0);
    TypeSerializer<Integer> serializer = new VerifyingIntSerializer(env.getUserCodeClassLoader().asClassLoader(), copyCounter);
    // write some state
    ListStateDescriptor<Integer> stateDescriptor = new ListStateDescriptor<>("test", serializer);
    ListState<Integer> listState = operatorStateBackend.getListState(stateDescriptor);
    listState.add(42);
    AtomicInteger keyCopyCounter = new AtomicInteger(0);
    AtomicInteger valueCopyCounter = new AtomicInteger(0);
    TypeSerializer<Integer> keySerializer = new VerifyingIntSerializer(env.getUserCodeClassLoader().asClassLoader(), keyCopyCounter);
    TypeSerializer<Integer> valueSerializer = new VerifyingIntSerializer(env.getUserCodeClassLoader().asClassLoader(), valueCopyCounter);
    MapStateDescriptor<Integer, Integer> broadcastStateDesc = new MapStateDescriptor<>("test-broadcast", keySerializer, valueSerializer);
    BroadcastState<Integer, Integer> broadcastState = operatorStateBackend.getBroadcastState(broadcastStateDesc);
    broadcastState.put(1, 2);
    broadcastState.put(3, 4);
    broadcastState.put(5, 6);
    CheckpointStreamFactory streamFactory = new MemCheckpointStreamFactory(4096);
    RunnableFuture<SnapshotResult<OperatorStateHandle>> runnableFuture = operatorStateBackend.snapshot(1, 1, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
    FutureUtils.runIfNotDoneAndGet(runnableFuture);
    // make sure that the copy method has been called
    assertTrue(copyCounter.get() > 0);
    assertTrue(keyCopyCounter.get() > 0);
    assertTrue(valueCopyCounter.get() > 0);
}

Example 13 with Environment

use of org.apache.flink.runtime.execution.Environment in project flink by apache.

the class StreamOperatorChainingTest method testMultiChainingWithSplit.

/**
 * Verify that multi-chaining works with object reuse enabled.
 */
private void testMultiChainingWithSplit(StreamExecutionEnvironment env) throws Exception {
    // set parallelism to 2 to avoid chaining with source in case when available processors is
    // 1.
    env.setParallelism(2);
    // the actual elements will not be used
    DataStream<Integer> input = env.fromElements(1, 2, 3);
    sink1Results = new ArrayList<>();
    sink2Results = new ArrayList<>();
    sink3Results = new ArrayList<>();
    input = input.map(value -> value);
    OutputTag<Integer> oneOutput = new OutputTag<Integer>("one") {
    };
    OutputTag<Integer> otherOutput = new OutputTag<Integer>("other") {
    };
    SingleOutputStreamOperator<Object> split = input.process(new ProcessFunction<Integer, Object>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void processElement(Integer value, Context ctx, Collector<Object> out) throws Exception {
            if (value.equals(1)) {
                ctx.output(oneOutput, value);
            } else {
                ctx.output(otherOutput, value);
            }
        }
    });
    split.getSideOutput(oneOutput).map(value -> "First 1: " + value).addSink(new SinkFunction<String>() {

        @Override
        public void invoke(String value, Context ctx) throws Exception {
            sink1Results.add(value);
        }
    });
    split.getSideOutput(oneOutput).map(value -> "First 2: " + value).addSink(new SinkFunction<String>() {

        @Override
        public void invoke(String value, Context ctx) throws Exception {
            sink2Results.add(value);
        }
    });
    split.getSideOutput(otherOutput).map(value -> "Second: " + value).addSink(new SinkFunction<String>() {

        @Override
        public void invoke(String value, Context ctx) throws Exception {
            sink3Results.add(value);
        }
    });
    // be build our own StreamTask and OperatorChain
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    Assert.assertTrue(jobGraph.getVerticesSortedTopologicallyFromSources().size() == 2);
    JobVertex chainedVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
    Configuration configuration = chainedVertex.getConfiguration();
    StreamConfig streamConfig = new StreamConfig(configuration);
    StreamMap<Integer, Integer> headOperator = streamConfig.getStreamOperator(Thread.currentThread().getContextClassLoader());
    try (MockEnvironment environment = createMockEnvironment(chainedVertex.getName())) {
        StreamTask<Integer, StreamMap<Integer, Integer>> mockTask = createMockTask(streamConfig, environment);
        OperatorChain<Integer, StreamMap<Integer, Integer>> operatorChain = createOperatorChain(streamConfig, environment, mockTask);
        headOperator.setup(mockTask, streamConfig, operatorChain.getMainOperatorOutput());
        operatorChain.initializeStateAndOpenOperators(null);
        headOperator.processElement(new StreamRecord<>(1));
        headOperator.processElement(new StreamRecord<>(2));
        headOperator.processElement(new StreamRecord<>(3));
        assertThat(sink1Results, contains("First 1: 1"));
        assertThat(sink2Results, contains("First 2: 1"));
        assertThat(sink3Results, contains("Second: 2", "Second: 3"));
    }
}

Example 14 with Environment

use of org.apache.flink.runtime.execution.Environment in project flink by apache.

the class StreamSourceOperatorLatencyMetricsTest method testLatencyMarkEmissionEnabledViaFlinkConfig.

/**
 * Verifies that latency metrics can be enabled via the configuration.
 */
@Test
public void testLatencyMarkEmissionEnabledViaFlinkConfig() throws Exception {
    testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> {
        Configuration tmConfig = new Configuration();
        tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval);
        Environment env = MockEnvironment.builder().setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig)).build();
        setupSourceOperator(operator, new ExecutionConfig(), env, timeProvider);
    });
}

Example 15 with Environment

use of org.apache.flink.runtime.execution.Environment in project flink by apache.

the class StreamSourceOperatorLatencyMetricsTest method testLatencyMarkEmissionEnabledOverrideViaExecutionConfig.

/**
 * Verifies that latency metrics can be enabled via the {@link ExecutionConfig} even if they are
 * disabled via the configuration.
 */
@Test
public void testLatencyMarkEmissionEnabledOverrideViaExecutionConfig() throws Exception {
    testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> {
        ExecutionConfig executionConfig = new ExecutionConfig();
        executionConfig.setLatencyTrackingInterval(latencyMarkInterval);
        Configuration tmConfig = new Configuration();
        tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, 0L);
        Environment env = MockEnvironment.builder().setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig)).build();
        setupSourceOperator(operator, executionConfig, env, timeProvider);
    });
}