Examples with RemoteStreamEnvironment org.apache.flink.streaming.api.environment.RemoteStreamEnvironment used on opensource projects

Example 1 with RemoteStreamEnvironment

use of org.apache.flink.streaming.api.environment.RemoteStreamEnvironment in project flink by apache.

the class DataStreamUtils method collect.

/**
	 * Returns an iterator to iterate over the elements of the DataStream.
	 * @return The iterator
	 */
public static <OUT> Iterator<OUT> collect(DataStream<OUT> stream) throws IOException {
    TypeSerializer<OUT> serializer = stream.getType().createSerializer(stream.getExecutionEnvironment().getConfig());
    SocketStreamIterator<OUT> iter = new SocketStreamIterator<OUT>(serializer);
    //Find out what IP of us should be given to CollectSink, that it will be able to connect to
    StreamExecutionEnvironment env = stream.getExecutionEnvironment();
    InetAddress clientAddress;
    if (env instanceof RemoteStreamEnvironment) {
        String host = ((RemoteStreamEnvironment) env).getHost();
        int port = ((RemoteStreamEnvironment) env).getPort();
        try {
            clientAddress = ConnectionUtils.findConnectingAddress(new InetSocketAddress(host, port), 2000, 400);
        } catch (Exception e) {
            throw new IOException("Could not determine an suitable network address to " + "receive back data from the streaming program.", e);
        }
    } else if (env instanceof LocalStreamEnvironment) {
        clientAddress = InetAddress.getLoopbackAddress();
    } else {
        try {
            clientAddress = InetAddress.getLocalHost();
        } catch (UnknownHostException e) {
            throw new IOException("Could not determine this machines own local address to " + "receive back data from the streaming program.", e);
        }
    }
    DataStreamSink<OUT> sink = stream.addSink(new CollectSink<OUT>(clientAddress, iter.getPort(), serializer));
    // It would not work if multiple instances would connect to the same port
    sink.setParallelism(1);
    (new CallExecute(env, iter)).start();
    return iter;
}

Example 2 with RemoteStreamEnvironment

use of org.apache.flink.streaming.api.environment.RemoteStreamEnvironment in project flink by apache.

the class RemoteStreamEnvironmentTest method testRemoteExecutionWithSavepoint.

@Test
public void testRemoteExecutionWithSavepoint() throws Exception {
    SavepointRestoreSettings restoreSettings = SavepointRestoreSettings.forPath("fakePath");
    JobID jobID = new JobID();
    TestExecutorServiceLoader testExecutorServiceLoader = new TestExecutorServiceLoader(jobID);
    RemoteStreamEnvironment env = new RemoteStreamEnvironment(testExecutorServiceLoader, "fakeHost", 1, null, new String[] {}, null, restoreSettings);
    env.fromElements(1).map(x -> x * 2);
    JobExecutionResult actualResult = env.execute("fakeJobName");
    assertThat(actualResult.getJobID(), is(jobID));
    assertThat(testExecutorServiceLoader.getActualSavepointRestoreSettings(), is(restoreSettings));
}

Example 3 with RemoteStreamEnvironment

use of org.apache.flink.streaming.api.environment.RemoteStreamEnvironment in project beam by apache.

the class FlinkExecutionEnvironments method createStreamExecutionEnvironment.

@VisibleForTesting
static StreamExecutionEnvironment createStreamExecutionEnvironment(FlinkPipelineOptions options, List<String> filesToStage, @Nullable String confDir) {
    LOG.info("Creating a Streaming Environment.");
    // Although Flink uses Rest, it expects the address not to contain a http scheme
    String masterUrl = stripHttpSchema(options.getFlinkMaster());
    Configuration flinkConfiguration = getFlinkConfiguration(confDir);
    StreamExecutionEnvironment flinkStreamEnv;
    // depending on the master, create the right environment.
    if ("[local]".equals(masterUrl)) {
        setManagedMemoryByFraction(flinkConfiguration);
        disableClassLoaderLeakCheck(flinkConfiguration);
        flinkStreamEnv = StreamExecutionEnvironment.createLocalEnvironment(getDefaultLocalParallelism(), flinkConfiguration);
    } else if ("[auto]".equals(masterUrl)) {
        flinkStreamEnv = StreamExecutionEnvironment.getExecutionEnvironment();
        if (flinkStreamEnv instanceof LocalStreamEnvironment) {
            disableClassLoaderLeakCheck(flinkConfiguration);
            flinkStreamEnv = StreamExecutionEnvironment.createLocalEnvironment(getDefaultLocalParallelism(), flinkConfiguration);
        }
    } else {
        int defaultPort = flinkConfiguration.getInteger(RestOptions.PORT);
        HostAndPort hostAndPort = HostAndPort.fromString(masterUrl).withDefaultPort(defaultPort);
        flinkConfiguration.setInteger(RestOptions.PORT, hostAndPort.getPort());
        final SavepointRestoreSettings savepointRestoreSettings;
        if (options.getSavepointPath() != null) {
            savepointRestoreSettings = SavepointRestoreSettings.forPath(options.getSavepointPath(), options.getAllowNonRestoredState());
        } else {
            savepointRestoreSettings = SavepointRestoreSettings.none();
        }
        flinkStreamEnv = new RemoteStreamEnvironment(hostAndPort.getHost(), hostAndPort.getPort(), flinkConfiguration, filesToStage.toArray(new String[filesToStage.size()]), null, savepointRestoreSettings);
        LOG.info("Using Flink Master URL {}:{}.", hostAndPort.getHost(), hostAndPort.getPort());
    }
    // Set the parallelism, required by UnboundedSourceWrapper to generate consistent splits.
    final int parallelism = determineParallelism(options.getParallelism(), flinkStreamEnv.getParallelism(), flinkConfiguration);
    flinkStreamEnv.setParallelism(parallelism);
    if (options.getMaxParallelism() > 0) {
        flinkStreamEnv.setMaxParallelism(options.getMaxParallelism());
    }
    // set parallelism in the options (required by some execution code)
    options.setParallelism(parallelism);
    if (options.getObjectReuse()) {
        flinkStreamEnv.getConfig().enableObjectReuse();
    } else {
        flinkStreamEnv.getConfig().disableObjectReuse();
    }
    // default to event time
    flinkStreamEnv.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    // for the following 2 parameters, a value of -1 means that Flink will use
    // the default values as specified in the configuration.
    int numRetries = options.getNumberOfExecutionRetries();
    if (numRetries != -1) {
        flinkStreamEnv.setNumberOfExecutionRetries(numRetries);
    }
    long retryDelay = options.getExecutionRetryDelay();
    if (retryDelay != -1) {
        flinkStreamEnv.getConfig().setExecutionRetryDelay(retryDelay);
    }
    configureCheckpointing(options, flinkStreamEnv);
    applyLatencyTrackingInterval(flinkStreamEnv.getConfig(), options);
    if (options.getAutoWatermarkInterval() != null) {
        flinkStreamEnv.getConfig().setAutoWatermarkInterval(options.getAutoWatermarkInterval());
    }
    configureStateBackend(options, flinkStreamEnv);
    return flinkStreamEnv;
}

Example 4 with RemoteStreamEnvironment

use of org.apache.flink.streaming.api.environment.RemoteStreamEnvironment in project flink by apache.

the class FlinkILoopTest method testConfigurationForwardingStreamEnvironment.

@Test
public void testConfigurationForwardingStreamEnvironment() {
    Configuration configuration = new Configuration();
    configuration.setString("foobar", "foobar");
    FlinkILoop flinkILoop = new FlinkILoop("localhost", 6123, configuration, Option.<String[]>empty());
    StreamExecutionEnvironment streamEnv = flinkILoop.scalaSenv().getJavaEnv();
    assertTrue(streamEnv instanceof RemoteStreamEnvironment);
    RemoteStreamEnvironment remoteStreamEnv = (RemoteStreamEnvironment) streamEnv;
    Configuration forwardedConfiguration = remoteStreamEnv.getClientConfiguration();
    assertEquals(configuration, forwardedConfiguration);
}

Example 5 with RemoteStreamEnvironment

use of org.apache.flink.streaming.api.environment.RemoteStreamEnvironment in project flink by apache.

the class RemoteStreamEnvironmentTest method testPortForwarding.

/**
 * Verifies that the port passed to the RemoteStreamEnvironment is used for connecting to the
 * cluster.
 */
@Test
public void testPortForwarding() throws Exception {
    String host = "fakeHost";
    int port = 99;
    JobID jobId = new JobID();
    final Configuration clientConfiguration = new Configuration();
    TestExecutorServiceLoader testExecutorServiceLoader = new TestExecutorServiceLoader(jobId);
    final StreamExecutionEnvironment env = new RemoteStreamEnvironment(testExecutorServiceLoader, host, port, clientConfiguration, null, null, null);
    env.fromElements(1).map(x -> x * 2);
    JobExecutionResult actualResult = env.execute("fakeJobName");
    TestClusterClient testClient = testExecutorServiceLoader.getCreatedClusterClient();
    assertThat(actualResult.getJobID(), is(jobId));
    assertThat(testClient.getConfiguration().getString(RestOptions.ADDRESS), is(host));
    assertThat(testClient.getConfiguration().getInteger(RestOptions.PORT), is(99));
}