Examples with MiniClusterConfiguration org.apache.flink.runtime.minicluster.MiniClusterConfiguration used on opensource projects

Example 11 with MiniClusterConfiguration

use of org.apache.flink.runtime.minicluster.MiniClusterConfiguration in project beam by apache.

the class FlinkMiniClusterEntryPoint method main.

public static void main(String[] args) throws Exception {
    MiniClusterArgs miniClusterArgs = parseArgs(args);
    Configuration flinkConfig = new Configuration();
    flinkConfig.setInteger(RestOptions.PORT, miniClusterArgs.restPort);
    if (!miniClusterArgs.restBindAddress.isEmpty()) {
        flinkConfig.setString(RestOptions.BIND_ADDRESS, miniClusterArgs.restBindAddress);
    }
    MiniClusterConfiguration clusterConfig = new MiniClusterConfiguration.Builder().setConfiguration(flinkConfig).setNumTaskManagers(miniClusterArgs.numTaskManagers).setNumSlotsPerTaskManager(miniClusterArgs.numSlotsPerTaskManager).build();
    try (MiniCluster miniCluster = new MiniCluster(clusterConfig)) {
        miniCluster.start();
        System.out.println(String.format("Started Flink mini cluster (%s TaskManagers with %s task slots) with Rest API at %s", miniClusterArgs.numTaskManagers, miniClusterArgs.numSlotsPerTaskManager, miniCluster.getRestAddress()));
        Thread.sleep(Long.MAX_VALUE);
    }
}

Example 12 with MiniClusterConfiguration

use of org.apache.flink.runtime.minicluster.MiniClusterConfiguration in project beam by apache.

the class FlinkSavepointTest method beforeClass.

@BeforeClass
public static void beforeClass() throws Exception {
    flinkJobExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    Configuration config = new Configuration();
    // Avoid port collision in parallel tests
    config.setInteger(RestOptions.PORT, 0);
    config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
    String savepointPath = "file://" + tempFolder.getRoot().getAbsolutePath();
    LOG.info("Savepoints will be written to {}", savepointPath);
    // It is necessary to configure the checkpoint directory for the state backend,
    // even though we only create savepoints in this test.
    config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, savepointPath);
    // Checkpoints will go into a subdirectory of this directory
    config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointPath);
    MiniClusterConfiguration clusterConfig = new MiniClusterConfiguration.Builder().setConfiguration(config).setNumTaskManagers(2).setNumSlotsPerTaskManager(2).build();
    flinkCluster = new MiniCluster(clusterConfig);
    flinkCluster.start();
}

Example 13 with MiniClusterConfiguration

use of org.apache.flink.runtime.minicluster.MiniClusterConfiguration in project beam by apache.

the class FlinkRequiresStableInputTest method beforeClass.

@BeforeClass
public static void beforeClass() throws Exception {
    final int parallelism = 1;
    Configuration config = new Configuration();
    // Avoid port collision in parallel tests
    config.setInteger(RestOptions.PORT, 0);
    config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
    // It is necessary to configure the checkpoint directory for the state backend,
    // even though we only create savepoints in this test.
    config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, "file://" + tempFolder.getRoot().getAbsolutePath());
    // Checkpoints will go into a subdirectory of this directory
    config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, "file://" + tempFolder.getRoot().getAbsolutePath());
    MiniClusterConfiguration clusterConfig = new MiniClusterConfiguration.Builder().setConfiguration(config).setNumTaskManagers(1).setNumSlotsPerTaskManager(1).build();
    flinkCluster = new MiniCluster(clusterConfig);
    flinkCluster.start();
    TestStreamEnvironment.setAsContext(flinkCluster, parallelism);
}

Example 14 with MiniClusterConfiguration

use of org.apache.flink.runtime.minicluster.MiniClusterConfiguration in project beam by apache.

the class FlinkSubmissionTest method beforeClass.

@BeforeClass
public static void beforeClass() throws Exception {
    Configuration config = new Configuration();
    // Avoid port collision in parallel tests on the same machine
    config.setInteger(RestOptions.PORT.key(), 0);
    MiniClusterConfiguration clusterConfig = new MiniClusterConfiguration.Builder().setConfiguration(config).setNumTaskManagers(1).setNumSlotsPerTaskManager(1).setRpcServiceSharing(RpcServiceSharing.SHARED).build();
    flinkCluster = new RemoteMiniClusterImpl(clusterConfig);
    flinkCluster.start();
    prepareEnvironment();
}

Example 15 with MiniClusterConfiguration

use of org.apache.flink.runtime.minicluster.MiniClusterConfiguration in project flink by apache.

the class MiniClusterResource method startMiniCluster.

private void startMiniCluster() throws Exception {
    final Configuration configuration = new Configuration(miniClusterResourceConfiguration.getConfiguration());
    configuration.setString(CoreOptions.TMP_DIRS, temporaryFolder.newFolder().getAbsolutePath());
    // enabled this by default
    if (!configuration.contains(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE)) {
        configuration.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
    }
    if (!configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) {
        configuration.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, DEFAULT_MANAGED_MEMORY_SIZE);
    }
    // set rest and rpc port to 0 to avoid clashes with concurrent MiniClusters
    configuration.setInteger(JobManagerOptions.PORT, 0);
    configuration.setString(RestOptions.BIND_PORT, "0");
    randomizeConfiguration(configuration);
    final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder().setConfiguration(configuration).setNumTaskManagers(miniClusterResourceConfiguration.getNumberTaskManagers()).setNumSlotsPerTaskManager(miniClusterResourceConfiguration.getNumberSlotsPerTaskManager()).setRpcServiceSharing(miniClusterResourceConfiguration.getRpcServiceSharing()).setHaServices(miniClusterResourceConfiguration.getHaServices()).build();
    miniCluster = new MiniCluster(miniClusterConfiguration, () -> Reference.borrowed(rpcSystem));
    miniCluster.start();
    final URI restAddress = miniCluster.getRestAddress().get();
    createClientConfiguration(restAddress);
}