Examples with MiniClusterWithClientResource org.apache.flink.test.util.MiniClusterWithClientResource used on opensource projects

Example 1 with MiniClusterWithClientResource

use of org.apache.flink.test.util.MiniClusterWithClientResource in project flink by apache.

the class HistoryServerTest method setUp.

@Before
public void setUp() throws Exception {
    jmDirectory = tmpFolder.newFolder("jm_" + versionLessThan14);
    hsDirectory = tmpFolder.newFolder("hs_" + versionLessThan14);
    Configuration clusterConfig = new Configuration();
    clusterConfig.setString(JobManagerOptions.ARCHIVE_DIR, jmDirectory.toURI().toString());
    cluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder().setConfiguration(clusterConfig).setNumberTaskManagers(1).setNumberSlotsPerTaskManager(2).build());
    cluster.before();
}

Example 2 with MiniClusterWithClientResource

use of org.apache.flink.test.util.MiniClusterWithClientResource in project flink by apache.

the class PartiallyFinishedSourcesITCase method init.

@Before
public void init() throws Exception {
    Configuration configuration = new Configuration();
    // set failover strategy on the cluster level
    // choose it from the parameter because it may affect the test
    // - "region" is currently the default
    // - "full" is enforced by Adaptive/Reactive scheduler (even when parameterized)
    configuration.set(EXECUTION_FAILOVER_STRATEGY, failoverStrategy);
    // If changelog backend is enabled then this test might run too slow with in-memory
    // implementation - use fs-based instead.
    // The randomization currently happens on the job level (environment); while this factory
    // can only be set on the cluster level; so we do it unconditionally here.
    configuration.setString(STATE_CHANGE_LOG_STORAGE, IDENTIFIER);
    configuration.setString(BASE_PATH, TEMPORARY_FOLDER.newFolder().getAbsolutePath());
    miniClusterResource = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder().setConfiguration(configuration).setNumberTaskManagers(1).setNumberSlotsPerTaskManager(4).build());
    miniClusterResource.before();
}

Example 3 with MiniClusterWithClientResource

use of org.apache.flink.test.util.MiniClusterWithClientResource in project flink by apache.

the class NetworkStackThroughputITCase method testThroughput.

// ------------------------------------------------------------------------
@Test
public void testThroughput() throws Exception {
    Object[][] configParams = new Object[][] { new Object[] { 1, false, false, false, 4, 2 }, new Object[] { 1, true, false, false, 4, 2 }, new Object[] { 1, true, true, false, 4, 2 }, new Object[] { 1, true, false, true, 4, 2 }, new Object[] { 2, true, false, false, 4, 2 }, new Object[] { 4, true, false, false, 4, 2 }, new Object[] { 4, true, false, false, 8, 4 } };
    for (Object[] p : configParams) {
        final int dataVolumeGb = (Integer) p[0];
        final boolean useForwarder = (Boolean) p[1];
        final boolean isSlowSender = (Boolean) p[2];
        final boolean isSlowReceiver = (Boolean) p[3];
        final int parallelism = (Integer) p[4];
        final int numSlotsPerTaskManager = (Integer) p[5];
        if (parallelism % numSlotsPerTaskManager != 0) {
            throw new RuntimeException("The test case defines a parallelism that is not a multiple of the slots per task manager.");
        }
        final int numTaskManagers = parallelism / numSlotsPerTaskManager;
        final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder().setNumberTaskManagers(numTaskManagers).setNumberSlotsPerTaskManager(numSlotsPerTaskManager).build());
        cluster.before();
        try {
            System.out.println(String.format("Running test with parameters: dataVolumeGB=%s, useForwarder=%s, isSlowSender=%s, isSlowReceiver=%s, parallelism=%s, numSlotsPerTM=%s", dataVolumeGb, useForwarder, isSlowSender, isSlowReceiver, parallelism, numSlotsPerTaskManager));
            testProgram(cluster, dataVolumeGb, useForwarder, isSlowSender, isSlowReceiver, parallelism);
        } finally {
            cluster.after();
        }
    }
}

Example 4 with MiniClusterWithClientResource

use of org.apache.flink.test.util.MiniClusterWithClientResource in project flink by apache.

the class NettyEpollITCase method testNettyEpoll.

@Test
public void testNettyEpoll() throws Exception {
    MiniClusterWithClientResource cluster = trySetUpCluster();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(NUM_TASK_MANAGERS);
        DataStream<Integer> input = env.fromElements(1, 2, 3, 4, 1, 2, 3, 42);
        input.keyBy(new KeySelector<Integer, Integer>() {

            @Override
            public Integer getKey(Integer value) throws Exception {
                return value;
            }
        }).sum(0).print();
        env.execute();
    } finally {
        cluster.after();
    }
}

Example 5 with MiniClusterWithClientResource

use of org.apache.flink.test.util.MiniClusterWithClientResource in project flink by apache.

the class StreamingScalabilityAndLatency method main.

public static void main(String[] args) throws Exception {
    if ((Runtime.getRuntime().maxMemory() >>> 20) < 5000) {
        throw new RuntimeException("This test program needs to run with at least 5GB of heap space.");
    }
    final int taskManagers = 1;
    final int slotsPerTaskManager = 80;
    final int parallelism = taskManagers * slotsPerTaskManager;
    MiniClusterWithClientResource cluster = null;
    try {
        Configuration config = new Configuration();
        config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("80m"));
        config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 20000);
        config.setInteger("taskmanager.net.server.numThreads", 1);
        config.setInteger("taskmanager.net.client.numThreads", 1);
        cluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder().setConfiguration(config).setNumberTaskManagers(taskManagers).setNumberSlotsPerTaskManager(slotsPerTaskManager).build());
        cluster.before();
        runPartitioningProgram(parallelism);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        if (cluster != null) {
            cluster.after();
        }
    }
}