Examples with MiniClusterWithClientResource org.apache.flink.test.util.MiniClusterWithClientResource used on opensource projects

Example 16 with MiniClusterWithClientResource

use of org.apache.flink.test.util.MiniClusterWithClientResource in project flink by apache.

the class UnalignedCheckpointTestBase method execute.

@Nullable
protected File execute(UnalignedSettings settings) throws Exception {
    final File checkpointDir = temp.newFolder();
    Configuration conf = settings.getConfiguration(checkpointDir);
    // Configure DFS DSTL for this test as it might produce too much GC pressure if
    // ChangelogStateBackend is used.
    // Doing it on cluster level unconditionally as randomization currently happens on the job
    // level (environment); while this factory can only be set on the cluster level.
    FsStateChangelogStorageFactory.configure(conf, temp.newFolder());
    final StreamGraph streamGraph = getStreamGraph(settings, conf);
    final int requiredSlots = streamGraph.getStreamNodes().stream().mapToInt(node -> node.getParallelism()).reduce(0, settings.channelType.slotSharing ? Integer::max : Integer::sum);
    int numberTaskmanagers = settings.channelType.slotsToTaskManagers.apply(requiredSlots);
    final int slotsPerTM = (requiredSlots + numberTaskmanagers - 1) / numberTaskmanagers;
    final MiniClusterWithClientResource miniCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder().setConfiguration(conf).setNumberTaskManagers(numberTaskmanagers).setNumberSlotsPerTaskManager(slotsPerTM).build());
    miniCluster.before();
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(conf);
    settings.configure(env);
    try {
        // print the test parameters to help debugging when the case is stuck
        System.out.println("Starting " + getClass().getCanonicalName() + "#" + name.getMethodName() + ".");
        waitForCleanShutdown();
        final CompletableFuture<JobSubmissionResult> result = miniCluster.getMiniCluster().submitJob(streamGraph.getJobGraph());
        checkCounters(miniCluster.getMiniCluster().requestJobResult(result.get().getJobID()).get().toJobExecutionResult(getClass().getClassLoader()));
        System.out.println("Finished " + getClass().getCanonicalName() + "#" + name.getMethodName() + ".");
    } catch (Exception e) {
        if (!ExceptionUtils.findThrowable(e, TestException.class).isPresent()) {
            throw e;
        }
    } finally {
        miniCluster.after();
    }
    if (settings.generateCheckpoint) {
        return TestUtils.getMostRecentCompletedCheckpoint(checkpointDir);
    }
    return null;
}

Example 17 with MiniClusterWithClientResource

use of org.apache.flink.test.util.MiniClusterWithClientResource in project flink by apache.

the class UnalignedCheckpointCompatibilityITCase method setupMiniCluster.

@BeforeClass
public static void setupMiniCluster() throws Exception {
    File folder = temporaryFolder.getRoot();
    final Configuration conf = new Configuration();
    conf.set(CHECKPOINTS_DIRECTORY, folder.toURI().toString());
    // prevent deletion of checkpoint files while it's being checked and used
    conf.set(MAX_RETAINED_CHECKPOINTS, Integer.MAX_VALUE);
    miniCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder().setConfiguration(conf).build());
    miniCluster.before();
}

Example 18 with MiniClusterWithClientResource

use of org.apache.flink.test.util.MiniClusterWithClientResource in project flink by apache.

the class JdbcExactlyOnceSinkE2eTest method before.

@Before
public void before() throws Exception {
    Configuration configuration = new Configuration();
    // single failover region to allow checkpointing even after some sources have finished and
    // restart all tasks if at least one fails
    configuration.set(EXECUTION_FAILOVER_STRATEGY, "full");
    // cancel tasks eagerly to reduce the risk of running out of memory with many restarts
    configuration.set(TASK_CANCELLATION_TIMEOUT, TASK_CANCELLATION_TIMEOUT_MS);
    configuration.set(CHECKPOINTING_TIMEOUT, Duration.ofMillis(CHECKPOINT_TIMEOUT_MS));
    cluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder().setConfiguration(configuration).setNumberTaskManagers(dbEnv.getParallelism()).build());
    cluster.before();
    dbEnv.start();
    super.before();
}

Example 19 with MiniClusterWithClientResource

use of org.apache.flink.test.util.MiniClusterWithClientResource in project flink by apache.

the class HAQueryableStateRocksDBBackendITCase method setup.

@BeforeClass
public static void setup() throws Exception {
    zkServer = new TestingServer();
    // we have to manage this manually because we have to create the ZooKeeper server
    // ahead of this
    miniClusterResource = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder().setConfiguration(getConfig()).setNumberTaskManagers(NUM_TMS).setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM).build());
    miniClusterResource.before();
    client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START);
    clusterClient = miniClusterResource.getClusterClient();
}

Example 20 with MiniClusterWithClientResource

use of org.apache.flink.test.util.MiniClusterWithClientResource in project flink by apache.

the class HAQueryableStateFsBackendITCase method setup.

@BeforeClass
public static void setup() throws Exception {
    zkServer = new TestingServer();
    // we have to manage this manually because we have to create the ZooKeeper server
    // ahead of this
    miniClusterResource = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder().setConfiguration(getConfig()).setNumberTaskManagers(NUM_TMS).setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM).build());
    miniClusterResource.before();
    client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START);
    clusterClient = miniClusterResource.getClusterClient();
}