Examples with MiniClusterConfiguration org.apache.flink.runtime.minicluster.MiniClusterConfiguration used on opensource projects

Example 6 with MiniClusterConfiguration

use of org.apache.flink.runtime.minicluster.MiniClusterConfiguration in project flink by apache.

the class Flip6LocalStreamEnvironment method execute.

/**
	 * Executes the JobGraph of the on a mini cluster of CLusterUtil with a user
	 * specified name.
	 * 
	 * @param jobName
	 *            name of the job
	 * @return The result of the job execution, containing elapsed time and accumulators.
	 */
@Override
public JobExecutionResult execute(String jobName) throws Exception {
    // transform the streaming program into a JobGraph
    StreamGraph streamGraph = getStreamGraph();
    streamGraph.setJobName(jobName);
    // TODO - temp fix to enforce restarts due to a bug in the allocation protocol
    streamGraph.getExecutionConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 5));
    JobGraph jobGraph = streamGraph.getJobGraph();
    jobGraph.setAllowQueuedScheduling(true);
    Configuration configuration = new Configuration();
    configuration.addAll(jobGraph.getJobConfiguration());
    configuration.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1L);
    // add (and override) the settings with what the user defined
    configuration.addAll(this.conf);
    MiniClusterConfiguration cfg = new MiniClusterConfiguration(configuration);
    // Currently we do not reuse slot anymore,
    // so we need to sum up the parallelism of all vertices
    int slotsCount = 0;
    for (JobVertex jobVertex : jobGraph.getVertices()) {
        slotsCount += jobVertex.getParallelism();
    }
    cfg.setNumTaskManagerSlots(slotsCount);
    if (LOG.isInfoEnabled()) {
        LOG.info("Running job on local embedded Flink mini cluster");
    }
    MiniCluster miniCluster = new MiniCluster(cfg);
    try {
        miniCluster.start();
        miniCluster.waitUntilTaskManagerRegistrationsComplete();
        return miniCluster.runJobBlocking(jobGraph);
    } finally {
        transformations.clear();
        miniCluster.shutdown();
    }
}

Example 7 with MiniClusterConfiguration

use of org.apache.flink.runtime.minicluster.MiniClusterConfiguration in project flink by apache.

the class FileSinkCompactionSwitchITCase method testSwitchingCompaction.

@Test
public void testSwitchingCompaction() throws Exception {
    String path = TEMPORARY_FOLDER.newFolder().getAbsolutePath();
    String cpPath = "file://" + TEMPORARY_FOLDER.newFolder().getAbsolutePath();
    SharedReference<ConcurrentHashMap<Integer, Integer>> sendCountMap = sharedObjects.add(new ConcurrentHashMap<>());
    JobGraph jobGraph = createJobGraph(path, cpPath, isOnToOff, false, sendCountMap);
    JobGraph restoringJobGraph = createJobGraph(path, cpPath, !isOnToOff, true, sendCountMap);
    final Configuration config = new Configuration();
    config.setString(RestOptions.BIND_PORT, "18081-19000");
    final MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder().setNumTaskManagers(1).setNumSlotsPerTaskManager(4).setConfiguration(config).build();
    try (MiniCluster miniCluster = new MiniCluster(cfg)) {
        miniCluster.start();
        miniCluster.submitJob(jobGraph);
        LATCH_MAP.get(latchId).await();
        String savepointPath = miniCluster.triggerSavepoint(jobGraph.getJobID(), TEMPORARY_FOLDER.newFolder().getAbsolutePath(), true, SavepointFormatType.CANONICAL).get();
        // We wait for two successful checkpoints in sources before shutting down. This ensures
        // that the sink can commit its data.
        LATCH_MAP.put(latchId, new CountDownLatch(NUM_SOURCES * 2));
        restoringJobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath, false));
        miniCluster.executeJobBlocking(restoringJobGraph);
    }
    checkIntegerSequenceSinkOutput(path, sendCountMap.get(), NUM_BUCKETS, NUM_SOURCES);
}

Example 8 with MiniClusterConfiguration

use of org.apache.flink.runtime.minicluster.MiniClusterConfiguration in project flink by apache.

the class FileSinkMigrationITCase method testMigration.

@Test
public void testMigration() throws Exception {
    String outputPath = TEMPORARY_FOLDER.newFolder().getAbsolutePath();
    String savepointBasePath = TEMPORARY_FOLDER.newFolder().getAbsolutePath();
    final Configuration config = new Configuration();
    config.setString(RestOptions.BIND_PORT, "18081-19000");
    final MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder().setNumTaskManagers(1).setNumSlotsPerTaskManager(4).setConfiguration(config).build();
    JobGraph streamingFileSinkJobGraph = createStreamingFileSinkJobGraph(outputPath);
    String savepointPath = executeAndTakeSavepoint(cfg, streamingFileSinkJobGraph, savepointBasePath);
    JobGraph fileSinkJobGraph = createFileSinkJobGraph(outputPath);
    loadSavepointAndExecute(cfg, fileSinkJobGraph, savepointPath);
    IntegerFileSinkTestDataUtils.checkIntegerSequenceSinkOutput(outputPath, NUM_RECORDS, NUM_BUCKETS, NUM_SOURCES);
}

Example 9 with MiniClusterConfiguration

use of org.apache.flink.runtime.minicluster.MiniClusterConfiguration in project flink by apache.

the class PerJobMiniClusterFactory method submitJob.

/**
 * Starts a {@link MiniCluster} and submits a job.
 */
public CompletableFuture<JobClient> submitJob(JobGraph jobGraph, ClassLoader userCodeClassloader) throws Exception {
    MiniClusterConfiguration miniClusterConfig = getMiniClusterConfig(jobGraph.getMaximumParallelism());
    MiniCluster miniCluster = miniClusterFactory.apply(miniClusterConfig);
    miniCluster.start();
    return miniCluster.submitJob(jobGraph).thenApplyAsync(FunctionUtils.uncheckedFunction(submissionResult -> {
        org.apache.flink.client.ClientUtils.waitUntilJobInitializationFinished(() -> miniCluster.getJobStatus(submissionResult.getJobID()).get(), () -> miniCluster.requestJobResult(submissionResult.getJobID()).get(), userCodeClassloader);
        return submissionResult;
    })).thenApply(result -> new MiniClusterJobClient(result.getJobID(), miniCluster, userCodeClassloader, MiniClusterJobClient.JobFinalizationBehavior.SHUTDOWN_CLUSTER)).whenComplete((ignored, throwable) -> {
        if (throwable != null) {
            // We failed to create the JobClient and must shutdown to ensure
            // cleanup.
            shutDownCluster(miniCluster);
        }
    }).thenApply(Function.identity());
}

Example 10 with MiniClusterConfiguration

use of org.apache.flink.runtime.minicluster.MiniClusterConfiguration in project flink by apache.

the class PerJobMiniClusterFactory method getMiniClusterConfig.

private MiniClusterConfiguration getMiniClusterConfig(int maximumParallelism) {
    Configuration configuration = new Configuration(this.configuration);
    if (!configuration.contains(RestOptions.BIND_PORT)) {
        configuration.setString(RestOptions.BIND_PORT, "0");
    }
    int numTaskManagers = configuration.getInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, ConfigConstants.DEFAULT_LOCAL_NUMBER_TASK_MANAGER);
    int numSlotsPerTaskManager = configuration.getOptional(TaskManagerOptions.NUM_TASK_SLOTS).orElseGet(() -> maximumParallelism > 0 ? MathUtils.divideRoundUp(maximumParallelism, numTaskManagers) : TaskManagerOptions.NUM_TASK_SLOTS.defaultValue());
    return new MiniClusterConfiguration.Builder().setConfiguration(configuration).setNumTaskManagers(numTaskManagers).setRpcServiceSharing(RpcServiceSharing.SHARED).setNumSlotsPerTaskManager(numSlotsPerTaskManager).build();
}