Examples with MiniCluster org.apache.flink.runtime.minicluster.MiniCluster used on opensource projects

Example 21 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class ManualCheckpointITCase method testTriggeringWhenPeriodicEnabled.

@Test
public void testTriggeringWhenPeriodicEnabled() throws Exception {
    int parallelism = MINI_CLUSTER_RESOURCE.getNumberSlots();
    final int checkpointingInterval = 500;
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(parallelism);
    env.enableCheckpointing(checkpointingInterval);
    env.getCheckpointConfig().setCheckpointStorage(storageSupplier.apply(temporaryFolder.newFolder().toURI().toString()));
    env.fromSource(MockSource.continuous(parallelism).build(), WatermarkStrategy.noWatermarks(), "generator").keyBy(key -> key % parallelism).flatMap(new StatefulMapper()).addSink(new DiscardingSink<>());
    final JobClient jobClient = env.executeAsync();
    final JobID jobID = jobClient.getJobID();
    final MiniCluster miniCluster = MINI_CLUSTER_RESOURCE.getMiniCluster();
    CommonTestUtils.waitForJobStatus(jobClient, Collections.singletonList(JobStatus.RUNNING), Deadline.fromNow(Duration.ofSeconds(30)));
    CommonTestUtils.waitForAllTaskRunning(miniCluster, jobID, false);
    CommonTestUtils.waitUntilCondition(() -> queryCompletedCheckpoints(miniCluster, jobID) > 0L, Deadline.fromNow(Duration.ofSeconds(30)), checkpointingInterval / 2);
    final long numberOfPeriodicCheckpoints = queryCompletedCheckpoints(miniCluster, jobID);
    // wait for the checkpoint to be taken
    miniCluster.triggerCheckpoint(jobID).get();
    miniCluster.cancelJob(jobID).get();
    queryCompletedCheckpointsUntil(miniCluster, jobID, count -> count >= numberOfPeriodicCheckpoints + 1);
}

Example 22 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class ManualCheckpointITCase method testTriggeringWhenPeriodicDisabled.

@Test
public void testTriggeringWhenPeriodicDisabled() throws Exception {
    int parallelism = MINI_CLUSTER_RESOURCE.getNumberSlots();
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(parallelism);
    env.getCheckpointConfig().setCheckpointStorage(storageSupplier.apply(temporaryFolder.newFolder().toURI().toString()));
    env.fromSource(MockSource.continuous(parallelism).build(), WatermarkStrategy.noWatermarks(), "generator").keyBy(key -> key % parallelism).flatMap(new StatefulMapper()).addSink(new DiscardingSink<>());
    final JobClient jobClient = env.executeAsync();
    final JobID jobID = jobClient.getJobID();
    final MiniCluster miniCluster = MINI_CLUSTER_RESOURCE.getMiniCluster();
    CommonTestUtils.waitForJobStatus(jobClient, Collections.singletonList(JobStatus.RUNNING), Deadline.fromNow(Duration.ofSeconds(30)));
    CommonTestUtils.waitForAllTaskRunning(miniCluster, jobID, false);
    // wait for the checkpoint to be taken
    miniCluster.triggerCheckpoint(jobID).get();
    miniCluster.cancelJob(jobID).get();
    queryCompletedCheckpointsUntil(miniCluster, jobID, count -> count == 1);
}

Example 23 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class Flip6LocalStreamEnvironment method execute.

/**
	 * Executes the JobGraph of the on a mini cluster of CLusterUtil with a user
	 * specified name.
	 * 
	 * @param jobName
	 *            name of the job
	 * @return The result of the job execution, containing elapsed time and accumulators.
	 */
@Override
public JobExecutionResult execute(String jobName) throws Exception {
    // transform the streaming program into a JobGraph
    StreamGraph streamGraph = getStreamGraph();
    streamGraph.setJobName(jobName);
    // TODO - temp fix to enforce restarts due to a bug in the allocation protocol
    streamGraph.getExecutionConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 5));
    JobGraph jobGraph = streamGraph.getJobGraph();
    jobGraph.setAllowQueuedScheduling(true);
    Configuration configuration = new Configuration();
    configuration.addAll(jobGraph.getJobConfiguration());
    configuration.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1L);
    // add (and override) the settings with what the user defined
    configuration.addAll(this.conf);
    MiniClusterConfiguration cfg = new MiniClusterConfiguration(configuration);
    // Currently we do not reuse slot anymore,
    // so we need to sum up the parallelism of all vertices
    int slotsCount = 0;
    for (JobVertex jobVertex : jobGraph.getVertices()) {
        slotsCount += jobVertex.getParallelism();
    }
    cfg.setNumTaskManagerSlots(slotsCount);
    if (LOG.isInfoEnabled()) {
        LOG.info("Running job on local embedded Flink mini cluster");
    }
    MiniCluster miniCluster = new MiniCluster(cfg);
    try {
        miniCluster.start();
        miniCluster.waitUntilTaskManagerRegistrationsComplete();
        return miniCluster.runJobBlocking(jobGraph);
    } finally {
        transformations.clear();
        miniCluster.shutdown();
    }
}

Example 24 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class FileSinkCompactionSwitchITCase method testSwitchingCompaction.

@Test
public void testSwitchingCompaction() throws Exception {
    String path = TEMPORARY_FOLDER.newFolder().getAbsolutePath();
    String cpPath = "file://" + TEMPORARY_FOLDER.newFolder().getAbsolutePath();
    SharedReference<ConcurrentHashMap<Integer, Integer>> sendCountMap = sharedObjects.add(new ConcurrentHashMap<>());
    JobGraph jobGraph = createJobGraph(path, cpPath, isOnToOff, false, sendCountMap);
    JobGraph restoringJobGraph = createJobGraph(path, cpPath, !isOnToOff, true, sendCountMap);
    final Configuration config = new Configuration();
    config.setString(RestOptions.BIND_PORT, "18081-19000");
    final MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder().setNumTaskManagers(1).setNumSlotsPerTaskManager(4).setConfiguration(config).build();
    try (MiniCluster miniCluster = new MiniCluster(cfg)) {
        miniCluster.start();
        miniCluster.submitJob(jobGraph);
        LATCH_MAP.get(latchId).await();
        String savepointPath = miniCluster.triggerSavepoint(jobGraph.getJobID(), TEMPORARY_FOLDER.newFolder().getAbsolutePath(), true, SavepointFormatType.CANONICAL).get();
        // We wait for two successful checkpoints in sources before shutting down. This ensures
        // that the sink can commit its data.
        LATCH_MAP.put(latchId, new CountDownLatch(NUM_SOURCES * 2));
        restoringJobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath, false));
        miniCluster.executeJobBlocking(restoringJobGraph);
    }
    checkIntegerSequenceSinkOutput(path, sendCountMap.get(), NUM_BUCKETS, NUM_SOURCES);
}

Example 25 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class FileSinkMigrationITCase method executeAndTakeSavepoint.

private String executeAndTakeSavepoint(MiniClusterConfiguration cfg, JobGraph jobGraph, String savepointBasePath) throws Exception {
    try (MiniCluster miniCluster = new MiniCluster(cfg)) {
        miniCluster.start();
        CompletableFuture<JobSubmissionResult> jobSubmissionResultFuture = miniCluster.submitJob(jobGraph);
        JobID jobId = jobSubmissionResultFuture.get().getJobID();
        waitForAllTaskRunning(miniCluster, jobId, false);
        CompletableFuture<String> savepointResultFuture = miniCluster.triggerSavepoint(jobId, savepointBasePath, true, SavepointFormatType.CANONICAL);
        return savepointResultFuture.get();
    }
}