Examples with MiniCluster org.apache.flink.runtime.minicluster.MiniCluster used on opensource projects

Example 16 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class AdaptiveSchedulerSimpleITCase method testSchedulingOfSimpleJob.

@Test
public void testSchedulingOfSimpleJob() throws Exception {
    final MiniCluster miniCluster = MINI_CLUSTER_RESOURCE.getMiniCluster();
    final JobGraph jobGraph = createJobGraph();
    miniCluster.submitJob(jobGraph).join();
    final JobResult jobResult = miniCluster.requestJobResult(jobGraph.getJobID()).join();
    final JobExecutionResult jobExecutionResult = jobResult.toJobExecutionResult(getClass().getClassLoader());
    assertTrue(jobResult.isSuccess());
}

Example 17 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class CommonTestUtils method waitForSubtasksToFinish.

public static void waitForSubtasksToFinish(MiniCluster miniCluster, JobID job, JobVertexID id, boolean allSubtasks) throws Exception {
    Predicate<AccessExecutionVertex> subtaskPredicate = subtask -> {
        ExecutionState state = subtask.getExecutionState();
        if (state == ExecutionState.FINISHED) {
            return true;
        } else if (state.isTerminal()) {
            throw new RuntimeException(format("Sub-Task %s is already in a terminal state %s", subtask, state));
        } else {
            return false;
        }
    };
    waitUntilCondition(() -> {
        AccessExecutionGraph graph = getGraph(miniCluster, job);
        if (graph.getState() != JobStatus.RUNNING) {
            return false;
        }
        Stream<AccessExecutionVertex> vertexStream = Arrays.stream(graph.getAllVertices().values().stream().filter(jv -> jv.getJobVertexId().equals(id)).findAny().orElseThrow(() -> new RuntimeException("Vertex not found " + id)).getTaskVertices());
        return allSubtasks ? vertexStream.allMatch(subtaskPredicate) : vertexStream.anyMatch(subtaskPredicate);
    }, Deadline.fromNow(Duration.of(1, ChronoUnit.MINUTES)));
}

Example 18 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class UnalignedCheckpointFailureHandlingITCase method testCheckpointSuccessAfterFailure.

@Test
public void testCheckpointSuccessAfterFailure() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    TestCheckpointStorage storage = new TestCheckpointStorage(new JobManagerCheckpointStorage(), sharedObjects, temporaryFolder);
    configure(env, storage);
    buildGraph(env);
    JobClient jobClient = env.executeAsync();
    JobID jobID = jobClient.getJobID();
    MiniCluster miniCluster = miniClusterResource.getMiniCluster();
    waitForJobStatus(jobClient, singletonList(RUNNING), fromNow(Duration.ofSeconds(30)));
    waitForAllTaskRunning(miniCluster, jobID, false);
    triggerFailingCheckpoint(jobID, TestException.class, miniCluster);
    miniCluster.triggerCheckpoint(jobID).get();
}

Example 19 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class CheckpointRestoreWithUidHashITCase method testRestoreFromSavepointBySetUidHash.

@Test
public void testRestoreFromSavepointBySetUidHash() throws Exception {
    final int maxNumber = 100;
    try (MiniCluster miniCluster = new MiniCluster(createMiniClusterConfig())) {
        miniCluster.start();
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        JobGraph firstJob = createJobGraph(env, StatefulSourceBehavior.HOLD_AFTER_CHECKPOINT_ON_FIRST_RUN, maxNumber, "test-uid", null, null);
        JobID jobId = miniCluster.submitJob(firstJob).get().getJobID();
        waitForAllTaskRunning(miniCluster, jobId, false);
        // The source would emit some records and start waiting for the checkpoint to happen.
        // With this latch we ensures the savepoint happens in a fixed position and no following
        // records are emitted after savepoint is triggered.
        startWaitingForCheckpointLatch.get().await();
        String savepointPath = miniCluster.triggerSavepoint(jobId, TMP_FOLDER.newFolder().getAbsolutePath(), true, SavepointFormatType.CANONICAL).get();
        // Get the operator id
        List<OperatorIDPair> operatorIds = firstJob.getVerticesSortedTopologicallyFromSources().get(0).getOperatorIDs();
        OperatorIDPair sourceOperatorIds = operatorIds.get(operatorIds.size() - 1);
        JobGraph secondJob = createJobGraph(env, StatefulSourceBehavior.PROCESS_ONLY, maxNumber, null, sourceOperatorIds.getGeneratedOperatorID().toHexString(), savepointPath);
        miniCluster.executeJobBlocking(secondJob);
    }
    assertThat(result.get(), contains(IntStream.range(0, maxNumber).boxed().toArray()));
}

Example 20 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class CheckpointRestoreWithUidHashITCase method testRestoreCheckpointAfterFailoverWithUidHashSet.

@Test
public void testRestoreCheckpointAfterFailoverWithUidHashSet() throws Exception {
    final int maxNumber = 100;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(2, 500));
    env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE);
    JobGraph jobGraph = createJobGraph(env, StatefulSourceBehavior.FAIL_AFTER_CHECKPOINT_ON_FIRST_RUN, maxNumber, null, new OperatorID().toHexString(), null);
    try (MiniCluster miniCluster = new MiniCluster(createMiniClusterConfig())) {
        miniCluster.start();
        miniCluster.executeJobBlocking(jobGraph);
    }
    assertThat(result.get(), contains(IntStream.range(0, maxNumber).boxed().toArray()));
}