Examples with MiniCluster org.apache.flink.runtime.minicluster.MiniCluster used on opensource projects

Example 11 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class ShuffleMasterTest method testStopTrackingPartition.

@Test
public void testStopTrackingPartition() throws Exception {
    try (MiniCluster cluster = new MiniCluster(createClusterConfiguration(true))) {
        cluster.start();
        cluster.executeJobBlocking(createJobGraph());
    }
    assertTrue(TestShuffleMaster.currentInstance.get().closed.get());
    String[] expectedPartitionEvents = new String[] { PARTITION_REGISTRATION_EVENT, PARTITION_REGISTRATION_EVENT, PARTITION_REGISTRATION_EVENT, PARTITION_REGISTRATION_EVENT, EXTERNAL_PARTITION_RELEASE_EVENT, EXTERNAL_PARTITION_RELEASE_EVENT };
    assertArrayEquals(expectedPartitionEvents, TestShuffleMaster.partitionEvents.toArray());
}

Example 12 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class BlobsCleanupITCase method testBlobServerCleanup.

private void testBlobServerCleanup(final TestCase testCase) throws Exception {
    final MiniCluster miniCluster = miniClusterResource.getMiniCluster();
    final int numTasks = 2;
    final Deadline timeout = Deadline.fromNow(Duration.ofSeconds(30L));
    final JobGraph jobGraph = createJobGraph(testCase, numTasks);
    final JobID jid = jobGraph.getJobID();
    // upload a blob
    final File tempBlob = File.createTempFile("Required", ".jar");
    final int blobPort = miniCluster.getClusterInformation().getBlobServerPort();
    List<PermanentBlobKey> keys = BlobClient.uploadFiles(new InetSocketAddress("localhost", blobPort), configuration, jid, Collections.singletonList(new Path(tempBlob.getAbsolutePath())));
    assertThat(keys, hasSize(1));
    jobGraph.addUserJarBlobKey(keys.get(0));
    if (testCase == TestCase.JOB_SUBMISSION_FAILS) {
        // add an invalid key so that the submission fails
        jobGraph.addUserJarBlobKey(new PermanentBlobKey());
    }
    final CompletableFuture<JobSubmissionResult> submissionFuture = miniCluster.submitJob(jobGraph);
    if (testCase == TestCase.JOB_SUBMISSION_FAILS) {
        try {
            submissionFuture.get();
            fail("Expected job submission failure.");
        } catch (ExecutionException e) {
            assertThat(ExceptionUtils.findThrowable(e, JobSubmissionException.class).isPresent(), is(true));
        }
    } else {
        final JobSubmissionResult jobSubmissionResult = submissionFuture.get();
        assertThat(jobSubmissionResult.getJobID(), is(jid));
        final CompletableFuture<JobResult> resultFuture = miniCluster.requestJobResult(jid);
        if (testCase == TestCase.JOB_FAILS) {
            // fail a task so that the job is going to be recovered (we actually do not
            // need the blocking part of the invokable and can start throwing right away)
            FailingBlockingInvokable.unblock();
            // job will get restarted, BlobCache may re-download the BLOB if already deleted
            // then the tasks will fail again and the restart strategy will finalise the job
            final JobResult jobResult = resultFuture.get();
            assertThat(jobResult.isSuccess(), is(false));
            assertThat(jobResult.getApplicationStatus(), is(ApplicationStatus.FAILED));
        } else if (testCase == TestCase.JOB_IS_CANCELLED) {
            miniCluster.cancelJob(jid);
            final JobResult jobResult = resultFuture.get();
            assertThat(jobResult.isSuccess(), is(false));
            assertThat(jobResult.getApplicationStatus(), is(ApplicationStatus.CANCELED));
        } else {
            final JobResult jobResult = resultFuture.get();
            Throwable cause = jobResult.getSerializedThrowable().map(throwable -> throwable.deserializeError(getClass().getClassLoader())).orElse(null);
            assertThat(ExceptionUtils.stringifyException(cause), jobResult.isSuccess(), is(true));
        }
    }
    // both BlobServer and BlobCache should eventually delete all files
    File[] blobDirs = blobBaseDir.listFiles((dir, name) -> name.startsWith("blobStore-"));
    assertNotNull(blobDirs);
    for (File blobDir : blobDirs) {
        waitForEmptyBlobDir(blobDir, timeout.timeLeft());
    }
}

Example 13 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class AdaptiveSchedulerClusterITCase method testAutomaticScaleDownInCaseOfLostSlots.

@Test
public void testAutomaticScaleDownInCaseOfLostSlots() throws Exception {
    final MiniCluster miniCluster = miniClusterResource.getMiniCluster();
    final JobGraph jobGraph = createBlockingJobGraph(PARALLELISM);
    miniCluster.submitJob(jobGraph).join();
    final CompletableFuture<JobResult> resultFuture = miniCluster.requestJobResult(jobGraph.getJobID());
    waitUntilParallelismForVertexReached(jobGraph.getJobID(), JOB_VERTEX_ID, NUMBER_SLOTS_PER_TASK_MANAGER * NUMBER_TASK_MANAGERS);
    miniCluster.terminateTaskManager(0);
    waitUntilParallelismForVertexReached(jobGraph.getJobID(), JOB_VERTEX_ID, NUMBER_SLOTS_PER_TASK_MANAGER * (NUMBER_TASK_MANAGERS - 1));
    OnceBlockingNoOpInvokable.unblock();
    final JobResult jobResult = resultFuture.join();
    assertTrue(jobResult.isSuccess());
}

Example 14 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class AdaptiveSchedulerClusterITCase method testCheckpointStatsPersistedAcrossRescale.

@Test
public void testCheckpointStatsPersistedAcrossRescale() throws Exception {
    final MiniCluster miniCluster = miniClusterResource.getMiniCluster();
    JobVertex jobVertex = new JobVertex("jobVertex", JOB_VERTEX_ID);
    jobVertex.setInvokableClass(CheckpointingNoOpInvokable.class);
    jobVertex.setParallelism(PARALLELISM);
    final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(jobVertex);
    jobGraph.setSnapshotSettings(new JobCheckpointingSettings(CheckpointCoordinatorConfiguration.builder().setCheckpointInterval(100).setCheckpointTimeout(1000).build(), null));
    miniCluster.submitJob(jobGraph).join();
    // wait until some checkpoints have been completed
    CommonTestUtils.waitUntilCondition(() -> miniCluster.getExecutionGraph(jobGraph.getJobID()).thenApply(eg -> eg.getCheckpointStatsSnapshot().getCounts().getNumberOfCompletedCheckpoints() > 0).get(), Deadline.fromNow(Duration.ofHours(1)));
    miniCluster.terminateTaskManager(0);
    waitUntilParallelismForVertexReached(jobGraph.getJobID(), JOB_VERTEX_ID, NUMBER_SLOTS_PER_TASK_MANAGER * (NUMBER_TASK_MANAGERS - 1));
    // check that the very first checkpoint is still accessible
    final List<AbstractCheckpointStats> checkpointHistory = miniCluster.getExecutionGraph(jobGraph.getJobID()).thenApply(eg -> eg.getCheckpointStatsSnapshot().getHistory().getCheckpoints()).get();
    assertThat(checkpointHistory.get(checkpointHistory.size() - 1).getCheckpointId(), is(1L));
}

Example 15 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class AdaptiveSchedulerSimpleITCase method testGlobalFailoverIfTaskFails.

@Test
public void testGlobalFailoverIfTaskFails() throws Throwable {
    final MiniCluster miniCluster = MINI_CLUSTER_RESOURCE.getMiniCluster();
    final JobGraph jobGraph = createOnceFailingJobGraph();
    miniCluster.submitJob(jobGraph).join();
    final JobResult jobResult = miniCluster.requestJobResult(jobGraph.getJobID()).join();
    if (!jobResult.isSuccess()) {
        throw jobResult.getSerializedThrowable().get().deserializeError(ClassLoader.getSystemClassLoader());
    }
}