Examples with MiniCluster org.apache.flink.runtime.minicluster.MiniCluster used on opensource projects

Example 6 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class FileBufferReaderITCase method testSequentialReading.

@Test
public void testSequentialReading() throws Exception {
    // setup
    final Configuration configuration;
    if (sslEnabled) {
        configuration = SSLUtilsTest.createInternalSslConfigWithKeyAndTrustStores("JDK");
    } else {
        configuration = new Configuration();
    }
    // Increases the handshake timeout to avoid connection reset/close issues
    // if the netty server thread could not response in time, like when it is
    // busy reading the files.
    configuration.setInteger(SecurityOptions.SSL_INTERNAL_HANDSHAKE_TIMEOUT, 100000);
    configuration.setString(RestOptions.BIND_PORT, "0");
    configuration.setString(NettyShuffleEnvironmentOptions.NETWORK_BLOCKING_SHUFFLE_TYPE, "file");
    configuration.set(TaskManagerOptions.TOTAL_FLINK_MEMORY, MemorySize.parse("1g"));
    configuration.set(TaskManagerOptions.MEMORY_SEGMENT_SIZE, MemorySize.parse(bufferSize + "b"));
    final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder().setConfiguration(configuration).setNumTaskManagers(parallelism).setNumSlotsPerTaskManager(1).build();
    try (final MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration)) {
        miniCluster.start();
        final JobGraph jobGraph = createJobGraph();
        // the job needs to complete without throwing an exception
        miniCluster.executeJobBlocking(jobGraph);
    }
}

Example 7 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class JobRecoveryITCase method runTaskFailureRecoveryTest.

private void runTaskFailureRecoveryTest(final JobGraph jobGraph) throws Exception {
    final MiniCluster miniCluster = MINI_CLUSTER_RESOURCE.getMiniCluster();
    miniCluster.submitJob(jobGraph).get();
    final CompletableFuture<JobResult> jobResultFuture = miniCluster.requestJobResult(jobGraph.getJobID());
    assertThat(jobResultFuture.get().isSuccess(), is(true));
}

Example 8 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class TaskExecutorITCase method setup.

@Before
public void setup() throws Exception {
    miniCluster = new MiniCluster(new MiniClusterConfiguration.Builder().setNumTaskManagers(NUM_TMS).setNumSlotsPerTaskManager(SLOTS_PER_TM).build());
    miniCluster.start();
}

Example 9 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class DefaultSchedulerLocalRecoveryITCase method executeSchedulingTest.

private ArchivedExecutionGraph executeSchedulingTest(Configuration configuration, int parallelism) throws Exception {
    configuration.setString(RestOptions.BIND_PORT, "0");
    final long slotIdleTimeout = TIMEOUT;
    configuration.setLong(JobManagerOptions.SLOT_IDLE_TIMEOUT, slotIdleTimeout);
    configuration.set(TaskManagerOptions.TOTAL_FLINK_MEMORY, MemorySize.parse("64mb"));
    configuration.set(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY, MemorySize.parse("16mb"));
    configuration.set(TaskManagerOptions.FRAMEWORK_OFF_HEAP_MEMORY, MemorySize.parse("16mb"));
    final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder().setConfiguration(configuration).setNumTaskManagers(parallelism).setNumSlotsPerTaskManager(1).build();
    try (MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration)) {
        miniCluster.start();
        MiniClusterClient miniClusterClient = new MiniClusterClient(configuration, miniCluster);
        JobGraph jobGraph = createJobGraph(parallelism);
        // wait for the submission to succeed
        JobID jobId = miniClusterClient.submitJob(jobGraph).get(TIMEOUT, TimeUnit.SECONDS);
        // wait until all tasks running before triggering task failures
        waitUntilAllVerticesRunning(jobId, miniCluster);
        // kill one TM to trigger task failure and remove one existing slot
        CompletableFuture<Void> terminationFuture = miniCluster.terminateTaskManager(0);
        terminationFuture.get();
        // restart a taskmanager as a replacement for the killed one
        miniCluster.startTaskManager();
        // wait until all tasks running again
        waitUntilAllVerticesRunning(jobId, miniCluster);
        ArchivedExecutionGraph graph = miniCluster.getArchivedExecutionGraph(jobGraph.getJobID()).get();
        miniCluster.cancelJob(jobId).get();
        return graph;
    }
}

Example 10 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class TaskCancelAsyncProducerConsumerITCase method testCancelAsyncProducerAndConsumer.

/**
 * Tests that a task waiting on an async producer/consumer that is stuck in a blocking buffer
 * request can be properly cancelled.
 *
 * <p>This is currently required for the Flink Kafka sources, which spawn a separate Thread
 * consuming from Kafka and producing the intermediate streams in the spawned Thread instead of
 * the main task Thread.
 */
@Test
public void testCancelAsyncProducerAndConsumer(@InjectMiniCluster MiniCluster flink) throws Exception {
    Deadline deadline = Deadline.now().plus(Duration.ofMinutes(2));
    // Job with async producer and consumer
    JobVertex producer = new JobVertex("AsyncProducer");
    producer.setParallelism(1);
    producer.setInvokableClass(AsyncProducer.class);
    JobVertex consumer = new JobVertex("AsyncConsumer");
    consumer.setParallelism(1);
    consumer.setInvokableClass(AsyncConsumer.class);
    consumer.connectNewDataSetAsInput(producer, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
    SlotSharingGroup slot = new SlotSharingGroup();
    producer.setSlotSharingGroup(slot);
    consumer.setSlotSharingGroup(slot);
    JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(producer, consumer);
    // Submit job and wait until running
    flink.runDetached(jobGraph);
    FutureUtils.retrySuccessfulWithDelay(() -> flink.getJobStatus(jobGraph.getJobID()), Time.milliseconds(10), deadline, status -> status == JobStatus.RUNNING, TestingUtils.defaultScheduledExecutor()).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
    boolean producerBlocked = false;
    for (int i = 0; i < 50; i++) {
        Thread thread = ASYNC_PRODUCER_THREAD;
        if (thread != null && thread.isAlive()) {
            StackTraceElement[] stackTrace = thread.getStackTrace();
            producerBlocked = isInBlockingBufferRequest(stackTrace);
        }
        if (producerBlocked) {
            break;
        } else {
            // Retry
            Thread.sleep(500L);
        }
    }
    // Verify that async producer is in blocking request
    assertTrue("Producer thread is not blocked: " + Arrays.toString(ASYNC_PRODUCER_THREAD.getStackTrace()), producerBlocked);
    boolean consumerWaiting = false;
    for (int i = 0; i < 50; i++) {
        Thread thread = ASYNC_CONSUMER_THREAD;
        if (thread != null && thread.isAlive()) {
            consumerWaiting = thread.getState() == Thread.State.WAITING;
        }
        if (consumerWaiting) {
            break;
        } else {
            // Retry
            Thread.sleep(500L);
        }
    }
    // Verify that async consumer is in blocking request
    assertTrue("Consumer thread is not blocked.", consumerWaiting);
    flink.cancelJob(jobGraph.getJobID()).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
    // wait until the job is canceled
    FutureUtils.retrySuccessfulWithDelay(() -> flink.getJobStatus(jobGraph.getJobID()), Time.milliseconds(10), deadline, status -> status == JobStatus.CANCELED, TestingUtils.defaultScheduledExecutor()).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
    // Verify the expected Exceptions
    assertNotNull(ASYNC_PRODUCER_EXCEPTION);
    assertEquals(CancelTaskException.class, ASYNC_PRODUCER_EXCEPTION.getClass());
    assertNotNull(ASYNC_CONSUMER_EXCEPTION);
    assertEquals(IllegalStateException.class, ASYNC_CONSUMER_EXCEPTION.getClass());
}