Examples with RestClient org.apache.flink.runtime.rest.RestClient used on opensource projects

Example 6 with RestClient

use of org.apache.flink.runtime.rest.RestClient in project flink by apache.

the class YarnConfigurationITCase method testFlinkContainerMemory.

/**
 * Tests that the Flink components are started with the correct memory settings.
 */
@Test(timeout = 60000)
public void testFlinkContainerMemory() throws Exception {
    runTest(() -> {
        final YarnClient yarnClient = getYarnClient();
        final Configuration configuration = new Configuration(flinkConfiguration);
        final int slotsPerTaskManager = 3;
        configuration.set(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);
        final int masterMemory = 768;
        configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(masterMemory));
        final TaskExecutorProcessSpec tmResourceSpec = TaskExecutorProcessUtils.processSpecFromConfig(configuration);
        final int taskManagerMemory = tmResourceSpec.getTotalProcessMemorySize().getMebiBytes();
        final YarnConfiguration yarnConfiguration = getYarnConfiguration();
        final YarnClusterDescriptor clusterDescriptor = YarnTestUtils.createClusterDescriptorWithLogging(CliFrontend.getConfigurationDirectoryFromEnv(), configuration, yarnConfiguration, yarnClient, true);
        clusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath()));
        clusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles()));
        final File streamingWordCountFile = getTestJarPath("WindowJoin.jar");
        final PackagedProgram packagedProgram = PackagedProgram.newBuilder().setJarFile(streamingWordCountFile).build();
        final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(packagedProgram, configuration, 1, false);
        try {
            final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder().setMasterMemoryMB(masterMemory).setTaskManagerMemoryMB(taskManagerMemory).setSlotsPerTaskManager(slotsPerTaskManager).createClusterSpecification();
            final ClusterClient<ApplicationId> clusterClient = clusterDescriptor.deployJobCluster(clusterSpecification, jobGraph, true).getClusterClient();
            final ApplicationId clusterId = clusterClient.getClusterId();
            final RestClient restClient = new RestClient(configuration, TestingUtils.defaultExecutor());
            try {
                final ApplicationReport applicationReport = yarnClient.getApplicationReport(clusterId);
                final ApplicationAttemptId currentApplicationAttemptId = applicationReport.getCurrentApplicationAttemptId();
                // wait until we have second container allocated
                List<ContainerReport> containers = yarnClient.getContainers(currentApplicationAttemptId);
                while (containers.size() < 2) {
                    // this is nasty but Yarn does not offer a better way to wait
                    Thread.sleep(50L);
                    containers = yarnClient.getContainers(currentApplicationAttemptId);
                }
                for (ContainerReport container : containers) {
                    if (container.getContainerId().getId() == 1) {
                        // this should be the application master
                        assertThat(container.getAllocatedResource().getMemory(), is(masterMemory));
                    } else {
                        assertThat(container.getAllocatedResource().getMemory(), is(taskManagerMemory));
                    }
                }
                final URI webURI = new URI(clusterClient.getWebInterfaceURL());
                CompletableFuture<TaskManagersInfo> taskManagersInfoCompletableFuture;
                Collection<TaskManagerInfo> taskManagerInfos;
                while (true) {
                    taskManagersInfoCompletableFuture = restClient.sendRequest(webURI.getHost(), webURI.getPort(), TaskManagersHeaders.getInstance(), EmptyMessageParameters.getInstance(), EmptyRequestBody.getInstance());
                    final TaskManagersInfo taskManagersInfo = taskManagersInfoCompletableFuture.get();
                    taskManagerInfos = taskManagersInfo.getTaskManagerInfos();
                    // wait until the task manager has registered and reported its slots
                    if (hasTaskManagerConnectedAndReportedSlots(taskManagerInfos)) {
                        break;
                    } else {
                        Thread.sleep(100L);
                    }
                }
                // there should be at least one TaskManagerInfo
                final TaskManagerInfo taskManagerInfo = taskManagerInfos.iterator().next();
                assertThat(taskManagerInfo.getNumberSlots(), is(slotsPerTaskManager));
                final long expectedHeapSizeBytes = tmResourceSpec.getJvmHeapMemorySize().getBytes();
                // We compare here physical memory assigned to a container with the heap
                // memory that we should pass to
                // jvm as Xmx parameter. Those value might differ significantly due to
                // system page size or jvm
                // implementation therefore we use 15% threshold here.
                assertThat((double) taskManagerInfo.getHardwareDescription().getSizeOfJvmHeap() / (double) expectedHeapSizeBytes, is(closeTo(1.0, 0.15)));
                final int expectedManagedMemoryMB = tmResourceSpec.getManagedMemorySize().getMebiBytes();
                assertThat((int) (taskManagerInfo.getHardwareDescription().getSizeOfManagedMemory() >> 20), is(expectedManagedMemoryMB));
            } finally {
                restClient.shutdown(TIMEOUT);
                clusterClient.close();
            }
            clusterDescriptor.killCluster(clusterId);
        } finally {
            clusterDescriptor.close();
        }
    });
}

Example 7 with RestClient

use of org.apache.flink.runtime.rest.RestClient in project flink by apache.

the class YARNSessionCapacitySchedulerITCase method setup.

@BeforeClass
public static void setup() throws Exception {
    YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    YARN_CONFIGURATION.set("yarn.scheduler.capacity.root.queues", "default,qa-team");
    YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.default.capacity", 40);
    YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.qa-team.capacity", 60);
    YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-capacityscheduler");
    startYARNWithConfig(YARN_CONFIGURATION);
    restClientExecutor = Executors.newSingleThreadExecutor();
    restClient = new RestClient(new Configuration(), restClientExecutor);
}

Example 8 with RestClient

use of org.apache.flink.runtime.rest.RestClient in project flink by apache.

the class SourceTestSuiteBase method testSourceMetrics.

/**
 * Test connector source metrics.
 *
 * <p>This test will create 4 splits in the external system first, write test data to all splits
 * and consume back via a Flink job with parallelism 4. Then read and compare the metrics.
 *
 * <p>Now test: numRecordsIn
 */
@TestTemplate
@DisplayName("Test source metrics")
public void testSourceMetrics(TestEnvironment testEnv, DataStreamSourceExternalContext<T> externalContext, CheckpointingMode semantic) throws Exception {
    TestingSourceSettings sourceSettings = TestingSourceSettings.builder().setBoundedness(Boundedness.CONTINUOUS_UNBOUNDED).setCheckpointingMode(semantic).build();
    TestEnvironmentSettings envOptions = TestEnvironmentSettings.builder().setConnectorJarPaths(externalContext.getConnectorJarPaths()).build();
    final int splitNumber = 4;
    final List<List<T>> testRecordCollections = new ArrayList<>();
    for (int i = 0; i < splitNumber; i++) {
        testRecordCollections.add(generateAndWriteTestData(i, externalContext, sourceSettings));
    }
    // make sure use different names when executes multi times
    String sourceName = "metricTestSource" + testRecordCollections.hashCode();
    final StreamExecutionEnvironment env = testEnv.createExecutionEnvironment(envOptions);
    final DataStreamSource<T> dataStreamSource = env.fromSource(tryCreateSource(externalContext, sourceSettings), WatermarkStrategy.noWatermarks(), sourceName).setParallelism(splitNumber);
    dataStreamSource.addSink(new DiscardingSink<>());
    final JobClient jobClient = env.executeAsync("Metrics Test");
    final MetricQuerier queryRestClient = new MetricQuerier(new Configuration());
    final ExecutorService executorService = Executors.newCachedThreadPool();
    try {
        waitForAllTaskRunning(() -> getJobDetails(new RestClient(new Configuration(), executorService), testEnv.getRestEndpoint(), jobClient.getJobID()), Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
        waitUntilCondition(() -> {
            // test metrics
            try {
                return checkSourceMetrics(queryRestClient, testEnv, jobClient.getJobID(), sourceName, getTestDataSize(testRecordCollections));
            } catch (Exception e) {
                // skip failed assert try
                return false;
            }
        }, Deadline.fromNow(DEFAULT_COLLECT_DATA_TIMEOUT));
    } finally {
        // Clean up
        executorService.shutdown();
        killJob(jobClient);
    }
}

Example 9 with RestClient

use of org.apache.flink.runtime.rest.RestClient in project flink by apache.

the class SinkTestSuiteBase method testMetrics.

/**
 * Test connector sink metrics.
 *
 * <p>This test will create a sink in the external system, generate test data and write them to
 * the sink via a Flink job. Then read and compare the metrics.
 *
 * <p>Now test: numRecordsOut
 */
@TestTemplate
@DisplayName("Test sink metrics")
public void testMetrics(TestEnvironment testEnv, DataStreamSinkExternalContext<T> externalContext, CheckpointingMode semantic) throws Exception {
    TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
    int parallelism = 1;
    final List<T> testRecords = generateTestData(sinkSettings, externalContext);
    // make sure use different names when executes multi times
    String sinkName = "metricTestSink" + testRecords.hashCode();
    final StreamExecutionEnvironment env = testEnv.createExecutionEnvironment(TestEnvironmentSettings.builder().setConnectorJarPaths(externalContext.getConnectorJarPaths()).build());
    env.enableCheckpointing(50);
    DataStreamSource<T> source = env.fromSource(new FromElementsSource<>(Boundedness.CONTINUOUS_UNBOUNDED, testRecords, testRecords.size()), WatermarkStrategy.noWatermarks(), "metricTestSource").setParallelism(1);
    DataStream<T> dataStream = source.returns(externalContext.getProducedType());
    tryCreateSink(dataStream, externalContext, sinkSettings).name(sinkName).setParallelism(parallelism);
    final JobClient jobClient = env.executeAsync("Metrics Test");
    final MetricQuerier queryRestClient = new MetricQuerier(new Configuration());
    final ExecutorService executorService = Executors.newCachedThreadPool();
    try {
        waitForAllTaskRunning(() -> getJobDetails(new RestClient(new Configuration(), executorService), testEnv.getRestEndpoint(), jobClient.getJobID()), Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
        waitUntilCondition(() -> {
            // test metrics
            try {
                return compareSinkMetrics(queryRestClient, testEnv, externalContext, jobClient.getJobID(), sinkName, testRecords.size());
            } catch (Exception e) {
                // skip failed assert try
                return false;
            }
        }, Deadline.fromNow(DEFAULT_COLLECT_DATA_TIMEOUT));
    } finally {
        // Clean up
        executorService.shutdown();
        killJob(jobClient);
    }
}

Example 10 with RestClient

use of org.apache.flink.runtime.rest.RestClient in project flink by apache.

the class SinkTestSuiteBase method restartFromSavepoint.

private void restartFromSavepoint(TestEnvironment testEnv, DataStreamSinkExternalContext<T> externalContext, CheckpointingMode semantic, final int beforeParallelism, final int afterParallelism) throws Exception {
    // Step 1: Preparation
    TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
    final StreamExecutionEnvironment execEnv = testEnv.createExecutionEnvironment(TestEnvironmentSettings.builder().setConnectorJarPaths(externalContext.getConnectorJarPaths()).build());
    execEnv.setRestartStrategy(RestartStrategies.noRestart());
    // Step 2: Generate test data
    final List<T> testRecords = generateTestData(sinkSettings, externalContext);
    // Step 3: Build and execute Flink job
    int numBeforeSuccess = testRecords.size() / 2;
    DataStreamSource<T> source = execEnv.fromSource(new FromElementsSource<>(Boundedness.CONTINUOUS_UNBOUNDED, testRecords, numBeforeSuccess), WatermarkStrategy.noWatermarks(), "beforeRestartSource").setParallelism(1);
    DataStream<T> dataStream = source.returns(externalContext.getProducedType());
    tryCreateSink(dataStream, externalContext, sinkSettings).name("Sink restart test").setParallelism(beforeParallelism);
    /**
     * The job should stop after consume a specified number of records. In order to know when
     * the specified number of records have been consumed, a collect sink is need to be watched.
     */
    CollectResultIterator<T> iterator = addCollectSink(source);
    final JobClient jobClient = execEnv.executeAsync("Restart Test");
    iterator.setJobClient(jobClient);
    // Step 4: Wait for the expected result and stop Flink job with a savepoint
    final ExecutorService executorService = Executors.newCachedThreadPool();
    String savepointPath;
    try {
        waitForAllTaskRunning(() -> getJobDetails(new RestClient(new Configuration(), executorService), testEnv.getRestEndpoint(), jobClient.getJobID()), Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
        waitExpectedSizeData(iterator, numBeforeSuccess);
        savepointPath = jobClient.stopWithSavepoint(true, testEnv.getCheckpointUri(), SavepointFormatType.CANONICAL).get(30, TimeUnit.SECONDS);
        waitForJobStatus(jobClient, Collections.singletonList(JobStatus.FINISHED), Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
    } catch (Exception e) {
        executorService.shutdown();
        killJob(jobClient);
        throw e;
    }
    List<T> target = testRecords.subList(0, numBeforeSuccess);
    checkResultWithSemantic(externalContext.createSinkDataReader(sinkSettings), target, semantic);
    // Step 4: restart the Flink job with the savepoint
    final StreamExecutionEnvironment restartEnv = testEnv.createExecutionEnvironment(TestEnvironmentSettings.builder().setConnectorJarPaths(externalContext.getConnectorJarPaths()).setSavepointRestorePath(savepointPath).build());
    restartEnv.enableCheckpointing(50);
    DataStreamSource<T> restartSource = restartEnv.fromSource(new FromElementsSource<>(Boundedness.CONTINUOUS_UNBOUNDED, testRecords, testRecords.size()), WatermarkStrategy.noWatermarks(), "restartSource").setParallelism(1);
    DataStream<T> sinkStream = restartSource.returns(externalContext.getProducedType());
    tryCreateSink(sinkStream, externalContext, sinkSettings).setParallelism(afterParallelism);
    addCollectSink(restartSource);
    final JobClient restartJobClient = restartEnv.executeAsync("Restart Test");
    try {
        // Check the result
        checkResultWithSemantic(externalContext.createSinkDataReader(sinkSettings), testRecords, semantic);
    } finally {
        executorService.shutdown();
        killJob(restartJobClient);
        iterator.close();
    }
}