Examples with RestfulGateway org.apache.flink.runtime.webmonitor.RestfulGateway used on opensource projects

Example 6 with RestfulGateway

use of org.apache.flink.runtime.webmonitor.RestfulGateway in project flink by apache.

the class MetricFetcherTest method testUpdate.

@Test
public void testUpdate() {
    final Time timeout = Time.seconds(10L);
    // ========= setup TaskManager
    // =================================================================================
    JobID jobID = new JobID();
    ResourceID tmRID = ResourceID.generate();
    // ========= setup QueryServices
    // ================================================================================
    final MetricQueryServiceGateway jmQueryService = new TestingMetricQueryServiceGateway.Builder().setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(new MetricDumpSerialization.MetricSerializationResult(new byte[0], new byte[0], new byte[0], new byte[0], 0, 0, 0, 0))).build();
    MetricDumpSerialization.MetricSerializationResult requestMetricsAnswer = createRequestDumpAnswer(tmRID, jobID);
    final MetricQueryServiceGateway tmQueryService = new TestingMetricQueryServiceGateway.Builder().setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(requestMetricsAnswer)).build();
    // ========= setup JobManager
    // ==================================================================================
    final TestingRestfulGateway restfulGateway = new TestingRestfulGateway.Builder().setRequestMultipleJobDetailsSupplier(() -> CompletableFuture.completedFuture(new MultipleJobsDetails(Collections.emptyList()))).setRequestMetricQueryServiceGatewaysSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(jmQueryService.getAddress()))).setRequestTaskManagerMetricQueryServiceGatewaysSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(Tuple2.of(tmRID, tmQueryService.getAddress())))).build();
    final GatewayRetriever<RestfulGateway> retriever = () -> CompletableFuture.completedFuture(restfulGateway);
    // ========= start MetricFetcher testing
    // =======================================================================
    MetricFetcher fetcher = new MetricFetcherImpl<>(retriever, address -> CompletableFuture.completedFuture(tmQueryService), Executors.directExecutor(), timeout, MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());
    // verify that update fetches metrics and updates the store
    fetcher.update();
    MetricStore store = fetcher.getMetricStore();
    synchronized (store) {
        assertEquals("7", store.getJobManagerMetricStore().getMetric("abc.hist_min"));
        assertEquals("6", store.getJobManagerMetricStore().getMetric("abc.hist_max"));
        assertEquals("4.0", store.getJobManagerMetricStore().getMetric("abc.hist_mean"));
        assertEquals("0.5", store.getJobManagerMetricStore().getMetric("abc.hist_median"));
        assertEquals("5.0", store.getJobManagerMetricStore().getMetric("abc.hist_stddev"));
        assertEquals("0.75", store.getJobManagerMetricStore().getMetric("abc.hist_p75"));
        assertEquals("0.9", store.getJobManagerMetricStore().getMetric("abc.hist_p90"));
        assertEquals("0.95", store.getJobManagerMetricStore().getMetric("abc.hist_p95"));
        assertEquals("0.98", store.getJobManagerMetricStore().getMetric("abc.hist_p98"));
        assertEquals("0.99", store.getJobManagerMetricStore().getMetric("abc.hist_p99"));
        assertEquals("0.999", store.getJobManagerMetricStore().getMetric("abc.hist_p999"));
        assertEquals("x", store.getTaskManagerMetricStore(tmRID.toString()).metrics.get("abc.gauge"));
        assertEquals("5.0", store.getJobMetricStore(jobID.toString()).metrics.get("abc.jc"));
        assertEquals("2", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.abc.tc"));
        assertEquals("1", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.opname.abc.oc"));
    }
}

Example 7 with RestfulGateway

use of org.apache.flink.runtime.webmonitor.RestfulGateway in project flink by apache.

the class MetricFetcherTest method testShortUpdateInterval.

@Test
public void testShortUpdateInterval() throws InterruptedException {
    final long updateInterval = 1L;
    final AtomicInteger requestMetricQueryServiceGatewaysCounter = new AtomicInteger(0);
    final RestfulGateway restfulGateway = createRestfulGateway(requestMetricQueryServiceGatewaysCounter);
    final MetricFetcher fetcher = createMetricFetcher(updateInterval, restfulGateway);
    fetcher.update();
    final long start = System.currentTimeMillis();
    long difference = 0L;
    while (difference <= updateInterval) {
        Thread.sleep(2L * updateInterval);
        difference = System.currentTimeMillis() - start;
    }
    fetcher.update();
    assertThat(requestMetricQueryServiceGatewaysCounter.get(), is(2));
}

Example 8 with RestfulGateway

use of org.apache.flink.runtime.webmonitor.RestfulGateway in project flink by apache.

the class DefaultExecutionGraphCacheTest method testCacheEntryCleanup.

/**
 * Tests that cache entries are cleaned up when their TTL has expired upon calling {@link
 * DefaultExecutionGraphCache#cleanup()}.
 */
@Test
public void testCacheEntryCleanup() throws Exception {
    final Time timeout = Time.milliseconds(100L);
    final Time timeToLive = Time.milliseconds(1L);
    final JobID expectedJobId2 = new JobID();
    final ExecutionGraphInfo expectedExecutionGraphInfo2 = new ExecutionGraphInfo(new ArchivedExecutionGraphBuilder().build());
    final AtomicInteger requestJobCalls = new AtomicInteger(0);
    final TestingRestfulGateway restfulGateway = new TestingRestfulGateway.Builder().setRequestExecutionGraphInfoFunction(jobId -> {
        requestJobCalls.incrementAndGet();
        if (jobId.equals(expectedJobId)) {
            return CompletableFuture.completedFuture(expectedExecutionGraphInfo);
        } else if (jobId.equals(expectedJobId2)) {
            return CompletableFuture.completedFuture(expectedExecutionGraphInfo2);
        } else {
            throw new AssertionError("Invalid job id received.");
        }
    }).build();
    try (ExecutionGraphCache executionGraphCache = new DefaultExecutionGraphCache(timeout, timeToLive)) {
        CompletableFuture<ExecutionGraphInfo> executionGraph1Future = executionGraphCache.getExecutionGraphInfo(expectedJobId, restfulGateway);
        CompletableFuture<ExecutionGraphInfo> executionGraph2Future = executionGraphCache.getExecutionGraphInfo(expectedJobId2, restfulGateway);
        assertEquals(expectedExecutionGraphInfo, executionGraph1Future.get());
        assertEquals(expectedExecutionGraphInfo2, executionGraph2Future.get());
        assertThat(requestJobCalls.get(), Matchers.equalTo(2));
        Thread.sleep(timeToLive.toMilliseconds());
        executionGraphCache.cleanup();
        assertTrue(executionGraphCache.size() == 0);
    }
}

Example 9 with RestfulGateway

use of org.apache.flink.runtime.webmonitor.RestfulGateway in project flink by apache.

the class AbstractHandlerITCase method testOOMErrorMessageEnrichment.

@Test
public void testOOMErrorMessageEnrichment() throws Exception {
    final TestMessageHeaders<EmptyRequestBody, EmptyResponseBody, EmptyMessageParameters> messageHeaders = TestMessageHeaders.emptyBuilder().setTargetRestEndpointURL("/test-handler").build();
    final TestRestHandler<RestfulGateway, EmptyRequestBody, EmptyResponseBody, EmptyMessageParameters> testRestHandler = new TestRestHandler<>(mockGatewayRetriever, messageHeaders, FutureUtils.completedExceptionally(new OutOfMemoryError("Metaspace")));
    try (final TestRestServerEndpoint server = TestRestServerEndpoint.builder(REST_BASE_CONFIG).withHandler(messageHeaders, testRestHandler).buildAndStart();
        final RestClient restClient = createRestClient(server.getServerAddress().getPort())) {
        CompletableFuture<EmptyResponseBody> response = restClient.sendRequest(server.getServerAddress().getHostName(), server.getServerAddress().getPort(), messageHeaders, EmptyMessageParameters.getInstance(), EmptyRequestBody.getInstance());
        try {
            response.get();
            fail("An ExecutionException was expected here being caused by the OutOfMemoryError.");
        } catch (ExecutionException e) {
            assertThat(e.getMessage(), StringContains.containsString("Metaspace. The metaspace out-of-memory error has occurred. "));
        }
    }
}

Example 10 with RestfulGateway

use of org.apache.flink.runtime.webmonitor.RestfulGateway in project flink by apache.

the class AbstractExecutionGraphHandler method handleRequest.

@Override
protected CompletableFuture<R> handleRequest(@Nonnull HandlerRequest<EmptyRequestBody> request, @Nonnull RestfulGateway gateway) throws RestHandlerException {
    JobID jobId = request.getPathParameter(JobIDPathParameter.class);
    CompletableFuture<ExecutionGraphInfo> executionGraphFuture = executionGraphCache.getExecutionGraphInfo(jobId, gateway);
    return executionGraphFuture.thenApplyAsync(executionGraph -> {
        try {
            return handleRequest(request, executionGraph);
        } catch (RestHandlerException rhe) {
            throw new CompletionException(rhe);
        }
    }, executor).exceptionally(throwable -> {
        throwable = ExceptionUtils.stripCompletionException(throwable);
        if (throwable instanceof FlinkJobNotFoundException) {
            throw new CompletionException(new NotFoundException(String.format("Job %s not found", jobId), throwable));
        } else {
            throw new CompletionException(throwable);
        }
    });
}