Search in sources :

Example 6 with AggregatedMetric

use of org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric in project flink by apache.

the class AggregatingMetricsHandlerTestBase method testSumAggregation.

@Test
public void testSumAggregation() throws Exception {
    Map<String, List<String>> queryParams = new HashMap<>(4);
    queryParams.put("get", Collections.singletonList("abc.metric1"));
    queryParams.put("agg", Collections.singletonList("sum"));
    HandlerRequest<EmptyRequestBody> request = HandlerRequest.resolveParametersAndCreate(EmptyRequestBody.getInstance(), handler.getMessageHeaders().getUnresolvedMessageParameters(), pathParameters, queryParams, Collections.emptyList());
    AggregatedMetricsResponseBody response = handler.handleRequest(request, MOCK_DISPATCHER_GATEWAY).get();
    Collection<AggregatedMetric> aggregatedMetrics = response.getMetrics();
    assertEquals(1, aggregatedMetrics.size());
    AggregatedMetric aggregatedMetric = aggregatedMetrics.iterator().next();
    assertEquals("abc.metric1", aggregatedMetric.getId());
    assertEquals(4.0, aggregatedMetric.getSum(), 0.1);
    assertNull(aggregatedMetric.getMin());
    assertNull(aggregatedMetric.getMax());
    assertNull(aggregatedMetric.getAvg());
}
Also used : AggregatedMetricsResponseBody(org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetricsResponseBody) AggregatedMetric(org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric) HashMap(java.util.HashMap) List(java.util.List) EmptyRequestBody(org.apache.flink.runtime.rest.messages.EmptyRequestBody) Test(org.junit.Test)

Example 7 with AggregatedMetric

use of org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric in project flink by apache.

the class AggregatingMetricsHandlerTestBase method testMultipleAggregation.

@Test
public void testMultipleAggregation() throws Exception {
    Map<String, List<String>> queryParams = new HashMap<>(4);
    queryParams.put("get", Collections.singletonList("abc.metric1"));
    queryParams.put("agg", Arrays.asList("min", "max", "avg"));
    HandlerRequest<EmptyRequestBody> request = HandlerRequest.resolveParametersAndCreate(EmptyRequestBody.getInstance(), handler.getMessageHeaders().getUnresolvedMessageParameters(), pathParameters, queryParams, Collections.emptyList());
    AggregatedMetricsResponseBody response = handler.handleRequest(request, MOCK_DISPATCHER_GATEWAY).get();
    Collection<AggregatedMetric> aggregatedMetrics = response.getMetrics();
    assertEquals(1, aggregatedMetrics.size());
    AggregatedMetric aggregatedMetric = aggregatedMetrics.iterator().next();
    assertEquals("abc.metric1", aggregatedMetric.getId());
    assertEquals(1.0, aggregatedMetric.getMin(), 0.1);
    assertEquals(3.0, aggregatedMetric.getMax(), 0.1);
    assertEquals(2.0, aggregatedMetric.getAvg(), 0.1);
    assertNull(aggregatedMetric.getSum());
}
Also used : AggregatedMetricsResponseBody(org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetricsResponseBody) AggregatedMetric(org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric) HashMap(java.util.HashMap) List(java.util.List) EmptyRequestBody(org.apache.flink.runtime.rest.messages.EmptyRequestBody) Test(org.junit.Test)

Example 8 with AggregatedMetric

use of org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric in project flink by apache.

the class AbstractAggregatingMetricsHandler method handleRequest.

@Override
protected CompletableFuture<AggregatedMetricsResponseBody> handleRequest(@Nonnull HandlerRequest<EmptyRequestBody> request, @Nonnull RestfulGateway gateway) throws RestHandlerException {
    return CompletableFuture.supplyAsync(() -> {
        try {
            fetcher.update();
            List<String> requestedMetrics = request.getQueryParameter(MetricsFilterParameter.class);
            List<MetricsAggregationParameter.AggregationMode> requestedAggregations = request.getQueryParameter(MetricsAggregationParameter.class);
            MetricStore store = fetcher.getMetricStore();
            Collection<? extends MetricStore.ComponentMetricStore> stores = getStores(store, request);
            if (requestedMetrics.isEmpty()) {
                Collection<String> list = getAvailableMetrics(stores);
                return new AggregatedMetricsResponseBody(list.stream().map(AggregatedMetric::new).collect(Collectors.toList()));
            }
            DoubleAccumulator.DoubleMinimumFactory minimumFactory = null;
            DoubleAccumulator.DoubleMaximumFactory maximumFactory = null;
            DoubleAccumulator.DoubleAverageFactory averageFactory = null;
            DoubleAccumulator.DoubleSumFactory sumFactory = null;
            // by default we return all aggregations
            if (requestedAggregations.isEmpty()) {
                minimumFactory = DoubleAccumulator.DoubleMinimumFactory.get();
                maximumFactory = DoubleAccumulator.DoubleMaximumFactory.get();
                averageFactory = DoubleAccumulator.DoubleAverageFactory.get();
                sumFactory = DoubleAccumulator.DoubleSumFactory.get();
            } else {
                for (MetricsAggregationParameter.AggregationMode aggregation : requestedAggregations) {
                    switch(aggregation) {
                        case MIN:
                            minimumFactory = DoubleAccumulator.DoubleMinimumFactory.get();
                            break;
                        case MAX:
                            maximumFactory = DoubleAccumulator.DoubleMaximumFactory.get();
                            break;
                        case AVG:
                            averageFactory = DoubleAccumulator.DoubleAverageFactory.get();
                            break;
                        case SUM:
                            sumFactory = DoubleAccumulator.DoubleSumFactory.get();
                            break;
                        default:
                            log.warn("Unsupported aggregation specified: {}", aggregation);
                    }
                }
            }
            MetricAccumulatorFactory metricAccumulatorFactory = new MetricAccumulatorFactory(minimumFactory, maximumFactory, averageFactory, sumFactory);
            return getAggregatedMetricValues(stores, requestedMetrics, metricAccumulatorFactory);
        } catch (Exception e) {
            log.warn("Could not retrieve metrics.", e);
            throw new CompletionException(new RestHandlerException("Could not retrieve metrics.", HttpResponseStatus.INTERNAL_SERVER_ERROR));
        }
    }, executor);
}
Also used : MetricStore(org.apache.flink.runtime.rest.handler.legacy.metrics.MetricStore) CompletionException(java.util.concurrent.CompletionException) RestHandlerException(org.apache.flink.runtime.rest.handler.RestHandlerException) RestHandlerException(org.apache.flink.runtime.rest.handler.RestHandlerException) AggregatedMetricsResponseBody(org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetricsResponseBody) AggregatedMetric(org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric) CompletionException(java.util.concurrent.CompletionException) MetricsAggregationParameter(org.apache.flink.runtime.rest.messages.job.metrics.MetricsAggregationParameter)

Example 9 with AggregatedMetric

use of org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric in project flink by apache.

the class MetricQuerier method getAggregatedMetricsByRestAPI.

public Double getAggregatedMetricsByRestAPI(TestEnvironment.Endpoint endpoint, JobID jobId, String sourceOrSinkName, String metricName, String filter) throws Exception {
    // get job details, including the vertex id
    JobDetailsInfo jobDetailsInfo = getJobDetails(restClient, endpoint, jobId);
    // get the vertex id for source/sink operator
    JobDetailsInfo.JobVertexDetailsInfo vertex = jobDetailsInfo.getJobVertexInfos().stream().filter(v -> v.getName().contains(sourceOrSinkName)).findAny().orElse(null);
    assertThat(vertex).isNotNull();
    JobVertexID vertexId = vertex.getJobVertexID();
    // get the metric list
    AggregatedMetricsResponseBody metricsResponseBody = getMetricList(endpoint, jobId, vertexId);
    // get the metric query filters
    String queryParam = metricsResponseBody.getMetrics().stream().filter(m -> filterByMetricName(m.getId(), sourceOrSinkName, metricName, filter)).map(m -> m.getId()).collect(Collectors.joining(","));
    if (StringUtils.isNullOrWhitespaceOnly(queryParam)) {
        throw new IllegalStateException(String.format("Cannot find metric[%s] for operator [%s].", metricName, sourceOrSinkName));
    }
    AggregatedMetricsResponseBody metricsResponse = getMetrics(endpoint, jobId, vertexId, queryParam);
    Collection<AggregatedMetric> metrics = metricsResponse.getMetrics();
    if (metrics == null || metrics.isEmpty()) {
        throw new IllegalStateException(String.format("Cannot find metric[%s] for operator [%s] with filter [%s].", metricName, sourceOrSinkName, filter));
    }
    return metrics.iterator().next().getSum();
}
Also used : AggregatedMetricsResponseBody(org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetricsResponseBody) MessagePathParameter(org.apache.flink.runtime.rest.messages.MessagePathParameter) ConfigurationException(org.apache.flink.util.ConfigurationException) LoggerFactory(org.slf4j.LoggerFactory) JobVertexIdPathParameter(org.apache.flink.runtime.rest.messages.JobVertexIdPathParameter) EmptyRequestBody(org.apache.flink.runtime.rest.messages.EmptyRequestBody) JobVertexID(org.apache.flink.runtime.jobgraph.JobVertexID) AggregatedMetricsResponseBody(org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetricsResponseBody) JobDetailsHeaders(org.apache.flink.runtime.rest.messages.job.JobDetailsHeaders) AssertionsForClassTypes.assertThat(org.assertj.core.api.AssertionsForClassTypes.assertThat) JobIDPathParameter(org.apache.flink.runtime.rest.messages.JobIDPathParameter) Nullable(javax.annotation.Nullable) TestEnvironment(org.apache.flink.connector.testframe.environment.TestEnvironment) AggregatedMetric(org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) MetricsFilterParameter(org.apache.flink.runtime.rest.messages.job.metrics.MetricsFilterParameter) JobDetailsInfo(org.apache.flink.runtime.rest.messages.job.JobDetailsInfo) Collection(java.util.Collection) Configuration(org.apache.flink.configuration.Configuration) StringUtils(org.apache.flink.util.StringUtils) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) TimeUnit(java.util.concurrent.TimeUnit) AggregatedSubtaskMetricsParameters(org.apache.flink.runtime.rest.messages.job.metrics.AggregatedSubtaskMetricsParameters) JobID(org.apache.flink.api.common.JobID) JobMessageParameters(org.apache.flink.runtime.rest.messages.JobMessageParameters) AggregatedSubtaskMetricsHeaders(org.apache.flink.runtime.rest.messages.job.metrics.AggregatedSubtaskMetricsHeaders) RestClient(org.apache.flink.runtime.rest.RestClient) AggregatedMetric(org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric) JobVertexID(org.apache.flink.runtime.jobgraph.JobVertexID) JobDetailsInfo(org.apache.flink.runtime.rest.messages.job.JobDetailsInfo)

Aggregations

AggregatedMetric (org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric)9 AggregatedMetricsResponseBody (org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetricsResponseBody)9 EmptyRequestBody (org.apache.flink.runtime.rest.messages.EmptyRequestBody)8 Test (org.junit.Test)7 HashMap (java.util.HashMap)6 List (java.util.List)6 Collection (java.util.Collection)1 Iterator (java.util.Iterator)1 CompletionException (java.util.concurrent.CompletionException)1 Executors (java.util.concurrent.Executors)1 TimeUnit (java.util.concurrent.TimeUnit)1 Collectors (java.util.stream.Collectors)1 Nullable (javax.annotation.Nullable)1 JobID (org.apache.flink.api.common.JobID)1 Configuration (org.apache.flink.configuration.Configuration)1 TestEnvironment (org.apache.flink.connector.testframe.environment.TestEnvironment)1 JobVertexID (org.apache.flink.runtime.jobgraph.JobVertexID)1 RestClient (org.apache.flink.runtime.rest.RestClient)1 RestHandlerException (org.apache.flink.runtime.rest.handler.RestHandlerException)1 MetricStore (org.apache.flink.runtime.rest.handler.legacy.metrics.MetricStore)1