Examples with TaskManagerMetricGroup org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup used on opensource projects

Example 11 with TaskManagerMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup in project flink by apache.

the class JMXReporterTest method testHistogramReporting.

/**
	 * Tests that histograms are properly reported via the JMXReporter.
	 */
@Test
public void testHistogramReporting() throws Exception {
    MetricRegistry registry = null;
    String histogramName = "histogram";
    try {
        Configuration config = new Configuration();
        config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "jmx_test");
        config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "jmx_test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
        registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
        TaskManagerMetricGroup metricGroup = new TaskManagerMetricGroup(registry, "localhost", "tmId");
        TestingHistogram histogram = new TestingHistogram();
        metricGroup.histogram(histogramName, histogram);
        MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer();
        ObjectName objectName = new ObjectName(JMX_DOMAIN_PREFIX + "taskmanager." + histogramName, JMXReporter.generateJmxTable(metricGroup.getAllVariables()));
        MBeanInfo info = mBeanServer.getMBeanInfo(objectName);
        MBeanAttributeInfo[] attributeInfos = info.getAttributes();
        assertEquals(11, attributeInfos.length);
        assertEquals(histogram.getCount(), mBeanServer.getAttribute(objectName, "Count"));
        assertEquals(histogram.getStatistics().getMean(), mBeanServer.getAttribute(objectName, "Mean"));
        assertEquals(histogram.getStatistics().getStdDev(), mBeanServer.getAttribute(objectName, "StdDev"));
        assertEquals(histogram.getStatistics().getMax(), mBeanServer.getAttribute(objectName, "Max"));
        assertEquals(histogram.getStatistics().getMin(), mBeanServer.getAttribute(objectName, "Min"));
        assertEquals(histogram.getStatistics().getQuantile(0.5), mBeanServer.getAttribute(objectName, "Median"));
        assertEquals(histogram.getStatistics().getQuantile(0.75), mBeanServer.getAttribute(objectName, "75thPercentile"));
        assertEquals(histogram.getStatistics().getQuantile(0.95), mBeanServer.getAttribute(objectName, "95thPercentile"));
        assertEquals(histogram.getStatistics().getQuantile(0.98), mBeanServer.getAttribute(objectName, "98thPercentile"));
        assertEquals(histogram.getStatistics().getQuantile(0.99), mBeanServer.getAttribute(objectName, "99thPercentile"));
        assertEquals(histogram.getStatistics().getQuantile(0.999), mBeanServer.getAttribute(objectName, "999thPercentile"));
    } finally {
        if (registry != null) {
            registry.shutdown();
        }
    }
}

Example 12 with TaskManagerMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup in project flink by apache.

the class MetricRegistryTest method testConfigurableDelimiterForReportersInGroup.

@Test
public void testConfigurableDelimiterForReportersInGroup() {
    Configuration config = new Configuration();
    config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test1,test2,test3,test4");
    config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "_");
    config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter8.class.getName());
    config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "-");
    config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter8.class.getName());
    config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test3." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "AA");
    config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test3." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter8.class.getName());
    config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test4." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter8.class.getName());
    config.setString(ConfigConstants.METRICS_SCOPE_NAMING_TM, "A.B");
    MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
    List<MetricReporter> reporters = registry.getReporters();
    //test1  reporter
    ((TestReporter8) reporters.get(0)).expectedDelimiter = '_';
    //test2 reporter
    ((TestReporter8) reporters.get(1)).expectedDelimiter = '-';
    //test3 reporter, because 'AA' - not correct delimiter
    ((TestReporter8) reporters.get(2)).expectedDelimiter = GLOBAL_DEFAULT_DELIMITER;
    //for test4 reporter use global delimiter
    ((TestReporter8) reporters.get(3)).expectedDelimiter = GLOBAL_DEFAULT_DELIMITER;
    TaskManagerMetricGroup group = new TaskManagerMetricGroup(registry, "host", "id");
    group.counter("C");
    group.close();
    registry.shutdown();
    assertEquals(4, TestReporter8.numCorrectDelimitersForRegister);
    assertEquals(4, TestReporter8.numCorrectDelimitersForUnregister);
}

Example 13 with TaskManagerMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup in project flink by apache.

the class MetricRegistryTest method testConfigurableDelimiter.

@Test
public void testConfigurableDelimiter() {
    Configuration config = new Configuration();
    config.setString(ConfigConstants.METRICS_SCOPE_DELIMITER, "_");
    config.setString(ConfigConstants.METRICS_SCOPE_NAMING_TM, "A.B.C.D.E");
    MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
    TaskManagerMetricGroup tmGroup = new TaskManagerMetricGroup(registry, "host", "id");
    assertEquals("A_B_C_D_E_name", tmGroup.getMetricIdentifier("name"));
    registry.shutdown();
}

Example 14 with TaskManagerMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup in project flink by apache.

the class MetricQueryServiceTest method testCreateDump.

@Test
public void testCreateDump() throws Exception {
    ActorSystem s = AkkaUtils.createLocalActorSystem(new Configuration());
    ActorRef serviceActor = MetricQueryService.startMetricQueryService(s, null);
    TestActorRef testActorRef = TestActorRef.create(s, Props.create(TestActor.class));
    TestActor testActor = (TestActor) testActorRef.underlyingActor();
    final Counter c = new SimpleCounter();
    final Gauge<String> g = new Gauge<String>() {

        @Override
        public String getValue() {
            return "Hello";
        }
    };
    final Histogram h = new TestingHistogram();
    final Meter m = new Meter() {

        @Override
        public void markEvent() {
        }

        @Override
        public void markEvent(long n) {
        }

        @Override
        public double getRate() {
            return 5;
        }

        @Override
        public long getCount() {
            return 10;
        }
    };
    MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.defaultMetricRegistryConfiguration());
    final TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
    MetricQueryService.notifyOfAddedMetric(serviceActor, c, "counter", tm);
    MetricQueryService.notifyOfAddedMetric(serviceActor, g, "gauge", tm);
    MetricQueryService.notifyOfAddedMetric(serviceActor, h, "histogram", tm);
    MetricQueryService.notifyOfAddedMetric(serviceActor, m, "meter", tm);
    serviceActor.tell(MetricQueryService.getCreateDump(), testActorRef);
    synchronized (testActor.lock) {
        if (testActor.message == null) {
            testActor.lock.wait();
        }
    }
    MetricDumpSerialization.MetricSerializationResult dump = (MetricDumpSerialization.MetricSerializationResult) testActor.message;
    testActor.message = null;
    assertTrue(dump.serializedMetrics.length > 0);
    MetricQueryService.notifyOfRemovedMetric(serviceActor, c);
    MetricQueryService.notifyOfRemovedMetric(serviceActor, g);
    MetricQueryService.notifyOfRemovedMetric(serviceActor, h);
    MetricQueryService.notifyOfRemovedMetric(serviceActor, m);
    serviceActor.tell(MetricQueryService.getCreateDump(), testActorRef);
    synchronized (testActor.lock) {
        if (testActor.message == null) {
            testActor.lock.wait();
        }
    }
    MetricDumpSerialization.MetricSerializationResult emptyDump = (MetricDumpSerialization.MetricSerializationResult) testActor.message;
    testActor.message = null;
    assertEquals(0, emptyDump.serializedMetrics.length);
    s.shutdown();
}

Example 15 with TaskManagerMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup in project flink by apache.

the class TaskExecutorITCase method testSlotAllocation.

@Test
public void testSlotAllocation() throws Exception {
    TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
    TestingHighAvailabilityServices testingHAServices = new TestingHighAvailabilityServices();
    final Configuration configuration = new Configuration();
    final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1);
    final ResourceID taskManagerResourceId = new ResourceID("foobar");
    final UUID rmLeaderId = UUID.randomUUID();
    final TestingLeaderElectionService rmLeaderElectionService = new TestingLeaderElectionService();
    final TestingLeaderRetrievalService rmLeaderRetrievalService = new TestingLeaderRetrievalService();
    final String rmAddress = "rm";
    final String jmAddress = "jm";
    final UUID jmLeaderId = UUID.randomUUID();
    final JobID jobId = new JobID();
    final ResourceProfile resourceProfile = new ResourceProfile(1.0, 1);
    testingHAServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
    testingHAServices.setResourceManagerLeaderRetriever(rmLeaderRetrievalService);
    testingHAServices.setJobMasterLeaderRetriever(jobId, new TestingLeaderRetrievalService(jmAddress, jmLeaderId));
    TestingSerialRpcService rpcService = new TestingSerialRpcService();
    ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.milliseconds(500L), Time.milliseconds(500L), Time.minutes(5L));
    SlotManagerFactory slotManagerFactory = new DefaultSlotManager.Factory();
    JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(testingHAServices, rpcService.getScheduledExecutor(), resourceManagerConfiguration.getJobTimeout());
    MetricRegistry metricRegistry = mock(MetricRegistry.class);
    HeartbeatServices heartbeatServices = mock(HeartbeatServices.class, RETURNS_MOCKS);
    final TaskManagerConfiguration taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(configuration);
    final TaskManagerLocation taskManagerLocation = new TaskManagerLocation(taskManagerResourceId, InetAddress.getLocalHost(), 1234);
    final MemoryManager memoryManager = mock(MemoryManager.class);
    final IOManager ioManager = mock(IOManager.class);
    final NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class);
    final TaskManagerMetricGroup taskManagerMetricGroup = mock(TaskManagerMetricGroup.class);
    final BroadcastVariableManager broadcastVariableManager = mock(BroadcastVariableManager.class);
    final FileCache fileCache = mock(FileCache.class);
    final TaskSlotTable taskSlotTable = new TaskSlotTable(Arrays.asList(resourceProfile), new TimerService<AllocationID>(scheduledExecutorService, 100L));
    final JobManagerTable jobManagerTable = new JobManagerTable();
    final JobLeaderService jobLeaderService = new JobLeaderService(taskManagerLocation);
    ResourceManager<ResourceID> resourceManager = new StandaloneResourceManager(rpcService, resourceManagerConfiguration, testingHAServices, slotManagerFactory, metricRegistry, jobLeaderIdService, testingFatalErrorHandler);
    TaskExecutor taskExecutor = new TaskExecutor(taskManagerConfiguration, taskManagerLocation, rpcService, memoryManager, ioManager, networkEnvironment, testingHAServices, heartbeatServices, metricRegistry, taskManagerMetricGroup, broadcastVariableManager, fileCache, taskSlotTable, jobManagerTable, jobLeaderService, testingFatalErrorHandler);
    JobMasterGateway jmGateway = mock(JobMasterGateway.class);
    when(jmGateway.registerTaskManager(any(String.class), any(TaskManagerLocation.class), eq(jmLeaderId), any(Time.class))).thenReturn(FlinkCompletableFuture.<RegistrationResponse>completed(new JMTMRegistrationSuccess(taskManagerResourceId, 1234)));
    when(jmGateway.getHostname()).thenReturn(jmAddress);
    rpcService.registerGateway(rmAddress, resourceManager.getSelf());
    rpcService.registerGateway(jmAddress, jmGateway);
    final AllocationID allocationId = new AllocationID();
    final SlotRequest slotRequest = new SlotRequest(jobId, allocationId, resourceProfile);
    final SlotOffer slotOffer = new SlotOffer(allocationId, 0, resourceProfile);
    try {
        resourceManager.start();
        taskExecutor.start();
        // notify the RM that it is the leader
        rmLeaderElectionService.isLeader(rmLeaderId);
        // notify the TM about the new RM leader
        rmLeaderRetrievalService.notifyListener(rmAddress, rmLeaderId);
        Future<RegistrationResponse> registrationResponseFuture = resourceManager.registerJobManager(rmLeaderId, jmLeaderId, jmAddress, jobId);
        RegistrationResponse registrationResponse = registrationResponseFuture.get();
        assertTrue(registrationResponse instanceof JobMasterRegistrationSuccess);
        resourceManager.requestSlot(jmLeaderId, rmLeaderId, slotRequest);
        verify(jmGateway).offerSlots(eq(taskManagerResourceId), (Iterable<SlotOffer>) argThat(Matchers.contains(slotOffer)), eq(jmLeaderId), any(Time.class));
    } finally {
        if (testingFatalErrorHandler.hasExceptionOccurred()) {
            testingFatalErrorHandler.rethrowError();
        }
    }
}