Examples with TaskManagerMetricGroup org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup used on opensource projects

Example 1 with TaskManagerMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup in project flink by apache.

the class TaskManagerServices method fromConfiguration.

// --------------------------------------------------------------------------------------------
//  Static factory methods for task manager services
// --------------------------------------------------------------------------------------------
/**
	 * Creates and returns the task manager services.
	 *
	 * @param resourceID resource ID of the task manager
	 * @param taskManagerServicesConfiguration task manager configuration
	 * @return task manager components
	 * @throws Exception
	 */
public static TaskManagerServices fromConfiguration(TaskManagerServicesConfiguration taskManagerServicesConfiguration, ResourceID resourceID) throws Exception {
    // pre-start checks
    checkTempDirs(taskManagerServicesConfiguration.getTmpDirPaths());
    final NetworkEnvironment network = createNetworkEnvironment(taskManagerServicesConfiguration);
    network.start();
    final TaskManagerLocation taskManagerLocation = new TaskManagerLocation(resourceID, taskManagerServicesConfiguration.getTaskManagerAddress(), network.getConnectionManager().getDataPort());
    // this call has to happen strictly after the network stack has been initialized
    final MemoryManager memoryManager = createMemoryManager(taskManagerServicesConfiguration);
    // start the I/O manager, it will create some temp directories.
    final IOManager ioManager = new IOManagerAsync(taskManagerServicesConfiguration.getTmpDirPaths());
    final MetricRegistry metricRegistry = new MetricRegistry(taskManagerServicesConfiguration.getMetricRegistryConfiguration());
    final TaskManagerMetricGroup taskManagerMetricGroup = new TaskManagerMetricGroup(metricRegistry, taskManagerLocation.getHostname(), taskManagerLocation.getResourceID().toString());
    // Initialize the TM metrics
    TaskExecutorMetricsInitializer.instantiateStatusMetrics(taskManagerMetricGroup, network);
    final BroadcastVariableManager broadcastVariableManager = new BroadcastVariableManager();
    final FileCache fileCache = new FileCache(taskManagerServicesConfiguration.getTmpDirPaths());
    final List<ResourceProfile> resourceProfiles = new ArrayList<>(taskManagerServicesConfiguration.getNumberOfSlots());
    for (int i = 0; i < taskManagerServicesConfiguration.getNumberOfSlots(); i++) {
        resourceProfiles.add(new ResourceProfile(1.0, 42));
    }
    final TimerService<AllocationID> timerService = new TimerService<>(new ScheduledThreadPoolExecutor(1), taskManagerServicesConfiguration.getTimerServiceShutdownTimeout());
    final TaskSlotTable taskSlotTable = new TaskSlotTable(resourceProfiles, timerService);
    final JobManagerTable jobManagerTable = new JobManagerTable();
    final JobLeaderService jobLeaderService = new JobLeaderService(taskManagerLocation);
    return new TaskManagerServices(taskManagerLocation, memoryManager, ioManager, network, metricRegistry, taskManagerMetricGroup, broadcastVariableManager, fileCache, taskSlotTable, jobManagerTable, jobLeaderService);
}

Example 2 with TaskManagerMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup in project flink by apache.

the class StatsDReporterTest method testStatsDHistogramReporting.

/**
	 * Tests that histograms are properly reported via the StatsD reporter
	 */
@Test
public void testStatsDHistogramReporting() throws Exception {
    MetricRegistry registry = null;
    DatagramSocketReceiver receiver = null;
    Thread receiverThread = null;
    long timeout = 5000;
    long joinTimeout = 30000;
    String histogramName = "histogram";
    try {
        receiver = new DatagramSocketReceiver();
        receiverThread = new Thread(receiver);
        receiverThread.start();
        int port = receiver.getPort();
        Configuration config = new Configuration();
        config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test");
        config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, StatsDReporter.class.getName());
        config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_INTERVAL_SUFFIX, "1 SECONDS");
        config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test.host", "localhost");
        config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test.port", "" + port);
        registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
        TaskManagerMetricGroup metricGroup = new TaskManagerMetricGroup(registry, "localhost", "tmId");
        TestingHistogram histogram = new TestingHistogram();
        metricGroup.histogram(histogramName, histogram);
        receiver.waitUntilNumLines(11, timeout);
        Set<String> lines = receiver.getLines();
        String prefix = metricGroup.getMetricIdentifier(histogramName);
        Set<String> expectedLines = new HashSet<>();
        expectedLines.add(prefix + ".count:1|g");
        expectedLines.add(prefix + ".mean:3.0|g");
        expectedLines.add(prefix + ".min:6|g");
        expectedLines.add(prefix + ".max:5|g");
        expectedLines.add(prefix + ".stddev:4.0|g");
        expectedLines.add(prefix + ".p75:0.75|g");
        expectedLines.add(prefix + ".p98:0.98|g");
        expectedLines.add(prefix + ".p99:0.99|g");
        expectedLines.add(prefix + ".p999:0.999|g");
        expectedLines.add(prefix + ".p95:0.95|g");
        expectedLines.add(prefix + ".p50:0.5|g");
        assertEquals(expectedLines, lines);
    } finally {
        if (registry != null) {
            registry.shutdown();
        }
        if (receiver != null) {
            receiver.stop();
        }
        if (receiverThread != null) {
            receiverThread.join(joinTimeout);
        }
    }
}

Example 3 with TaskManagerMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup in project flink by apache.

the class JMXReporterTest method testJMXAvailability.

/**
	 * Verifies that we can connect to multiple JMXReporters running on the same machine.
	 *
	 * @throws Exception
	 */
@Test
public void testJMXAvailability() throws Exception {
    Configuration cfg = new Configuration();
    cfg.setString(ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter.class.getName());
    cfg.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test1,test2");
    cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
    cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1.port", "9040-9055");
    cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
    cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2.port", "9040-9055");
    MetricRegistry reg = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(cfg));
    TaskManagerMetricGroup mg = new TaskManagerMetricGroup(reg, "host", "tm");
    List<MetricReporter> reporters = reg.getReporters();
    assertTrue(reporters.size() == 2);
    MetricReporter rep1 = reporters.get(0);
    MetricReporter rep2 = reporters.get(1);
    Gauge<Integer> g1 = new Gauge<Integer>() {

        @Override
        public Integer getValue() {
            return 1;
        }
    };
    Gauge<Integer> g2 = new Gauge<Integer>() {

        @Override
        public Integer getValue() {
            return 2;
        }
    };
    rep1.notifyOfAddedMetric(g1, "rep1", new FrontMetricGroup<>(0, new TaskManagerMetricGroup(reg, "host", "tm")));
    rep2.notifyOfAddedMetric(g2, "rep2", new FrontMetricGroup<>(1, new TaskManagerMetricGroup(reg, "host", "tm")));
    ObjectName objectName1 = new ObjectName(JMX_DOMAIN_PREFIX + "taskmanager.rep1", JMXReporter.generateJmxTable(mg.getAllVariables()));
    ObjectName objectName2 = new ObjectName(JMX_DOMAIN_PREFIX + "taskmanager.rep2", JMXReporter.generateJmxTable(mg.getAllVariables()));
    JMXServiceURL url1 = new JMXServiceURL("service:jmx:rmi://localhost:" + ((JMXReporter) rep1).getPort() + "/jndi/rmi://localhost:" + ((JMXReporter) rep1).getPort() + "/jmxrmi");
    JMXConnector jmxCon1 = JMXConnectorFactory.connect(url1);
    MBeanServerConnection mCon1 = jmxCon1.getMBeanServerConnection();
    assertEquals(1, mCon1.getAttribute(objectName1, "Value"));
    assertEquals(2, mCon1.getAttribute(objectName2, "Value"));
    jmxCon1.close();
    JMXServiceURL url2 = new JMXServiceURL("service:jmx:rmi://localhost:" + ((JMXReporter) rep2).getPort() + "/jndi/rmi://localhost:" + ((JMXReporter) rep2).getPort() + "/jmxrmi");
    JMXConnector jmxCon2 = JMXConnectorFactory.connect(url2);
    MBeanServerConnection mCon2 = jmxCon2.getMBeanServerConnection();
    assertEquals(1, mCon2.getAttribute(objectName1, "Value"));
    assertEquals(2, mCon2.getAttribute(objectName2, "Value"));
    rep1.notifyOfRemovedMetric(g1, "rep1", null);
    rep1.notifyOfRemovedMetric(g2, "rep2", null);
    jmxCon2.close();
    rep1.close();
    rep2.close();
    mg.close();
    reg.shutdown();
}

Example 4 with TaskManagerMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup in project flink by apache.

the class JMXReporterTest method testMeterReporting.

/**
	 * Tests that meters are properly reported via the JMXReporter.
	 */
@Test
public void testMeterReporting() throws Exception {
    MetricRegistry registry = null;
    String meterName = "meter";
    try {
        Configuration config = new Configuration();
        config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "jmx_test");
        config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "jmx_test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
        registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
        TaskManagerMetricGroup metricGroup = new TaskManagerMetricGroup(registry, "localhost", "tmId");
        TestMeter meter = new TestMeter();
        metricGroup.meter(meterName, meter);
        MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer();
        ObjectName objectName = new ObjectName(JMX_DOMAIN_PREFIX + "taskmanager." + meterName, JMXReporter.generateJmxTable(metricGroup.getAllVariables()));
        MBeanInfo info = mBeanServer.getMBeanInfo(objectName);
        MBeanAttributeInfo[] attributeInfos = info.getAttributes();
        assertEquals(2, attributeInfos.length);
        assertEquals(meter.getRate(), mBeanServer.getAttribute(objectName, "Rate"));
        assertEquals(meter.getCount(), mBeanServer.getAttribute(objectName, "Count"));
    } finally {
        if (registry != null) {
            registry.shutdown();
        }
    }
}

Example 5 with TaskManagerMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup in project flink by apache.

the class JMXReporterTest method testPortConflictHandling.

/**
	 * Verifies that multiple JMXReporters can be started on the same machine and register metrics at the MBeanServer.
	 *
	 * @throws Exception if the attribute/mbean could not be found or the test is broken
	 */
@Test
public void testPortConflictHandling() throws Exception {
    Configuration cfg = new Configuration();
    cfg.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test1,test2");
    cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
    cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1.port", "9020-9035");
    cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
    cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2.port", "9020-9035");
    MetricRegistry reg = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(cfg));
    TaskManagerMetricGroup mg = new TaskManagerMetricGroup(reg, "host", "tm");
    List<MetricReporter> reporters = reg.getReporters();
    assertTrue(reporters.size() == 2);
    MetricReporter rep1 = reporters.get(0);
    MetricReporter rep2 = reporters.get(1);
    Gauge<Integer> g1 = new Gauge<Integer>() {

        @Override
        public Integer getValue() {
            return 1;
        }
    };
    Gauge<Integer> g2 = new Gauge<Integer>() {

        @Override
        public Integer getValue() {
            return 2;
        }
    };
    rep1.notifyOfAddedMetric(g1, "rep1", new FrontMetricGroup<>(0, new TaskManagerMetricGroup(reg, "host", "tm")));
    rep2.notifyOfAddedMetric(g2, "rep2", new FrontMetricGroup<>(0, new TaskManagerMetricGroup(reg, "host", "tm")));
    MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer();
    ObjectName objectName1 = new ObjectName(JMX_DOMAIN_PREFIX + "taskmanager.rep1", JMXReporter.generateJmxTable(mg.getAllVariables()));
    ObjectName objectName2 = new ObjectName(JMX_DOMAIN_PREFIX + "taskmanager.rep2", JMXReporter.generateJmxTable(mg.getAllVariables()));
    assertEquals(1, mBeanServer.getAttribute(objectName1, "Value"));
    assertEquals(2, mBeanServer.getAttribute(objectName2, "Value"));
    rep1.notifyOfRemovedMetric(g1, "rep1", null);
    rep1.notifyOfRemovedMetric(g2, "rep2", null);
    mg.close();
    reg.shutdown();
}