Examples with MetricRegistry org.apache.flink.runtime.metrics.MetricRegistry used on opensource projects

Example 1 with MetricRegistry

use of org.apache.flink.runtime.metrics.MetricRegistry in project flink by apache.

the class TaskManagerServices method fromConfiguration.

// --------------------------------------------------------------------------------------------
//  Static factory methods for task manager services
// --------------------------------------------------------------------------------------------
/**
	 * Creates and returns the task manager services.
	 *
	 * @param resourceID resource ID of the task manager
	 * @param taskManagerServicesConfiguration task manager configuration
	 * @return task manager components
	 * @throws Exception
	 */
public static TaskManagerServices fromConfiguration(TaskManagerServicesConfiguration taskManagerServicesConfiguration, ResourceID resourceID) throws Exception {
    // pre-start checks
    checkTempDirs(taskManagerServicesConfiguration.getTmpDirPaths());
    final NetworkEnvironment network = createNetworkEnvironment(taskManagerServicesConfiguration);
    network.start();
    final TaskManagerLocation taskManagerLocation = new TaskManagerLocation(resourceID, taskManagerServicesConfiguration.getTaskManagerAddress(), network.getConnectionManager().getDataPort());
    // this call has to happen strictly after the network stack has been initialized
    final MemoryManager memoryManager = createMemoryManager(taskManagerServicesConfiguration);
    // start the I/O manager, it will create some temp directories.
    final IOManager ioManager = new IOManagerAsync(taskManagerServicesConfiguration.getTmpDirPaths());
    final MetricRegistry metricRegistry = new MetricRegistry(taskManagerServicesConfiguration.getMetricRegistryConfiguration());
    final TaskManagerMetricGroup taskManagerMetricGroup = new TaskManagerMetricGroup(metricRegistry, taskManagerLocation.getHostname(), taskManagerLocation.getResourceID().toString());
    // Initialize the TM metrics
    TaskExecutorMetricsInitializer.instantiateStatusMetrics(taskManagerMetricGroup, network);
    final BroadcastVariableManager broadcastVariableManager = new BroadcastVariableManager();
    final FileCache fileCache = new FileCache(taskManagerServicesConfiguration.getTmpDirPaths());
    final List<ResourceProfile> resourceProfiles = new ArrayList<>(taskManagerServicesConfiguration.getNumberOfSlots());
    for (int i = 0; i < taskManagerServicesConfiguration.getNumberOfSlots(); i++) {
        resourceProfiles.add(new ResourceProfile(1.0, 42));
    }
    final TimerService<AllocationID> timerService = new TimerService<>(new ScheduledThreadPoolExecutor(1), taskManagerServicesConfiguration.getTimerServiceShutdownTimeout());
    final TaskSlotTable taskSlotTable = new TaskSlotTable(resourceProfiles, timerService);
    final JobManagerTable jobManagerTable = new JobManagerTable();
    final JobLeaderService jobLeaderService = new JobLeaderService(taskManagerLocation);
    return new TaskManagerServices(taskManagerLocation, memoryManager, ioManager, network, metricRegistry, taskManagerMetricGroup, broadcastVariableManager, fileCache, taskSlotTable, jobManagerTable, jobLeaderService);
}

Example 2 with MetricRegistry

use of org.apache.flink.runtime.metrics.MetricRegistry in project flink by apache.

the class JobManagerRunnerMockTest method setUp.

@Before
public void setUp() throws Exception {
    RpcService mockRpc = mock(RpcService.class);
    when(mockRpc.getAddress()).thenReturn("localhost");
    jobManager = mock(JobMaster.class);
    jobManagerGateway = mock(JobMasterGateway.class);
    when(jobManager.getSelf()).thenReturn(jobManagerGateway);
    when(jobManager.getRpcService()).thenReturn(mockRpc);
    PowerMockito.whenNew(JobMaster.class).withAnyArguments().thenReturn(jobManager);
    jobCompletion = new TestingOnCompletionActions();
    leaderElectionService = mock(LeaderElectionService.class);
    when(leaderElectionService.hasLeadership()).thenReturn(true);
    SubmittedJobGraphStore submittedJobGraphStore = mock(SubmittedJobGraphStore.class);
    blobStore = mock(BlobStore.class);
    HighAvailabilityServices haServices = mock(HighAvailabilityServices.class);
    when(haServices.getJobManagerLeaderElectionService(any(JobID.class))).thenReturn(leaderElectionService);
    when(haServices.getSubmittedJobGraphStore()).thenReturn(submittedJobGraphStore);
    when(haServices.createBlobStore()).thenReturn(blobStore);
    when(haServices.getRunningJobsRegistry()).thenReturn(runningJobsRegistry);
    HeartbeatServices heartbeatServices = mock(HeartbeatServices.class);
    runner = PowerMockito.spy(new JobManagerRunner(ResourceID.generate(), new JobGraph("test", new JobVertex("vertex")), mock(Configuration.class), mockRpc, haServices, heartbeatServices, JobManagerServices.fromConfiguration(new Configuration(), haServices), new MetricRegistry(MetricRegistryConfiguration.defaultMetricRegistryConfiguration()), jobCompletion, jobCompletion));
}

Example 3 with MetricRegistry

use of org.apache.flink.runtime.metrics.MetricRegistry in project flink by apache.

the class ResourceManagerTaskExecutorTest method createAndStartResourceManager.

private StandaloneResourceManager createAndStartResourceManager(TestingLeaderElectionService rmLeaderElectionService, FatalErrorHandler fatalErrorHandler) throws Exception {
    TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
    highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
    TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
    ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.seconds(5L), Time.seconds(5L), Time.minutes(5L));
    MetricRegistry metricRegistry = mock(MetricRegistry.class);
    JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(highAvailabilityServices, rpcService.getScheduledExecutor(), resourceManagerConfiguration.getJobTimeout());
    StandaloneResourceManager resourceManager = new StandaloneResourceManager(rpcService, resourceManagerConfiguration, highAvailabilityServices, slotManagerFactory, metricRegistry, jobLeaderIdService, fatalErrorHandler);
    resourceManager.start();
    return resourceManager;
}

Example 4 with MetricRegistry

use of org.apache.flink.runtime.metrics.MetricRegistry in project flink by apache.

the class StatsDReporterTest method testStatsDHistogramReporting.

/**
	 * Tests that histograms are properly reported via the StatsD reporter
	 */
@Test
public void testStatsDHistogramReporting() throws Exception {
    MetricRegistry registry = null;
    DatagramSocketReceiver receiver = null;
    Thread receiverThread = null;
    long timeout = 5000;
    long joinTimeout = 30000;
    String histogramName = "histogram";
    try {
        receiver = new DatagramSocketReceiver();
        receiverThread = new Thread(receiver);
        receiverThread.start();
        int port = receiver.getPort();
        Configuration config = new Configuration();
        config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test");
        config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, StatsDReporter.class.getName());
        config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_INTERVAL_SUFFIX, "1 SECONDS");
        config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test.host", "localhost");
        config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test.port", "" + port);
        registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
        TaskManagerMetricGroup metricGroup = new TaskManagerMetricGroup(registry, "localhost", "tmId");
        TestingHistogram histogram = new TestingHistogram();
        metricGroup.histogram(histogramName, histogram);
        receiver.waitUntilNumLines(11, timeout);
        Set<String> lines = receiver.getLines();
        String prefix = metricGroup.getMetricIdentifier(histogramName);
        Set<String> expectedLines = new HashSet<>();
        expectedLines.add(prefix + ".count:1|g");
        expectedLines.add(prefix + ".mean:3.0|g");
        expectedLines.add(prefix + ".min:6|g");
        expectedLines.add(prefix + ".max:5|g");
        expectedLines.add(prefix + ".stddev:4.0|g");
        expectedLines.add(prefix + ".p75:0.75|g");
        expectedLines.add(prefix + ".p98:0.98|g");
        expectedLines.add(prefix + ".p99:0.99|g");
        expectedLines.add(prefix + ".p999:0.999|g");
        expectedLines.add(prefix + ".p95:0.95|g");
        expectedLines.add(prefix + ".p50:0.5|g");
        assertEquals(expectedLines, lines);
    } finally {
        if (registry != null) {
            registry.shutdown();
        }
        if (receiver != null) {
            receiver.stop();
        }
        if (receiverThread != null) {
            receiverThread.join(joinTimeout);
        }
    }
}

Example 5 with MetricRegistry

use of org.apache.flink.runtime.metrics.MetricRegistry in project flink by apache.

the class JMXReporterTest method testJMXAvailability.

/**
	 * Verifies that we can connect to multiple JMXReporters running on the same machine.
	 *
	 * @throws Exception
	 */
@Test
public void testJMXAvailability() throws Exception {
    Configuration cfg = new Configuration();
    cfg.setString(ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter.class.getName());
    cfg.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test1,test2");
    cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
    cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1.port", "9040-9055");
    cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
    cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2.port", "9040-9055");
    MetricRegistry reg = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(cfg));
    TaskManagerMetricGroup mg = new TaskManagerMetricGroup(reg, "host", "tm");
    List<MetricReporter> reporters = reg.getReporters();
    assertTrue(reporters.size() == 2);
    MetricReporter rep1 = reporters.get(0);
    MetricReporter rep2 = reporters.get(1);
    Gauge<Integer> g1 = new Gauge<Integer>() {

        @Override
        public Integer getValue() {
            return 1;
        }
    };
    Gauge<Integer> g2 = new Gauge<Integer>() {

        @Override
        public Integer getValue() {
            return 2;
        }
    };
    rep1.notifyOfAddedMetric(g1, "rep1", new FrontMetricGroup<>(0, new TaskManagerMetricGroup(reg, "host", "tm")));
    rep2.notifyOfAddedMetric(g2, "rep2", new FrontMetricGroup<>(1, new TaskManagerMetricGroup(reg, "host", "tm")));
    ObjectName objectName1 = new ObjectName(JMX_DOMAIN_PREFIX + "taskmanager.rep1", JMXReporter.generateJmxTable(mg.getAllVariables()));
    ObjectName objectName2 = new ObjectName(JMX_DOMAIN_PREFIX + "taskmanager.rep2", JMXReporter.generateJmxTable(mg.getAllVariables()));
    JMXServiceURL url1 = new JMXServiceURL("service:jmx:rmi://localhost:" + ((JMXReporter) rep1).getPort() + "/jndi/rmi://localhost:" + ((JMXReporter) rep1).getPort() + "/jmxrmi");
    JMXConnector jmxCon1 = JMXConnectorFactory.connect(url1);
    MBeanServerConnection mCon1 = jmxCon1.getMBeanServerConnection();
    assertEquals(1, mCon1.getAttribute(objectName1, "Value"));
    assertEquals(2, mCon1.getAttribute(objectName2, "Value"));
    jmxCon1.close();
    JMXServiceURL url2 = new JMXServiceURL("service:jmx:rmi://localhost:" + ((JMXReporter) rep2).getPort() + "/jndi/rmi://localhost:" + ((JMXReporter) rep2).getPort() + "/jmxrmi");
    JMXConnector jmxCon2 = JMXConnectorFactory.connect(url2);
    MBeanServerConnection mCon2 = jmxCon2.getMBeanServerConnection();
    assertEquals(1, mCon2.getAttribute(objectName1, "Value"));
    assertEquals(2, mCon2.getAttribute(objectName2, "Value"));
    rep1.notifyOfRemovedMetric(g1, "rep1", null);
    rep1.notifyOfRemovedMetric(g2, "rep2", null);
    jmxCon2.close();
    rep1.close();
    rep2.close();
    mg.close();
    reg.shutdown();
}