use of com.twitter.heron.api.metric.MultiCountMetric in project heron by twitter.
the class TMasterSinkTest method testHandleTMasterLocation.
/**
* Test whether TMasterSink would handle TMasterLocation in SingletonRegistry automatically
*/
@Test
public void testHandleTMasterLocation() throws Exception {
// create a new TMasterClientService
TMasterSink tMasterSink = new TMasterSink();
Map<String, Object> sinkConfig = new HashMap<String, Object>();
// Fill with necessary config
sinkConfig.put("tmaster-location-check-interval-sec", TMASTER_LOCATION_CHECK_INTERVAL_SECONDS);
// These are config for TMasterClient
Map<String, Object> serviceConfig = new HashMap<String, Object>();
serviceConfig.put("reconnect-interval-second", RECONNECT_INTERVAL_SECONDS);
serviceConfig.put("network-write-batch-size-bytes", 1);
serviceConfig.put("network-write-batch-time-ms", 1);
serviceConfig.put("network-read-batch-size-bytes", 1);
serviceConfig.put("network-read-batch-time-ms", 1);
serviceConfig.put("socket-send-buffer-size-bytes", 1);
serviceConfig.put("socket-received-buffer-size-bytes", 1);
sinkConfig.put("tmaster-client", serviceConfig);
// It is null since we have not set it
Assert.assertNull(tMasterSink.getCurrentTMasterLocation());
SinkContext sinkContext = new SinkContextImpl("topology-name", "metricsmgr-id", "sink-id", new MultiCountMetric());
// Start the TMasterSink
tMasterSink.init(sinkConfig, sinkContext);
// Put the TMasterLocation into SingletonRegistry
TopologyMaster.TMasterLocation oldLoc = TopologyMaster.TMasterLocation.newBuilder().setTopologyName("topology-name").setTopologyId("topology-id").setHost("host").setControllerPort(0).setMasterPort(0).build();
SingletonRegistry.INSTANCE.registerSingleton(TMASTER_LOCATION_BEAN_NAME, oldLoc);
Thread.sleep(WAIT_SECONDS * 1000);
// The TMasterService should start
Assert.assertTrue(tMasterSink.getTMasterStartedAttempts() > 0);
Assert.assertEquals(oldLoc, tMasterSink.getCurrentTMasterLocation());
Assert.assertEquals(oldLoc, tMasterSink.getCurrentTMasterLocationInService());
// Update it, the TMasterSink should pick up the new one.
TopologyMaster.TMasterLocation newLoc = TopologyMaster.TMasterLocation.newBuilder().setTopologyName("topology-name").setTopologyId("topology-id").setHost("host").setControllerPort(0).setMasterPort(1).build();
SingletonRegistry.INSTANCE.updateSingleton(TMASTER_LOCATION_BEAN_NAME, newLoc);
int lastTMasterStartedAttempts = tMasterSink.getTMasterStartedAttempts();
Thread.sleep(WAIT_SECONDS * 1000);
// The TMasterService should use the new TMasterLocation
Assert.assertTrue(tMasterSink.getTMasterStartedAttempts() > lastTMasterStartedAttempts);
Assert.assertNotSame(oldLoc, tMasterSink.getCurrentTMasterLocation());
Assert.assertNotSame(oldLoc, tMasterSink.getCurrentTMasterLocationInService());
Assert.assertEquals(newLoc, tMasterSink.getCurrentTMasterLocation());
Assert.assertEquals(newLoc, tMasterSink.getCurrentTMasterLocationInService());
tMasterSink.close();
}
use of com.twitter.heron.api.metric.MultiCountMetric in project heron by twitter.
the class MetricsManager method initSinkExecutor.
private SinkExecutor initSinkExecutor(String sinkId) {
IMetricsSink sink;
String classname = (String) config.getConfigForSink(sinkId).get(MetricsSinksConfig.CONFIG_KEY_CLASSNAME);
try {
sink = (IMetricsSink) Class.forName(classname).newInstance();
} catch (InstantiationException e) {
throw new RuntimeException(e + " IMetricsSink class must have a no-arg constructor.");
} catch (IllegalAccessException e) {
throw new RuntimeException(e + " IMetricsSink class must be concrete.");
} catch (ClassNotFoundException e) {
throw new RuntimeException(e + " IMetricsSink class must be a class path.");
}
SlaveLooper sinkExecutorLoop = new SlaveLooper();
Communicator<MetricsRecord> executorInMetricsQueue = new Communicator<MetricsRecord>(null, sinkExecutorLoop);
// Since MetricsCollector is not thread-safe,
// we need to specify individual MetricsCollector and MultiCountMetric
// for different SinkExecutor
MetricsCollector sinkMetricsCollector = new MetricsCollector(sinkExecutorLoop, metricsQueue);
MultiCountMetric internalCounters = new MultiCountMetric();
sinkMetricsCollector.registerMetric(sinkId, internalCounters, heronMetricsExportIntervalSec);
// Set up the SinkContext
SinkContext sinkContext = new SinkContextImpl(topologyName, metricsmgrId, sinkId, internalCounters);
SinkExecutor sinkExecutor = new SinkExecutor(sinkId, sink, sinkExecutorLoop, executorInMetricsQueue, sinkContext);
sinkExecutor.setPropertyMap(config.getConfigForSink(sinkId));
return sinkExecutor;
}
use of com.twitter.heron.api.metric.MultiCountMetric in project heron by twitter.
the class HandleTMasterLocationTest method before.
@Before
public void before() throws Exception {
// Get an available port
serverPort = SysUtils.getFreePort();
threadsPool = Executors.newFixedThreadPool(2);
HeronSocketOptions serverSocketOptions = new HeronSocketOptions(100 * 1024 * 1024, 100, 100 * 1024 * 1024, 100, 5 * 1024 * 1024, 5 * 1024 * 1024);
serverLooper = new NIOLooper();
// Spy it for unit test
metricsManagerServer = Mockito.spy(new MetricsManagerServer(serverLooper, SERVER_HOST, serverPort, serverSocketOptions, new MultiCountMetric()));
}
use of com.twitter.heron.api.metric.MultiCountMetric in project heron by twitter.
the class MetricsManagerServerTest method before.
@Before
public void before() throws Exception {
// Get an available port
serverPort = SysUtils.getFreePort();
threadsPool = Executors.newFixedThreadPool(2);
HeronSocketOptions serverSocketOptions = new HeronSocketOptions(100 * 1024 * 1024, 100, 100 * 1024 * 1024, 100, 5 * 1024 * 1024, 5 * 1024 * 1024);
serverLooper = new NIOLooper();
metricsManagerServer = new MetricsManagerServer(serverLooper, SERVER_HOST, serverPort, serverSocketOptions, new MultiCountMetric());
}
use of com.twitter.heron.api.metric.MultiCountMetric in project heron by twitter.
the class SinkExecutorTest method before.
@Before
public void before() throws Exception {
metricsSink = new DummyMetricsSink();
slaveLooper = new SlaveLooper();
communicator = new Communicator<>(null, slaveLooper);
SinkContext sinkContext = new SinkContextImpl("topology-name", "metricsmgr-id", "sink-id", new MultiCountMetric());
sinkExecutor = new SinkExecutor("testSinkId", metricsSink, slaveLooper, communicator, sinkContext);
}
Aggregations