use of org.apache.flink.metrics.groups.UnregisteredMetricsGroup in project flink by apache.
the class StatsDReporterTest method testMetricAndAssert.
private void testMetricAndAssert(Metric metric, String metricName, Set<String> expectation) throws Exception {
StatsDReporter reporter = null;
DatagramSocketReceiver receiver = null;
Thread receiverThread = null;
long timeout = 5000;
long joinTimeout = 30000;
try {
receiver = new DatagramSocketReceiver();
receiverThread = new Thread(receiver);
receiverThread.start();
int port = receiver.getPort();
MetricConfig config = new MetricConfig();
config.setProperty("host", "localhost");
config.setProperty("port", String.valueOf(port));
reporter = new StatsDReporter();
reporter.open(config);
MetricGroup metricGroup = new UnregisteredMetricsGroup();
reporter.notifyOfAddedMetric(metric, metricName, metricGroup);
reporter.report();
receiver.waitUntilNumLines(expectation.size(), timeout);
assertThat(receiver.getLines()).isEqualTo(expectation);
} finally {
if (reporter != null) {
reporter.close();
}
if (receiver != null) {
receiver.stop();
}
if (receiverThread != null) {
receiverThread.join(joinTimeout);
}
}
}
use of org.apache.flink.metrics.groups.UnregisteredMetricsGroup in project flink by apache.
the class ScheduledDropwizardReporterTest method testMetricCleanup.
/**
* This test verifies that metrics are properly added and removed to/from the
* ScheduledDropwizardReporter and the underlying Dropwizard MetricRegistry.
*/
@Test
void testMetricCleanup() {
TestingScheduledDropwizardReporter rep = new TestingScheduledDropwizardReporter();
MetricGroup mp = new UnregisteredMetricsGroup();
Counter c = new SimpleCounter();
Meter m = new TestMeter();
Histogram h = new TestHistogram();
Gauge<?> g = () -> null;
rep.notifyOfAddedMetric(c, "counter", mp);
assertThat(rep.getCounters()).hasSize(1);
assertThat(rep.registry.getCounters()).hasSize(1);
rep.notifyOfAddedMetric(m, "meter", mp);
assertThat(rep.getMeters()).hasSize(1);
assertThat(rep.registry.getMeters()).hasSize(1);
rep.notifyOfAddedMetric(h, "histogram", mp);
assertThat(rep.getHistograms()).hasSize(1);
assertThat(rep.registry.getHistograms()).hasSize(1);
rep.notifyOfAddedMetric(g, "gauge", mp);
assertThat(rep.getGauges()).hasSize(1);
assertThat(rep.registry.getGauges()).hasSize(1);
rep.notifyOfRemovedMetric(c, "counter", mp);
assertThat(rep.getCounters()).hasSize(0);
assertThat(rep.registry.getCounters()).hasSize(0);
rep.notifyOfRemovedMetric(m, "meter", mp);
assertThat(rep.getMeters()).hasSize(0);
assertThat(rep.registry.getMeters()).hasSize(0);
rep.notifyOfRemovedMetric(h, "histogram", mp);
assertThat(rep.getHistograms()).hasSize(0);
assertThat(rep.registry.getHistograms()).hasSize(0);
rep.notifyOfRemovedMetric(g, "gauge", mp);
assertThat(rep.getGauges()).hasSize(0);
assertThat(rep.registry.getGauges()).hasSize(0);
}
use of org.apache.flink.metrics.groups.UnregisteredMetricsGroup in project flink by apache.
the class ClientTest method testClientServerIntegration.
/**
* Tests multiple clients querying multiple servers until 100k queries have been processed. At
* this point, the client is shut down and its verified that all ongoing requests are failed.
*/
@Test
public void testClientServerIntegration() throws Throwable {
// Config
final int numServers = 2;
final int numServerEventLoopThreads = 2;
final int numServerQueryThreads = 2;
final int numClientEventLoopThreads = 4;
final int numClientsTasks = 8;
final int batchSize = 16;
final int numKeyGroups = 1;
AbstractStateBackend abstractBackend = new MemoryStateBackend();
KvStateRegistry dummyRegistry = new KvStateRegistry();
DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
dummyEnv.setKvStateRegistry(dummyRegistry);
AbstractKeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), dummyRegistry.createTaskRegistry(new JobID(), new JobVertexID()), TtlTimeProvider.DEFAULT, new UnregisteredMetricsGroup(), Collections.emptyList(), new CloseableRegistry());
AtomicKvStateRequestStats clientStats = new AtomicKvStateRequestStats();
final MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
Client<KvStateInternalRequest, KvStateResponse> client = null;
ExecutorService clientTaskExecutor = null;
final KvStateServerImpl[] server = new KvStateServerImpl[numServers];
try {
client = new Client<>("Test Client", numClientEventLoopThreads, serializer, clientStats);
clientTaskExecutor = Executors.newFixedThreadPool(numClientsTasks);
// Create state
ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
desc.setQueryable("any");
// Create servers
KvStateRegistry[] registry = new KvStateRegistry[numServers];
AtomicKvStateRequestStats[] serverStats = new AtomicKvStateRequestStats[numServers];
final KvStateID[] ids = new KvStateID[numServers];
for (int i = 0; i < numServers; i++) {
registry[i] = new KvStateRegistry();
serverStats[i] = new AtomicKvStateRequestStats();
server[i] = new KvStateServerImpl(InetAddress.getLocalHost().getHostName(), Collections.singletonList(0).iterator(), numServerEventLoopThreads, numServerQueryThreads, registry[i], serverStats[i]);
server[i].start();
backend.setCurrentKey(1010 + i);
// Value per server
ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
state.update(201 + i);
// we know it must be a KvState but this is not exposed to the user via State
InternalKvState<Integer, ?, Integer> kvState = (InternalKvState<Integer, ?, Integer>) state;
// Register KvState (one state instance for all server)
ids[i] = registry[i].registerKvState(new JobID(), new JobVertexID(), new KeyGroupRange(0, 0), "any", kvState, getClass().getClassLoader());
}
final Client<KvStateInternalRequest, KvStateResponse> finalClient = client;
Callable<Void> queryTask = () -> {
while (true) {
if (Thread.interrupted()) {
throw new InterruptedException();
}
// Random server permutation
List<Integer> random = new ArrayList<>();
for (int j = 0; j < batchSize; j++) {
random.add(j);
}
Collections.shuffle(random);
// Dispatch queries
List<CompletableFuture<KvStateResponse>> futures = new ArrayList<>(batchSize);
for (int j = 0; j < batchSize; j++) {
int targetServer = random.get(j) % numServers;
byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(1010 + targetServer, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
KvStateInternalRequest request = new KvStateInternalRequest(ids[targetServer], serializedKeyAndNamespace);
futures.add(finalClient.sendRequest(server[targetServer].getServerAddress(), request));
}
// Verify results
for (int j = 0; j < batchSize; j++) {
int targetServer = random.get(j) % numServers;
Future<KvStateResponse> future = futures.get(j);
byte[] buf = future.get().getContent();
int value = KvStateSerializer.deserializeValue(buf, IntSerializer.INSTANCE);
assertEquals(201L + targetServer, value);
}
}
};
// Submit tasks
List<Future<Void>> taskFutures = new ArrayList<>();
for (int i = 0; i < numClientsTasks; i++) {
taskFutures.add(clientTaskExecutor.submit(queryTask));
}
long numRequests;
while ((numRequests = clientStats.getNumRequests()) < 100_000L) {
Thread.sleep(100L);
LOG.info("Number of requests {}/100_000", numRequests);
}
try {
client.shutdown().get();
} catch (Exception e) {
e.printStackTrace();
}
Assert.assertTrue(client.isEventGroupShutdown());
final CombinableMatcher<Throwable> exceptionMatcher = either(FlinkMatchers.containsCause(ClosedChannelException.class)).or(FlinkMatchers.containsCause(IllegalStateException.class));
for (Future<Void> future : taskFutures) {
try {
future.get();
fail("Did not throw expected Exception after shut down");
} catch (ExecutionException t) {
assertThat(t, exceptionMatcher);
}
}
assertEquals("Connection leak (client)", 0L, clientStats.getNumConnections());
for (int i = 0; i < numServers; i++) {
boolean success = false;
int numRetries = 0;
while (!success) {
try {
assertEquals("Connection leak (server)", 0L, serverStats[i].getNumConnections());
success = true;
} catch (Throwable t) {
if (numRetries < 10) {
LOG.info("Retrying connection leak check (server)");
Thread.sleep((numRetries + 1) * 50L);
numRetries++;
} else {
throw t;
}
}
}
}
} finally {
if (client != null) {
try {
client.shutdown().get();
} catch (Exception e) {
e.printStackTrace();
}
Assert.assertTrue(client.isEventGroupShutdown());
}
for (int i = 0; i < numServers; i++) {
if (server[i] != null) {
server[i].shutdown();
}
}
if (clientTaskExecutor != null) {
clientTaskExecutor.shutdown();
}
}
}
use of org.apache.flink.metrics.groups.UnregisteredMetricsGroup in project flink by apache.
the class BatchExecutionInternalTimeServiceTest method testBatchExecutionManagerCanBeInstantiatedWithBatchStateBackend.
@Test
public void testBatchExecutionManagerCanBeInstantiatedWithBatchStateBackend() throws Exception {
expectedException.expect(IllegalStateException.class);
expectedException.expectMessage("Batch execution specific time service can work only with BatchExecutionKeyedStateBackend");
MockEnvironment mockEnvironment = MockEnvironment.builder().build();
AbstractKeyedStateBackend<Integer> stateBackend = new MemoryStateBackend().createKeyedStateBackend(mockEnvironment, new JobID(), "dummy", KEY_SERIALIZER, 2, new KeyGroupRange(0, 1), mockEnvironment.getTaskKvStateRegistry(), TtlTimeProvider.DEFAULT, new UnregisteredMetricsGroup(), Collections.emptyList(), new CloseableRegistry());
BatchExecutionInternalTimeServiceManager.create(stateBackend, this.getClass().getClassLoader(), new DummyKeyContext(), new TestProcessingTimeService(), Collections.emptyList());
}
use of org.apache.flink.metrics.groups.UnregisteredMetricsGroup in project flink by apache.
the class LatencyTrackingStateTestBase method createKeyedBackend.
protected AbstractKeyedStateBackend<K> createKeyedBackend(TypeSerializer<K> keySerializer) throws Exception {
Environment env = new DummyEnvironment();
KeyGroupRange keyGroupRange = new KeyGroupRange(0, 127);
int numberOfKeyGroups = keyGroupRange.getNumberOfKeyGroups();
Configuration configuration = new Configuration();
configuration.setBoolean(StateBackendOptions.LATENCY_TRACK_ENABLED, true);
configuration.setInteger(StateBackendOptions.LATENCY_TRACK_SAMPLE_INTERVAL, SAMPLE_INTERVAL);
// use a very large value to not let metrics data overridden.
int historySize = 1000_000;
configuration.setInteger(StateBackendOptions.LATENCY_TRACK_HISTORY_SIZE, historySize);
HashMapStateBackend stateBackend = new HashMapStateBackend().configure(configuration, Thread.currentThread().getContextClassLoader());
return stateBackend.createKeyedStateBackend(env, new JobID(), "test_op", keySerializer, numberOfKeyGroups, keyGroupRange, env.getTaskKvStateRegistry(), TtlTimeProvider.DEFAULT, new UnregisteredMetricsGroup(), Collections.emptyList(), new CloseableRegistry());
}
Aggregations