Search in sources :

Example 1 with SimpleTextOutputStream

use of org.apache.pulsar.common.util.SimpleTextOutputStream in project incubator-pulsar by apache.

the class PrometheusMetricsGenerator method generate.

public static void generate(PulsarService pulsar, boolean includeTopicMetrics, OutputStream out) throws IOException {
    ByteBuf buf = ByteBufAllocator.DEFAULT.heapBuffer();
    try {
        SimpleTextOutputStream stream = new SimpleTextOutputStream(buf);
        generateSystemMetrics(stream, pulsar.getConfiguration().getClusterName());
        NamespaceStatsAggregator.generate(pulsar, includeTopicMetrics, stream);
        FunctionsStatsGenerator.generate(pulsar.getWorkerService(), pulsar.getConfiguration().getClusterName(), stream);
        out.write(buf.array(), buf.arrayOffset(), buf.readableBytes());
    } finally {
        buf.release();
    }
}
Also used : SimpleTextOutputStream(org.apache.pulsar.common.util.SimpleTextOutputStream) ByteBuf(io.netty.buffer.ByteBuf)

Example 2 with SimpleTextOutputStream

use of org.apache.pulsar.common.util.SimpleTextOutputStream in project incubator-pulsar by apache.

the class FunctionsMetricsResource method getMetrics.

@Path("metrics")
@GET
@Produces(MediaType.TEXT_PLAIN)
public Response getMetrics() throws JsonProcessingException {
    WorkerService workerService = get();
    ByteBuf buf = ByteBufAllocator.DEFAULT.heapBuffer();
    try {
        SimpleTextOutputStream stream = new SimpleTextOutputStream(buf);
        FunctionsStatsGenerator.generate(workerService, "default", stream);
        byte[] payload = buf.array();
        int arrayOffset = buf.arrayOffset();
        int readableBytes = buf.readableBytes();
        StreamingOutput streamOut = out -> {
            out.write(payload, arrayOffset, readableBytes);
            out.flush();
        };
        return Response.ok(streamOut).build();
    } finally {
        buf.release();
    }
}
Also used : Produces(javax.ws.rs.Produces) GET(javax.ws.rs.GET) ByteBufAllocator(io.netty.buffer.ByteBufAllocator) Path(javax.ws.rs.Path) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) StreamingOutput(javax.ws.rs.core.StreamingOutput) SimpleTextOutputStream(org.apache.pulsar.common.util.SimpleTextOutputStream) MediaType(javax.ws.rs.core.MediaType) ByteBuf(io.netty.buffer.ByteBuf) Response(javax.ws.rs.core.Response) WorkerService(org.apache.pulsar.functions.worker.WorkerService) FunctionsStatsGenerator(org.apache.pulsar.functions.worker.FunctionsStatsGenerator) FunctionApiResource(org.apache.pulsar.functions.worker.rest.FunctionApiResource) SimpleTextOutputStream(org.apache.pulsar.common.util.SimpleTextOutputStream) StreamingOutput(javax.ws.rs.core.StreamingOutput) ByteBuf(io.netty.buffer.ByteBuf) WorkerService(org.apache.pulsar.functions.worker.WorkerService) Path(javax.ws.rs.Path) Produces(javax.ws.rs.Produces) GET(javax.ws.rs.GET)

Example 3 with SimpleTextOutputStream

use of org.apache.pulsar.common.util.SimpleTextOutputStream in project incubator-pulsar by apache.

the class FunctionStatsGeneratorTest method testFunctionsStatsGenerate.

@Test
public void testFunctionsStatsGenerate() {
    FunctionRuntimeManager functionRuntimeManager = mock(FunctionRuntimeManager.class);
    Map<String, FunctionRuntimeInfo> functionRuntimeInfoMap = new HashMap<>();
    WorkerService workerService = mock(WorkerService.class);
    doReturn(functionRuntimeManager).when(workerService).getFunctionRuntimeManager();
    CompletableFuture<InstanceCommunication.MetricsData> metricsDataCompletableFuture = new CompletableFuture<>();
    InstanceCommunication.MetricsData metricsData = InstanceCommunication.MetricsData.newBuilder().putMetrics("__total_processed__", InstanceCommunication.MetricsData.DataDigest.newBuilder().setCount(100.0).setMax(200.0).setSum(300.0).setMin(0.0).build()).putMetrics("__avg_latency_ms__", InstanceCommunication.MetricsData.DataDigest.newBuilder().setCount(10.0).setMax(20.0).setSum(30.0).setMin(0.0).build()).build();
    metricsDataCompletableFuture.complete(metricsData);
    Runtime runtime = mock(Runtime.class);
    doReturn(metricsDataCompletableFuture).when(runtime).getAndResetMetrics();
    RuntimeSpawner runtimeSpawner = mock(RuntimeSpawner.class);
    doReturn(runtime).when(runtimeSpawner).getRuntime();
    Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig(Function.FunctionConfig.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build();
    Function.Instance instance = Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build();
    FunctionRuntimeInfo functionRuntimeInfo = mock(FunctionRuntimeInfo.class);
    doReturn(runtimeSpawner).when(functionRuntimeInfo).getRuntimeSpawner();
    doReturn(instance).when(functionRuntimeInfo).getFunctionInstance();
    functionRuntimeInfoMap.put(Utils.getFullyQualifiedInstanceId(instance), functionRuntimeInfo);
    doReturn(functionRuntimeInfoMap).when(functionRuntimeManager).getFunctionRuntimeInfos();
    ByteBuf buf = ByteBufAllocator.DEFAULT.heapBuffer();
    SimpleTextOutputStream statsOut = new SimpleTextOutputStream(buf);
    FunctionsStatsGenerator.generate(workerService, "default", statsOut);
    String str = buf.toString(Charset.defaultCharset());
    buf.release();
    Map<String, Metric> metrics = parseMetrics(str);
    Assert.assertEquals(metrics.size(), 8);
    Metric m = metrics.get("pulsar_function__total_processed__count");
    assertEquals(m.tags.get("cluster"), "default");
    assertEquals(m.tags.get("instanceId"), "0");
    assertEquals(m.tags.get("name"), "func-1");
    assertEquals(m.tags.get("namespace"), "test-tenant/test-namespace");
    assertEquals(m.value, 100.0);
    m = metrics.get("pulsar_function__total_processed__max");
    assertEquals(m.tags.get("cluster"), "default");
    assertEquals(m.tags.get("instanceId"), "0");
    assertEquals(m.tags.get("name"), "func-1");
    assertEquals(m.tags.get("namespace"), "test-tenant/test-namespace");
    assertEquals(m.value, 200.0);
    m = metrics.get("pulsar_function__total_processed__sum");
    assertEquals(m.tags.get("cluster"), "default");
    assertEquals(m.tags.get("instanceId"), "0");
    assertEquals(m.tags.get("name"), "func-1");
    assertEquals(m.tags.get("namespace"), "test-tenant/test-namespace");
    assertEquals(m.value, 300.0);
    m = metrics.get("pulsar_function__total_processed__min");
    assertEquals(m.tags.get("cluster"), "default");
    assertEquals(m.tags.get("instanceId"), "0");
    assertEquals(m.tags.get("name"), "func-1");
    assertEquals(m.tags.get("namespace"), "test-tenant/test-namespace");
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_function__avg_latency_ms__count");
    assertEquals(m.tags.get("cluster"), "default");
    assertEquals(m.tags.get("instanceId"), "0");
    assertEquals(m.tags.get("name"), "func-1");
    assertEquals(m.tags.get("namespace"), "test-tenant/test-namespace");
    assertEquals(m.value, 10.0);
    m = metrics.get("pulsar_function__avg_latency_ms__max");
    assertEquals(m.tags.get("cluster"), "default");
    assertEquals(m.tags.get("instanceId"), "0");
    assertEquals(m.tags.get("name"), "func-1");
    assertEquals(m.tags.get("namespace"), "test-tenant/test-namespace");
    assertEquals(m.value, 20.0);
    m = metrics.get("pulsar_function__avg_latency_ms__sum");
    assertEquals(m.tags.get("cluster"), "default");
    assertEquals(m.tags.get("instanceId"), "0");
    assertEquals(m.tags.get("name"), "func-1");
    assertEquals(m.tags.get("namespace"), "test-tenant/test-namespace");
    assertEquals(m.value, 30.0);
    m = metrics.get("pulsar_function__avg_latency_ms__min");
    assertEquals(m.tags.get("cluster"), "default");
    assertEquals(m.tags.get("instanceId"), "0");
    assertEquals(m.tags.get("name"), "func-1");
    assertEquals(m.tags.get("namespace"), "test-tenant/test-namespace");
    assertEquals(m.value, 0.0);
}
Also used : SimpleTextOutputStream(org.apache.pulsar.common.util.SimpleTextOutputStream) HashMap(java.util.HashMap) ToString(lombok.ToString) ByteBuf(io.netty.buffer.ByteBuf) Function(org.apache.pulsar.functions.proto.Function) CompletableFuture(java.util.concurrent.CompletableFuture) Runtime(org.apache.pulsar.functions.runtime.Runtime) InstanceCommunication(org.apache.pulsar.functions.proto.InstanceCommunication) RuntimeSpawner(org.apache.pulsar.functions.runtime.RuntimeSpawner) Test(org.testng.annotations.Test)

Aggregations

ByteBuf (io.netty.buffer.ByteBuf)3 SimpleTextOutputStream (org.apache.pulsar.common.util.SimpleTextOutputStream)3 JsonProcessingException (com.fasterxml.jackson.core.JsonProcessingException)1 ByteBufAllocator (io.netty.buffer.ByteBufAllocator)1 HashMap (java.util.HashMap)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 GET (javax.ws.rs.GET)1 Path (javax.ws.rs.Path)1 Produces (javax.ws.rs.Produces)1 MediaType (javax.ws.rs.core.MediaType)1 Response (javax.ws.rs.core.Response)1 StreamingOutput (javax.ws.rs.core.StreamingOutput)1 ToString (lombok.ToString)1 Function (org.apache.pulsar.functions.proto.Function)1 InstanceCommunication (org.apache.pulsar.functions.proto.InstanceCommunication)1 Runtime (org.apache.pulsar.functions.runtime.Runtime)1 RuntimeSpawner (org.apache.pulsar.functions.runtime.RuntimeSpawner)1 FunctionsStatsGenerator (org.apache.pulsar.functions.worker.FunctionsStatsGenerator)1 WorkerService (org.apache.pulsar.functions.worker.WorkerService)1 FunctionApiResource (org.apache.pulsar.functions.worker.rest.FunctionApiResource)1