Examples with Metrics io.mantisrx.common.metrics.Metrics used on opensource projects

Example 11 with Metrics

use of io.mantisrx.common.metrics.Metrics in project mantis by Netflix.

the class LegacyTcpPushServer method createServer.

@Override
public RxServer<?, ?> createServer() {
    RxServer<RemoteRxEvent, RemoteRxEvent> server = RxNetty.newTcpServerBuilder(port, new ConnectionHandler<RemoteRxEvent, RemoteRxEvent>() {

        @Override
        public Observable<Void> handle(final ObservableConnection<RemoteRxEvent, RemoteRxEvent> newConnection) {
            final InetSocketAddress socketAddress = (InetSocketAddress) newConnection.getChannel().remoteAddress();
            // extract groupId, id, predicate from incoming byte[]
            return newConnection.getInput().flatMap(new Func1<RemoteRxEvent, Observable<Void>>() {

                @Override
                public Observable<Void> call(RemoteRxEvent incomingRequest) {
                    if (incomingRequest.getType() == RemoteRxEvent.Type.subscribed) {
                        Map<String, String> params = incomingRequest.getSubscribeParameters();
                        // client state
                        String id = null;
                        String slotId = null;
                        String groupId = null;
                        // sample state
                        boolean enableSampling = false;
                        long samplingTimeMsec = 0;
                        // predicate state
                        Map<String, List<String>> predicateParams = null;
                        if (params != null && !params.isEmpty()) {
                            predicateParams = new HashMap<String, List<String>>();
                            for (Entry<String, String> entry : params.entrySet()) {
                                List<String> values = new LinkedList<>();
                                values.add(entry.getValue());
                                predicateParams.put(entry.getKey(), values);
                            }
                            if (params.containsKey("id")) {
                                id = params.get("id");
                            }
                            if (params.containsKey("slotId")) {
                                slotId = params.get("slotId");
                            }
                            if (params.containsKey("groupId")) {
                                groupId = params.get("groupId");
                            }
                            if (params.containsKey("sample")) {
                                samplingTimeMsec = Long.parseLong(params.get("sample")) * 1000;
                                if (samplingTimeMsec < 50) {
                                    throw new IllegalArgumentException("Sampling rate too low: " + samplingTimeMsec);
                                }
                                enableSampling = true;
                            }
                            if (params.containsKey("sampleMSec")) {
                                samplingTimeMsec = Long.parseLong(params.get("sampleMSec"));
                                if (samplingTimeMsec < 50) {
                                    throw new IllegalArgumentException("Sampling rate too low: " + samplingTimeMsec);
                                }
                                enableSampling = true;
                            }
                        }
                        Func1<T, Boolean> predicateFunction = null;
                        if (predicate != null) {
                            predicateFunction = predicate.call(predicateParams);
                        }
                        // support legacy metrics per connection
                        Metrics sseSinkMetrics = new Metrics.Builder().name("DropOperator_outgoing_subject_" + slotId).addCounter("onNext").addCounter("dropped").build();
                        sseSinkMetrics = metricsRegistry.registerAndGet(sseSinkMetrics);
                        Counter legacyMsgProcessedCounter = sseSinkMetrics.getCounter("onNext");
                        Counter legacyDroppedWrites = sseSinkMetrics.getCounter("dropped");
                        return manageConnection(newConnection, socketAddress.getHostString(), socketAddress.getPort(), groupId, slotId, id, null, false, null, enableSampling, samplingTimeMsec, predicateFunction, null, legacyMsgProcessedCounter, legacyDroppedWrites, null);
                    }
                    return null;
                }
            });
        }
    }).pipelineConfigurator(new PipelineConfiguratorComposite<RemoteRxEvent, RemoteRxEvent>(new PipelineConfigurator<RemoteRxEvent, RemoteRxEvent>() {

        @Override
        public void configureNewPipeline(ChannelPipeline pipeline) {
            // pipeline.addLast(new LoggingHandler(LogLevel.ERROR)); // uncomment to enable debug logging
            pipeline.addLast("idleStateHandler", new IdleStateHandler(10, 2, 0));
            pipeline.addLast("heartbeat", new HeartbeatHandler());
            pipeline.addLast("gzipInflater", new JdkZlibEncoder(ZlibWrapper.GZIP));
            pipeline.addLast("gzipDeflater", new JdkZlibDecoder(ZlibWrapper.GZIP));
            // 4 bytes to encode length
            pipeline.addLast("frameEncoder", new LengthFieldPrepender(4));
            // max frame = half MB
            pipeline.addLast("frameDecoder", new LengthFieldBasedFrameDecoder(5242880, 0, 4, 0, 4));
        }
    }, new LegacyTcpPipelineConfigurator(name))).channelOption(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(1024 * 1024, 5 * 1024 * 1024)).build();
    return server;
}

Example 12 with Metrics

use of io.mantisrx.common.metrics.Metrics in project mantis by Netflix.

the class TaggingStage method tagData.

private List<TaggedData> tagData(Map<String, Object> d, Context context) {
    List<TaggedData> taggedDataList = new ArrayList<>();
    Metrics metrics = context.getMetricsRegistry().getMetric(new MetricGroupId("mql"));
    Collection<Query> queries = MQLQueryManager.getInstance().getRegisteredQueries();
    Iterator<Query> it = queries.iterator();
    while (it.hasNext()) {
        Query query = it.next();
        try {
            if (query.matches(d)) {
                Map<String, Object> projected = query.project(d);
                projected.put(MANTIS_META_SOURCE_NAME, SYNTHETIC_REQUEST_SOURCE);
                projected.put(MANTIS_META_SOURCE_TIMESTAMP, System.currentTimeMillis());
                TaggedData tg = new TaggedData(projected);
                tg.addMatchedClient(query.getSubscriptionId());
                taggedDataList.add(tg);
            }
        } catch (Exception ex) {
            if (ex instanceof ClassNotFoundException) {
                log.error("Error loading MQL: " + ex.getMessage());
                ex.printStackTrace();
                metrics.getCounter(MQL_CLASSLOADER_ERROR).increment();
            } else {
                ex.printStackTrace();
                metrics.getCounter(MQL_FAILURE).increment();
                log.error("MQL Error: " + ex.getMessage());
                log.error("MQL Query: " + query.getRawQuery());
                log.error("MQL Datum: " + d);
            }
        } catch (Error e) {
            metrics.getCounter(MQL_FAILURE).increment();
            if (!errorLogged.get()) {
                log.error("caught Error when processing MQL {} on {}", query.getRawQuery(), d.toString(), e);
                errorLogged.set(true);
            }
        }
    }
    return taggedDataList;
}

Example 13 with Metrics

use of io.mantisrx.common.metrics.Metrics in project mantis by Netflix.

the class ObservableTrigger method mantisGroupTrigger.

private static <K, V> PushTrigger<KeyValuePair<K, V>> mantisGroupTrigger(final String name, final Observable<MantisGroup<K, V>> o, final Action0 doOnComplete, final Action1<Throwable> doOnError, final long groupExpirySeconds, final Func1<K, byte[]> keyEncoder, final HashFunction hashFunction) {
    final AtomicReference<Subscription> subRef = new AtomicReference<>();
    final Gauge subscriptionActive;
    Metrics metrics = new Metrics.Builder().name("ObservableTrigger_" + name).addGauge("subscriptionActive").build();
    subscriptionActive = metrics.getGauge("subscriptionActive");
    Action1<MonitoredQueue<KeyValuePair<K, V>>> doOnStart = new Action1<MonitoredQueue<KeyValuePair<K, V>>>() {

        @Override
        public void call(final MonitoredQueue<KeyValuePair<K, V>> queue) {
            subRef.set(o.doOnSubscribe(() -> {
                logger.info("Subscription is ACTIVE for observable trigger with name: " + name);
                subscriptionActive.set(1);
            }).doOnUnsubscribe(() -> {
                logger.info("Subscription is INACTIVE for observable trigger with name: " + name);
                subscriptionActive.set(0);
            }).map((MantisGroup<K, V> data) -> {
                final byte[] keyBytes = keyEncoder.call(data.getKeyValue());
                final long keyBytesHashed = hashFunction.computeHash(keyBytes);
                return (new KeyValuePair<K, V>(keyBytesHashed, keyBytes, data.getValue()));
            }).subscribe((KeyValuePair<K, V> data) -> {
                queue.write(data);
            }, (Throwable e) -> {
                logger.warn("Observable used to push data errored, on server with name: " + name, e);
                if (doOnError != null) {
                    doOnError.call(e);
                }
            }, () -> {
                logger.info("Observable used to push data completed, on server with name: " + name);
                if (doOnComplete != null) {
                    doOnComplete.call();
                }
            }));
        }
    };
    Action1<MonitoredQueue<KeyValuePair<K, V>>> doOnStop = new Action1<MonitoredQueue<KeyValuePair<K, V>>>() {

        @Override
        public void call(MonitoredQueue<KeyValuePair<K, V>> t1) {
            if (subRef.get() != null) {
                logger.warn("Connections from next stage has dropped to 0. Do not propagate unsubscribe");
            // subRef.get().unsubscribe();
            }
        }
    };
    return new PushTrigger<>(doOnStart, doOnStop, metrics);
}

Example 14 with Metrics

use of io.mantisrx.common.metrics.Metrics in project mantis by Netflix.

the class SseWorkerConnection method hasDataDrop.

private boolean hasDataDrop() {
    final Collection<Metrics> metrics = MetricsRegistry.getInstance().getMetrics(metricNamePrefix);
    long totalDataDrop = 0L;
    if (metrics != null && !metrics.isEmpty()) {
        // logger.info("Got " + metrics.size() + " metrics for DropOperator");
        for (Metrics m : metrics) {
            final Counter dropped = m.getCounter("" + DropOperator.Counters.dropped);
            final Counter onNext = m.getCounter("" + DropOperator.Counters.onNext);
            if (dropped != null)
                totalDataDrop += dropped.value();
        }
    }
    if (totalDataDrop > lastDataDropValue) {
        lastDataDropValue = totalDataDrop;
        return true;
    }
    return false;
}

Example 15 with Metrics

use of io.mantisrx.common.metrics.Metrics in project mantis by Netflix.

the class DataDroppedPayloadSetter method setPayload.

protected void setPayload(final long intervalSecs) {
    final Collection<Metrics> metrics = MetricsRegistry.getInstance().getMetrics(metricNamePrefix);
    long totalDropped = 0L;
    long totalOnNext = 0L;
    try {
        if (metrics != null && !metrics.isEmpty()) {
            // logger.info("Got " + metrics.size() + " metrics for DropOperator");
            for (Metrics m : metrics) {
                final Counter dropped = m.getCounter("" + DropOperator.Counters.dropped);
                final Counter onNext = m.getCounter("" + DropOperator.Counters.onNext);
                if (dropped != null)
                    totalDropped += dropped.value();
                else
                    logger.warn("Unexpected to get null dropped counter for metric " + m.getMetricGroupId().id());
                if (onNext != null)
                    totalOnNext += onNext.value();
                else
                    logger.warn("Unexpected to get null onNext counter for metric " + m.getMetricGroupId().id());
            }
            final StatusPayloads.DataDropCounts dataDrop = new StatusPayloads.DataDropCounts(totalOnNext, totalDropped);
            try {
                heartbeat.addSingleUsePayload("" + StatusPayloads.Type.IncomingDataDrop, objectMapper.writeValueAsString(dataDrop));
            } catch (JsonProcessingException e) {
                logger.warn("Error writing json for dataDrop payload: " + e.getMessage());
            }
            dropCountGauge.set(dataDrop.getDroppedCount());
            onNextCountGauge.set(dataDrop.getOnNextCount());
        } else
            logger.debug("Got no metrics from DropOperator");
    } catch (Exception e) {
        logger.error(e.getMessage(), e);
    }
}