Examples with Metrics io.cdap.cdap.api.metrics.Metrics used on opensource projects

Example 6 with Metrics

use of io.cdap.cdap.api.metrics.Metrics in project cdap by caskdata.

the class BatchSQLEngineAdapter method tryRelationalTransform.

/**
 * This method is called when engine is present and is willing to try performing a relational transform.
 *
 * @param stageSpec stage specification
 * @param transform transform plugin
 * @param input     input collections
 * @return resulting collection or empty optional if tranform can't be done with this engine
 */
public Optional<SQLEngineJob<SQLDataset>> tryRelationalTransform(StageSpec stageSpec, RelationalTransform transform, Map<String, SparkCollection<Object>> input) {
    String stageName = stageSpec.getName();
    Map<String, Relation> inputRelations = input.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> sqlEngine.getRelation(new SQLRelationDefinition(e.getKey(), stageSpec.getInputSchemas().get(e.getKey())))));
    BasicRelationalTransformContext pluginContext = new BasicRelationalTransformContext(getSQLRelationalEngine(), inputRelations, stageSpec.getInputSchemas(), stageSpec.getOutputSchema());
    if (!transform.transform(pluginContext)) {
        // Plugin was not able to do relational tranform with this engine
        return Optional.empty();
    }
    if (pluginContext.getOutputRelation() == null) {
        // Plugin said that tranformation was success but failed to set output
        throw new IllegalStateException("Plugin " + transform + " did not produce a relational output");
    }
    if (!pluginContext.getOutputRelation().isValid()) {
        // An output is set to invalid relation, probably some of transforms are not supported by an engine
        return Optional.empty();
    }
    // Ensure input and output schemas for this stage are supported by the engine
    if (stageSpec.getInputSchemas().values().stream().anyMatch(s -> !sqlEngine.supportsInputSchema(s))) {
        return Optional.empty();
    }
    if (!sqlEngine.supportsOutputSchema(stageSpec.getOutputSchema())) {
        return Optional.empty();
    }
    // Validate transformation definition with engine
    SQLTransformDefinition transformDefinition = new SQLTransformDefinition(stageName, pluginContext.getOutputRelation(), stageSpec.getOutputSchema(), Collections.emptyMap(), Collections.emptyMap());
    if (!sqlEngine.canTransform(transformDefinition)) {
        return Optional.empty();
    }
    return Optional.of(runJob(stageSpec.getName(), SQLEngineJobType.EXECUTE, () -> {
        // Push all stages that need to be pushed to execute this aggregation
        input.forEach((name, collection) -> {
            if (!exists(name)) {
                push(name, stageSpec.getInputSchemas().get(name), collection);
            }
        });
        // Initialize metrics collector
        DefaultStageMetrics stageMetrics = new DefaultStageMetrics(metrics, stageName);
        StageStatisticsCollector statisticsCollector = statsCollectors.get(stageName);
        // Collect input datasets and execute transformation
        Map<String, SQLDataset> inputDatasets = input.keySet().stream().collect(Collectors.toMap(Function.identity(), this::getDatasetForStage));
        // Count input records
        for (SQLDataset inputDataset : inputDatasets.values()) {
            countRecordsIn(inputDataset, statisticsCollector, stageMetrics);
        }
        // Execute transform
        SQLTransformRequest sqlContext = new SQLTransformRequest(inputDatasets, stageSpec.getName(), pluginContext.getOutputRelation(), stageSpec.getOutputSchema());
        SQLDataset transformed = sqlEngine.transform(sqlContext);
        // Count output records
        countRecordsOut(transformed, statisticsCollector, stageMetrics);
        return transformed;
    }));
}

Example 7 with Metrics

use of io.cdap.cdap.api.metrics.Metrics in project cdap by caskdata.

the class ConnectionHandler method browse.

/**
 * Browse the connection on a given path.
 */
@POST
@TransactionPolicy(value = TransactionControl.EXPLICIT)
@Path(API_VERSION + "/contexts/{context}/connections/{connection}/browse")
public void browse(HttpServiceRequest request, HttpServiceResponder responder, @PathParam("context") String namespace, @PathParam("connection") String connection) {
    respond(namespace, responder, namespaceSummary -> {
        if (namespaceSummary.getName().equalsIgnoreCase(NamespaceId.SYSTEM.getNamespace())) {
            responder.sendError(HttpURLConnection.HTTP_BAD_REQUEST, "Browsing connection in system namespace is currently not supported");
            return;
        }
        contextAccessEnforcer.enforce(new ConnectionEntityId(namespace, ConnectionId.getConnectionId(connection)), StandardPermission.USE);
        String browseRequestString = StandardCharsets.UTF_8.decode(request.getContent()).toString();
        BrowseRequest browseRequest = GSON.fromJson(browseRequestString, BrowseRequest.class);
        if (browseRequest == null) {
            responder.sendError(HttpURLConnection.HTTP_BAD_REQUEST, "The request body is empty");
            return;
        }
        if (browseRequest.getPath() == null) {
            responder.sendError(HttpURLConnection.HTTP_BAD_REQUEST, "Path is not provided in the browse request");
            return;
        }
        Connection conn = store.getConnection(new ConnectionId(namespaceSummary, connection));
        if (getContext().isRemoteTaskEnabled()) {
            executeRemotely(namespace, browseRequestString, conn, RemoteConnectionBrowseTask.class, responder);
        } else {
            browseLocally(namespaceSummary.getName(), browseRequest, conn, responder);
        }
        Metrics child = metrics.child(ImmutableMap.of(Constants.Metrics.Tag.APP_ENTITY_TYPE, Constants.CONNECTION_SERVICE_NAME, Constants.Metrics.Tag.APP_ENTITY_TYPE_NAME, conn.getConnectionType()));
        child.count(Constants.Metrics.Connection.CONNECTION_BROWSE_COUNT, 1);
    });
}

Example 8 with Metrics

use of io.cdap.cdap.api.metrics.Metrics in project cdap by caskdata.

the class ConnectionHandler method sample.

/**
 * Retrive sample result for the connection
 */
@POST
@TransactionPolicy(value = TransactionControl.EXPLICIT)
@Path(API_VERSION + "/contexts/{context}/connections/{connection}/sample")
public void sample(HttpServiceRequest request, HttpServiceResponder responder, @PathParam("context") String namespace, @PathParam("connection") String connection) {
    respond(namespace, responder, namespaceSummary -> {
        if (namespaceSummary.getName().equalsIgnoreCase(NamespaceId.SYSTEM.getNamespace())) {
            responder.sendError(HttpURLConnection.HTTP_BAD_REQUEST, "Sampling connection in system namespace is currently not supported");
            return;
        }
        contextAccessEnforcer.enforce(new ConnectionEntityId(namespace, ConnectionId.getConnectionId(connection)), StandardPermission.USE);
        String sampleRequestString = StandardCharsets.UTF_8.decode(request.getContent()).toString();
        SampleRequest sampleRequest = GSON.fromJson(sampleRequestString, SampleRequest.class);
        if (sampleRequest == null) {
            responder.sendError(HttpURLConnection.HTTP_BAD_REQUEST, "The request body is empty");
            return;
        }
        if (sampleRequest.getPath() == null) {
            responder.sendError(HttpURLConnection.HTTP_BAD_REQUEST, "Path is not provided in the sample request");
            return;
        }
        if (sampleRequest.getLimit() <= 0) {
            responder.sendError(HttpURLConnection.HTTP_BAD_REQUEST, "Limit should be greater than 0");
            return;
        }
        Connection conn = store.getConnection(new ConnectionId(namespaceSummary, connection));
        if (getContext().isRemoteTaskEnabled()) {
            executeRemotely(namespace, sampleRequestString, conn, RemoteConnectionSampleTask.class, responder);
        } else {
            sampleLocally(namespaceSummary.getName(), sampleRequestString, conn, responder);
        }
        Metrics child = metrics.child(ImmutableMap.of(Constants.Metrics.Tag.APP_ENTITY_TYPE, Constants.CONNECTION_SERVICE_NAME, Constants.Metrics.Tag.APP_ENTITY_TYPE_NAME, conn.getConnectionType()));
        child.count(Constants.Metrics.Connection.CONNECTION_SAMPLE_COUNT, 1);
        // sample will also generate the spec, so add the metric for it
        child.count(Constants.Metrics.Connection.CONNECTION_SPEC_COUNT, 1);
    });
}

Example 9 with Metrics

use of io.cdap.cdap.api.metrics.Metrics in project cdap by caskdata.

the class BatchSQLEngineAdapterTest method setUp.

@Before
public void setUp() {
    invocationCounts = new HashMap<>();
    stageMetrics = new StageMetrics() {

        @Override
        public void count(String metricName, int delta) {
            throw new UnsupportedOperationException("not implemented");
        }

        @Override
        public void countLong(String metricName, long delta) {
            invocationCounts.compute(delta, (k, v) -> (v == null) ? 1 : v + 1);
        }

        @Override
        public void gauge(String metricName, long value) {
            throw new UnsupportedOperationException("not implemented");
        }

        @Override
        public void pipelineCount(String metricName, int delta) {
            throw new UnsupportedOperationException("not implemented");
        }

        @Override
        public void pipelineGauge(String metricName, long value) {
            throw new UnsupportedOperationException("not implemented");
        }

        @Override
        public Metrics child(Map<String, String> tags) {
            throw new UnsupportedOperationException("not implemented");
        }

        @Override
        public Map<String, String> getTags() {
            throw new UnsupportedOperationException("not implemented");
        }
    };
}

Example 10 with Metrics

use of io.cdap.cdap.api.metrics.Metrics in project cdap by caskdata.

the class MetricsHandlerTest method setupMetrics.

private static void setupMetrics() throws Exception {
    // Adding metrics for app "WordCount1" in namespace "myspace", "WCount1" in "yourspace"
    MetricsContext collector = collectionService.getContext(getServiceContext("myspace", "WordCount1", "WordCounter", "run1", "splitter"));
    collector.increment("reads", 1);
    collector.increment("writes", 1);
    collector = collectionService.getContext(getServiceContext("yourspace", "WCount1", "WordCounter", "run1", "splitter"));
    collector.increment("reads", 1);
    collector = collectionService.getContext(getServiceContext("yourspace", "WCount1", "WCounter", "run1", "splitter"));
    emitTs = System.currentTimeMillis();
    // we want to emit in two different seconds
    // todo : figure out why we need this
    TimeUnit.SECONDS.sleep(1);
    collector.increment("reads", 1);
    TimeUnit.MILLISECONDS.sleep(2000);
    collector.increment("reads", 2);
    collector = collectionService.getContext(getServiceContext("yourspace", "WCount1", "WCounter", "run1", "counter"));
    collector.increment("reads", 1);
    collector = collectionService.getContext(getMapReduceTaskContext("yourspace", "WCount1", "ClassicWordCount", MapReduceMetrics.TaskType.Mapper, "run1", "task1"));
    collector.increment("reads", 1);
    collector = collectionService.getContext(getMapReduceTaskContext("yourspace", "WCount1", "ClassicWordCount", MapReduceMetrics.TaskType.Reducer, "run1", "task2"));
    collector.increment("reads", 1);
    collector = collectionService.getContext(getServiceContext("myspace", "WordCount1", "WordCounter", "run1", "splitter"));
    collector.increment("reads", 1);
    collector.increment("writes", 1);
    collector = collectionService.getContext(getServiceContext("myspace", "WordCount1", "WordCounter", "run1", "collector"));
    collector.increment("aa", 1);
    collector.increment("zz", 1);
    collector.increment("ab", 1);
    collector = collectionService.getContext(getWorkerContext("yourspace", "WCount1", "WorkerWordCount", "run1", "task1"));
    collector.increment("workerreads", 5);
    collector.increment("workerwrites", 6);
    collector = collectionService.getContext(getWorkerContext("yourspace", "WCount1", "WorkerWordCount", "run2", "task1"));
    collector.increment("workerreads", 5);
    collector.increment("workerwrites", 6);
    // also: user metrics
    Metrics userMetrics = new ProgramUserMetrics(collectionService.getContext(getServiceContext("myspace", "WordCount1", "WordCounter", "run1", "splitter")));
    userMetrics.count("reads", 1);
    userMetrics.count("writes", 2);
    collector = collectionService.getContext(new HashMap<String, String>());
    collector.increment("resources.total.storage", 10);
    Metrics replicatorMetrics = new ProgramUserMetrics(collectionService.getContext(getWorkerContext("childctx", "Replicator", "ReplicatorWorker", "somerunid", "instance1")));
    Metrics tableMetrics = replicatorMetrics.child(ImmutableMap.of("ent", "mytable"));
    tableMetrics.count("inserts", 10);
    tableMetrics.count("ddls", 1);
    try {
        replicatorMetrics.child(ImmutableMap.of("ns", "anothernamespace"));
        Assert.fail("Creating child Metrics with duplicate tag name 'ns' should have failed.");
    } catch (IllegalArgumentException ignored) {
    }
    // need a better way to do this
    TimeUnit.SECONDS.sleep(2);
}