Examples with MetricGroup org.apache.flink.metrics.MetricGroup used on opensource projects

Example 16 with MetricGroup

use of org.apache.flink.metrics.MetricGroup in project flink by apache.

the class CollectionExecutor method executeBinaryOperator.

private <IN1, IN2, OUT> List<OUT> executeBinaryOperator(DualInputOperator<?, ?, ?, ?> operator, int superStep) throws Exception {
    Operator<?> inputOp1 = operator.getFirstInput();
    Operator<?> inputOp2 = operator.getSecondInput();
    if (inputOp1 == null) {
        throw new InvalidProgramException("The binary operation " + operator.getName() + " has no first input.");
    }
    if (inputOp2 == null) {
        throw new InvalidProgramException("The binary operation " + operator.getName() + " has no second input.");
    }
    // compute inputs
    @SuppressWarnings("unchecked") List<IN1> inputData1 = (List<IN1>) execute(inputOp1, superStep);
    @SuppressWarnings("unchecked") List<IN2> inputData2 = (List<IN2>) execute(inputOp2, superStep);
    @SuppressWarnings("unchecked") DualInputOperator<IN1, IN2, OUT, ?> typedOp = (DualInputOperator<IN1, IN2, OUT, ?>) operator;
    // build the runtime context and compute broadcast variables, if necessary
    TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
    RuntimeUDFContext ctx;
    MetricGroup metrics = new UnregisteredMetricsGroup();
    if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
        ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) : new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);
        for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
            List<?> bcData = execute(bcInputs.getValue());
            ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
        }
    } else {
        ctx = null;
    }
    return typedOp.executeOnCollections(inputData1, inputData2, ctx, executionConfig);
}

Example 17 with MetricGroup

use of org.apache.flink.metrics.MetricGroup in project flink by apache.

the class CollectionExecutor method executeDataSource.

private <OUT> List<OUT> executeDataSource(GenericDataSourceBase<?, ?> source, int superStep) throws Exception {
    @SuppressWarnings("unchecked") GenericDataSourceBase<OUT, ?> typedSource = (GenericDataSourceBase<OUT, ?>) source;
    // build the runtime context and compute broadcast variables, if necessary
    TaskInfo taskInfo = new TaskInfo(typedSource.getName(), 1, 0, 1, 0);
    RuntimeUDFContext ctx;
    MetricGroup metrics = new UnregisteredMetricsGroup();
    if (RichInputFormat.class.isAssignableFrom(typedSource.getUserCodeWrapper().getUserCodeClass())) {
        ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) : new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);
    } else {
        ctx = null;
    }
    return typedSource.executeOnCollections(ctx, executionConfig);
}

Example 18 with MetricGroup

use of org.apache.flink.metrics.MetricGroup in project flink by apache.

the class CollectionExecutor method executeDataSink.

// --------------------------------------------------------------------------------------------
//  Operator class specific execution methods
// --------------------------------------------------------------------------------------------
private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
    Operator<?> inputOp = sink.getInput();
    if (inputOp == null) {
        throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
    }
    @SuppressWarnings("unchecked") List<IN> input = (List<IN>) execute(inputOp);
    @SuppressWarnings("unchecked") GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;
    // build the runtime context and compute broadcast variables, if necessary
    TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
    RuntimeUDFContext ctx;
    MetricGroup metrics = new UnregisteredMetricsGroup();
    if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
        ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) : new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);
    } else {
        ctx = null;
    }
    typedSink.executeOnCollections(input, ctx, executionConfig);
}

Example 19 with MetricGroup

use of org.apache.flink.metrics.MetricGroup in project flink by apache.

the class AbstractMetricGroupTest method testScopeCachingForMultipleReporters.

@Test
public void testScopeCachingForMultipleReporters() throws Exception {
    Configuration config = new Configuration();
    config.setString(ConfigConstants.METRICS_SCOPE_NAMING_TM, "A.B.C.D");
    config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test1,test2");
    config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter1.class.getName());
    config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "-");
    config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter2.class.getName());
    config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "!");
    MetricRegistry testRegistry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
    try {
        MetricGroup tmGroup = new TaskManagerMetricGroup(testRegistry, "host", "id");
        tmGroup.counter("1");
        assertEquals("Reporters were not properly instantiated", 2, testRegistry.getReporters().size());
        for (MetricReporter reporter : testRegistry.getReporters()) {
            ScopeCheckingTestReporter typedReporter = (ScopeCheckingTestReporter) reporter;
            if (typedReporter.failureCause != null) {
                throw typedReporter.failureCause;
            }
        }
    } finally {
        testRegistry.shutdown();
    }
}

Example 20 with MetricGroup

use of org.apache.flink.metrics.MetricGroup in project flink by apache.

the class MetricGroupRegistrationTest method testDuplicateGroupName.

/**
	 * Verifies that when attempting to create a group with the name of an existing one the existing one will be returned instead.
	 */
@Test
public void testDuplicateGroupName() {
    Configuration config = new Configuration();
    MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
    MetricGroup root = new TaskManagerMetricGroup(registry, "host", "id");
    MetricGroup group1 = root.addGroup("group");
    MetricGroup group2 = root.addGroup("group");
    MetricGroup group3 = root.addGroup("group");
    Assert.assertTrue(group1 == group2 && group2 == group3);
    registry.shutdown();
}