Examples with UnregisteredMetricsGroup org.apache.flink.metrics.groups.UnregisteredMetricsGroup used on opensource projects

Example 11 with UnregisteredMetricsGroup

use of org.apache.flink.metrics.groups.UnregisteredMetricsGroup in project flink by apache.

the class CollectionExecutor method executeUnaryOperator.

private <IN, OUT> List<OUT> executeUnaryOperator(SingleInputOperator<?, ?, ?> operator, int superStep) throws Exception {
    Operator<?> inputOp = operator.getInput();
    if (inputOp == null) {
        throw new InvalidProgramException("The unary operation " + operator.getName() + " has no input.");
    }
    @SuppressWarnings("unchecked") List<IN> inputData = (List<IN>) execute(inputOp, superStep);
    @SuppressWarnings("unchecked") SingleInputOperator<IN, OUT, ?> typedOp = (SingleInputOperator<IN, OUT, ?>) operator;
    // build the runtime context and compute broadcast variables, if necessary
    TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
    RuntimeUDFContext ctx;
    MetricGroup metrics = new UnregisteredMetricsGroup();
    if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
        ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) : new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);
        for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
            List<?> bcData = execute(bcInputs.getValue());
            ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
        }
    } else {
        ctx = null;
    }
    return typedOp.executeOnCollections(inputData, ctx, executionConfig);
}

Example 12 with UnregisteredMetricsGroup

use of org.apache.flink.metrics.groups.UnregisteredMetricsGroup in project flink by apache.

the class CheckpointCoordinatorTest method testReportStatsAfterFailure.

private void testReportStatsAfterFailure(long checkpointId, TriFunctionWithException<CheckpointCoordinator, Execution, CheckpointMetrics, ?, CheckpointException> reportFn) throws Exception {
    JobVertexID decliningVertexID = new JobVertexID();
    JobVertexID lateReportVertexID = new JobVertexID();
    ExecutionGraph executionGraph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(decliningVertexID).addJobVertex(lateReportVertexID).build();
    ExecutionVertex decliningVertex = executionGraph.getJobVertex(decliningVertexID).getTaskVertices()[0];
    ExecutionVertex lateReportVertex = executionGraph.getJobVertex(lateReportVertexID).getTaskVertices()[0];
    CheckpointStatsTracker statsTracker = new CheckpointStatsTracker(Integer.MAX_VALUE, new UnregisteredMetricsGroup());
    CheckpointCoordinator coordinator = new CheckpointCoordinatorBuilder().setExecutionGraph(executionGraph).setTimer(manuallyTriggeredScheduledExecutor).setCheckpointStatsTracker(statsTracker).build();
    CompletableFuture<CompletedCheckpoint> result = coordinator.triggerCheckpoint(false);
    manuallyTriggeredScheduledExecutor.triggerAll();
    checkState(coordinator.getNumberOfPendingCheckpoints() == 1, "wrong number of pending checkpoints: %s", coordinator.getNumberOfPendingCheckpoints());
    if (result.isDone()) {
        result.get();
    }
    coordinator.receiveDeclineMessage(new DeclineCheckpoint(executionGraph.getJobID(), decliningVertex.getCurrentExecutionAttempt().getAttemptId(), checkpointId, new CheckpointException(CHECKPOINT_DECLINED)), "test");
    CheckpointMetrics lateReportedMetrics = new CheckpointMetricsBuilder().setTotalBytesPersisted(18).setBytesPersistedOfThisCheckpoint(18).setBytesProcessedDuringAlignment(19).setAsyncDurationMillis(20).setAlignmentDurationNanos(123 * 1_000_000).setCheckpointStartDelayNanos(567 * 1_000_000).build();
    reportFn.apply(coordinator, lateReportVertex.getCurrentExecutionAttempt(), lateReportedMetrics);
    assertStatsEqual(checkpointId, lateReportVertex.getJobvertexId(), 0, lateReportedMetrics, statsTracker.createSnapshot().getHistory().getCheckpointById(checkpointId));
}

Example 13 with UnregisteredMetricsGroup

use of org.apache.flink.metrics.groups.UnregisteredMetricsGroup in project flink by apache.

the class CheckpointStatsTrackerTest method testTrackerWithoutHistory.

/**
 * Tests that the number of remembered checkpoints configuration is respected.
 */
@Test
public void testTrackerWithoutHistory() throws Exception {
    JobVertexID jobVertexID = new JobVertexID();
    ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(jobVertexID, 3, 256).build();
    ExecutionJobVertex jobVertex = graph.getJobVertex(jobVertexID);
    CheckpointStatsTracker tracker = new CheckpointStatsTracker(0, new UnregisteredMetricsGroup());
    PendingCheckpointStats pending = tracker.reportPendingCheckpoint(0, 1, CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION), singletonMap(jobVertexID, jobVertex.getParallelism()));
    pending.reportSubtaskStats(jobVertexID, createSubtaskStats(0));
    pending.reportSubtaskStats(jobVertexID, createSubtaskStats(1));
    pending.reportSubtaskStats(jobVertexID, createSubtaskStats(2));
    pending.reportCompletedCheckpoint(null);
    CheckpointStatsSnapshot snapshot = tracker.createSnapshot();
    // History should be empty
    assertFalse(snapshot.getHistory().getCheckpoints().iterator().hasNext());
    // Counts should be available
    CheckpointStatsCounts counts = snapshot.getCounts();
    assertEquals(1, counts.getNumberOfCompletedCheckpoints());
    assertEquals(1, counts.getTotalNumberOfCheckpoints());
    // Summary should be available
    CompletedCheckpointStatsSummarySnapshot summary = snapshot.getSummaryStats();
    assertEquals(1, summary.getStateSizeStats().getCount());
    assertEquals(1, summary.getEndToEndDurationStats().getCount());
    // Latest completed checkpoint
    assertNotNull(snapshot.getHistory().getLatestCompletedCheckpoint());
    assertEquals(0, snapshot.getHistory().getLatestCompletedCheckpoint().getCheckpointId());
}

Example 14 with UnregisteredMetricsGroup

use of org.apache.flink.metrics.groups.UnregisteredMetricsGroup in project flink by apache.

the class CheckpointStatsTrackerTest method testCheckpointTracking.

/**
 * Tests tracking of checkpoints.
 */
@Test
public void testCheckpointTracking() throws Exception {
    JobVertexID jobVertexID = new JobVertexID();
    ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(jobVertexID, 3, 256).build();
    ExecutionJobVertex jobVertex = graph.getJobVertex(jobVertexID);
    Map<JobVertexID, Integer> vertexToDop = singletonMap(jobVertexID, jobVertex.getParallelism());
    CheckpointStatsTracker tracker = new CheckpointStatsTracker(10, new UnregisteredMetricsGroup());
    // Completed checkpoint
    PendingCheckpointStats completed1 = tracker.reportPendingCheckpoint(0, 1, CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION), vertexToDop);
    completed1.reportSubtaskStats(jobVertexID, createSubtaskStats(0));
    completed1.reportSubtaskStats(jobVertexID, createSubtaskStats(1));
    completed1.reportSubtaskStats(jobVertexID, createSubtaskStats(2));
    completed1.reportCompletedCheckpoint(null);
    // Failed checkpoint
    PendingCheckpointStats failed = tracker.reportPendingCheckpoint(1, 1, CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION), vertexToDop);
    failed.reportFailedCheckpoint(12, null);
    // Completed savepoint
    PendingCheckpointStats savepoint = tracker.reportPendingCheckpoint(2, 1, CheckpointProperties.forSavepoint(true, SavepointFormatType.CANONICAL), vertexToDop);
    savepoint.reportSubtaskStats(jobVertexID, createSubtaskStats(0));
    savepoint.reportSubtaskStats(jobVertexID, createSubtaskStats(1));
    savepoint.reportSubtaskStats(jobVertexID, createSubtaskStats(2));
    savepoint.reportCompletedCheckpoint(null);
    // In Progress
    PendingCheckpointStats inProgress = tracker.reportPendingCheckpoint(3, 1, CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION), vertexToDop);
    RestoredCheckpointStats restored = new RestoredCheckpointStats(81, CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION), 123, null);
    tracker.reportRestoredCheckpoint(restored);
    CheckpointStatsSnapshot snapshot = tracker.createSnapshot();
    // Counts
    CheckpointStatsCounts counts = snapshot.getCounts();
    assertEquals(4, counts.getTotalNumberOfCheckpoints());
    assertEquals(1, counts.getNumberOfInProgressCheckpoints());
    assertEquals(2, counts.getNumberOfCompletedCheckpoints());
    assertEquals(1, counts.getNumberOfFailedCheckpoints());
    // Summary stats
    CompletedCheckpointStatsSummarySnapshot summary = snapshot.getSummaryStats();
    assertEquals(2, summary.getStateSizeStats().getCount());
    assertEquals(2, summary.getEndToEndDurationStats().getCount());
    // History
    CheckpointStatsHistory history = snapshot.getHistory();
    Iterator<AbstractCheckpointStats> it = history.getCheckpoints().iterator();
    assertTrue(it.hasNext());
    AbstractCheckpointStats stats = it.next();
    assertEquals(3, stats.getCheckpointId());
    assertTrue(stats.getStatus().isInProgress());
    assertTrue(it.hasNext());
    stats = it.next();
    assertEquals(2, stats.getCheckpointId());
    assertTrue(stats.getStatus().isCompleted());
    assertTrue(it.hasNext());
    stats = it.next();
    assertEquals(1, stats.getCheckpointId());
    assertTrue(stats.getStatus().isFailed());
    assertTrue(it.hasNext());
    stats = it.next();
    assertEquals(0, stats.getCheckpointId());
    assertTrue(stats.getStatus().isCompleted());
    assertFalse(it.hasNext());
    // Latest checkpoints
    assertEquals(completed1.getCheckpointId(), snapshot.getHistory().getLatestCompletedCheckpoint().getCheckpointId());
    assertEquals(savepoint.getCheckpointId(), snapshot.getHistory().getLatestSavepoint().getCheckpointId());
    assertEquals(failed.getCheckpointId(), snapshot.getHistory().getLatestFailedCheckpoint().getCheckpointId());
    assertEquals(restored, snapshot.getLatestRestoredCheckpoint());
}

Example 15 with UnregisteredMetricsGroup

use of org.apache.flink.metrics.groups.UnregisteredMetricsGroup in project flink by apache.

the class CheckpointStatsTrackerTest method testMetricsRegistration.

/**
 * Tests the registration of the checkpoint metrics.
 */
@Test
public void testMetricsRegistration() throws Exception {
    final Collection<String> registeredGaugeNames = new ArrayList<>();
    MetricGroup metricGroup = new UnregisteredMetricsGroup() {

        @Override
        public <T, G extends Gauge<T>> G gauge(String name, G gauge) {
            if (gauge != null) {
                registeredGaugeNames.add(name);
            }
            return gauge;
        }
    };
    new CheckpointStatsTracker(0, metricGroup);
    // Make sure this test is adjusted when further metrics are added
    assertTrue(registeredGaugeNames.containsAll(Arrays.asList(CheckpointStatsTracker.NUMBER_OF_CHECKPOINTS_METRIC, CheckpointStatsTracker.NUMBER_OF_IN_PROGRESS_CHECKPOINTS_METRIC, CheckpointStatsTracker.NUMBER_OF_COMPLETED_CHECKPOINTS_METRIC, CheckpointStatsTracker.NUMBER_OF_FAILED_CHECKPOINTS_METRIC, CheckpointStatsTracker.LATEST_RESTORED_CHECKPOINT_TIMESTAMP_METRIC, CheckpointStatsTracker.LATEST_COMPLETED_CHECKPOINT_SIZE_METRIC, CheckpointStatsTracker.LATEST_COMPLETED_CHECKPOINT_DURATION_METRIC, CheckpointStatsTracker.LATEST_COMPLETED_CHECKPOINT_PROCESSED_DATA_METRIC, CheckpointStatsTracker.LATEST_COMPLETED_CHECKPOINT_PERSISTED_DATA_METRIC, CheckpointStatsTracker.LATEST_COMPLETED_CHECKPOINT_EXTERNAL_PATH_METRIC)));
    assertEquals(10, registeredGaugeNames.size());
}