Examples with JobVertexID org.apache.flink.runtime.jobgraph.JobVertexID used on opensource projects

Example 61 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class SavepointV1Test method createTaskStates.

static Collection<TaskState> createTaskStates(int numTaskStates, int numSubtasksPerTask) throws IOException {
    Random random = new Random(numTaskStates * 31 + numSubtasksPerTask);
    List<TaskState> taskStates = new ArrayList<>(numTaskStates);
    for (int stateIdx = 0; stateIdx < numTaskStates; ++stateIdx) {
        int chainLength = 1 + random.nextInt(8);
        TaskState taskState = new TaskState(new JobVertexID(), numSubtasksPerTask, 128, chainLength);
        int noNonPartitionableStateAtIndex = random.nextInt(chainLength);
        int noOperatorStateBackendAtIndex = random.nextInt(chainLength);
        int noOperatorStateStreamAtIndex = random.nextInt(chainLength);
        boolean hasKeyedBackend = random.nextInt(4) != 0;
        boolean hasKeyedStream = random.nextInt(4) != 0;
        for (int subtaskIdx = 0; subtaskIdx < numSubtasksPerTask; subtaskIdx++) {
            List<StreamStateHandle> nonPartitionableStates = new ArrayList<>(chainLength);
            List<OperatorStateHandle> operatorStatesBackend = new ArrayList<>(chainLength);
            List<OperatorStateHandle> operatorStatesStream = new ArrayList<>(chainLength);
            for (int chainIdx = 0; chainIdx < chainLength; ++chainIdx) {
                StreamStateHandle nonPartitionableState = new TestByteStreamStateHandleDeepCompare("a-" + chainIdx, ("Hi-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
                StreamStateHandle operatorStateBackend = new TestByteStreamStateHandleDeepCompare("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
                StreamStateHandle operatorStateStream = new TestByteStreamStateHandleDeepCompare("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
                Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>();
                offsetsMap.put("A", new OperatorStateHandle.StateMetaInfo(new long[] { 0, 10, 20 }, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
                offsetsMap.put("B", new OperatorStateHandle.StateMetaInfo(new long[] { 30, 40, 50 }, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
                offsetsMap.put("C", new OperatorStateHandle.StateMetaInfo(new long[] { 60, 70, 80 }, OperatorStateHandle.Mode.BROADCAST));
                if (chainIdx != noNonPartitionableStateAtIndex) {
                    nonPartitionableStates.add(nonPartitionableState);
                }
                if (chainIdx != noOperatorStateBackendAtIndex) {
                    OperatorStateHandle operatorStateHandleBackend = new OperatorStateHandle(offsetsMap, operatorStateBackend);
                    operatorStatesBackend.add(operatorStateHandleBackend);
                }
                if (chainIdx != noOperatorStateStreamAtIndex) {
                    OperatorStateHandle operatorStateHandleStream = new OperatorStateHandle(offsetsMap, operatorStateStream);
                    operatorStatesStream.add(operatorStateHandleStream);
                }
            }
            KeyGroupsStateHandle keyedStateBackend = null;
            KeyGroupsStateHandle keyedStateStream = null;
            if (hasKeyedBackend) {
                keyedStateBackend = new KeyGroupsStateHandle(new KeyGroupRangeOffsets(1, 1, new long[] { 42 }), new TestByteStreamStateHandleDeepCompare("c", "Hello".getBytes(ConfigConstants.DEFAULT_CHARSET)));
            }
            if (hasKeyedStream) {
                keyedStateStream = new KeyGroupsStateHandle(new KeyGroupRangeOffsets(1, 1, new long[] { 23 }), new TestByteStreamStateHandleDeepCompare("d", "World".getBytes(ConfigConstants.DEFAULT_CHARSET)));
            }
            taskState.putState(subtaskIdx, new SubtaskState(new ChainedStateHandle<>(nonPartitionableStates), new ChainedStateHandle<>(operatorStatesBackend), new ChainedStateHandle<>(operatorStatesStream), keyedStateStream, keyedStateBackend));
        }
        taskStates.add(taskState);
    }
    return taskStates;
}

Example 62 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class CheckpointStatsTrackerTest method testTrackerWithoutHistory.

/**
	 * Tests that the number of remembered checkpoints configuration
	 * is respected.
	 */
@Test
public void testTrackerWithoutHistory() throws Exception {
    int numberOfSubtasks = 3;
    ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
    when(jobVertex.getJobVertexId()).thenReturn(new JobVertexID());
    when(jobVertex.getParallelism()).thenReturn(numberOfSubtasks);
    CheckpointStatsTracker tracker = new CheckpointStatsTracker(0, Collections.singletonList(jobVertex), mock(JobSnapshottingSettings.class), new UnregisteredMetricsGroup());
    PendingCheckpointStats pending = tracker.reportPendingCheckpoint(0, 1, CheckpointProperties.forStandardCheckpoint());
    pending.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(0));
    pending.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(1));
    pending.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(2));
    pending.reportCompletedCheckpoint(null);
    CheckpointStatsSnapshot snapshot = tracker.createSnapshot();
    // History should be empty
    assertFalse(snapshot.getHistory().getCheckpoints().iterator().hasNext());
    // Counts should be available
    CheckpointStatsCounts counts = snapshot.getCounts();
    assertEquals(1, counts.getNumberOfCompletedCheckpoints());
    assertEquals(1, counts.getTotalNumberOfCheckpoints());
    // Summary should be available
    CompletedCheckpointStatsSummary summary = snapshot.getSummaryStats();
    assertEquals(1, summary.getStateSizeStats().getCount());
    assertEquals(1, summary.getEndToEndDurationStats().getCount());
    assertEquals(1, summary.getAlignmentBufferedStats().getCount());
    // Latest completed checkpoint
    assertNotNull(snapshot.getHistory().getLatestCompletedCheckpoint());
    assertEquals(0, snapshot.getHistory().getLatestCompletedCheckpoint().getCheckpointId());
}

Example 63 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class CheckpointStatsTrackerTest method testMetrics.

/**
	 * Tests the registered metrics.
	 */
@Test
public void testMetrics() throws Exception {
    MetricGroup metricGroup = mock(MetricGroup.class);
    ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
    when(jobVertex.getJobVertexId()).thenReturn(new JobVertexID());
    when(jobVertex.getParallelism()).thenReturn(1);
    new CheckpointStatsTracker(0, Collections.singletonList(jobVertex), mock(JobSnapshottingSettings.class), metricGroup);
    verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.NUMBER_OF_CHECKPOINTS_METRIC), any(Gauge.class));
    verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.NUMBER_OF_IN_PROGRESS_CHECKPOINTS_METRIC), any(Gauge.class));
    verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.NUMBER_OF_COMPLETED_CHECKPOINTS_METRIC), any(Gauge.class));
    verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.NUMBER_OF_FAILED_CHECKPOINTS_METRIC), any(Gauge.class));
    verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.LATEST_RESTORED_CHECKPOINT_TIMESTAMP_METRIC), any(Gauge.class));
    verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.LATEST_COMPLETED_CHECKPOINT_SIZE_METRIC), any(Gauge.class));
    verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.LATEST_COMPLETED_CHECKPOINT_DURATION_METRIC), any(Gauge.class));
    verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.LATEST_COMPLETED_CHECKPOINT_ALIGNMENT_BUFFERED_METRIC), any(Gauge.class));
    verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.LATEST_COMPLETED_CHECKPOINT_EXTERNAL_PATH_METRIC), any(Gauge.class));
    // Make sure this test is adjusted when further metrics are added
    verify(metricGroup, times(9)).gauge(any(String.class), any(Gauge.class));
}

Example 64 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class CheckpointStatsTrackerTest method testCheckpointTracking.

/**
	 * Tests tracking of checkpoints.
	 */
@Test
public void testCheckpointTracking() throws Exception {
    int numberOfSubtasks = 3;
    ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
    when(jobVertex.getJobVertexId()).thenReturn(new JobVertexID());
    when(jobVertex.getParallelism()).thenReturn(numberOfSubtasks);
    CheckpointStatsTracker tracker = new CheckpointStatsTracker(10, Collections.singletonList(jobVertex), mock(JobSnapshottingSettings.class), new UnregisteredMetricsGroup());
    // Completed checkpoint
    PendingCheckpointStats completed1 = tracker.reportPendingCheckpoint(0, 1, CheckpointProperties.forStandardCheckpoint());
    completed1.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(0));
    completed1.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(1));
    completed1.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(2));
    completed1.reportCompletedCheckpoint(null);
    // Failed checkpoint
    PendingCheckpointStats failed = tracker.reportPendingCheckpoint(1, 1, CheckpointProperties.forStandardCheckpoint());
    failed.reportFailedCheckpoint(12, null);
    // Completed savepoint
    PendingCheckpointStats savepoint = tracker.reportPendingCheckpoint(2, 1, CheckpointProperties.forStandardSavepoint());
    savepoint.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(0));
    savepoint.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(1));
    savepoint.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(2));
    savepoint.reportCompletedCheckpoint(null);
    // In Progress
    PendingCheckpointStats inProgress = tracker.reportPendingCheckpoint(3, 1, CheckpointProperties.forStandardCheckpoint());
    RestoredCheckpointStats restored = new RestoredCheckpointStats(81, CheckpointProperties.forStandardCheckpoint(), 123, null);
    tracker.reportRestoredCheckpoint(restored);
    CheckpointStatsSnapshot snapshot = tracker.createSnapshot();
    // Counts
    CheckpointStatsCounts counts = snapshot.getCounts();
    assertEquals(4, counts.getTotalNumberOfCheckpoints());
    assertEquals(1, counts.getNumberOfInProgressCheckpoints());
    assertEquals(2, counts.getNumberOfCompletedCheckpoints());
    assertEquals(1, counts.getNumberOfFailedCheckpoints());
    // Summary stats
    CompletedCheckpointStatsSummary summary = snapshot.getSummaryStats();
    assertEquals(2, summary.getStateSizeStats().getCount());
    assertEquals(2, summary.getEndToEndDurationStats().getCount());
    assertEquals(2, summary.getAlignmentBufferedStats().getCount());
    // History
    CheckpointStatsHistory history = snapshot.getHistory();
    Iterator<AbstractCheckpointStats> it = history.getCheckpoints().iterator();
    assertTrue(it.hasNext());
    AbstractCheckpointStats stats = it.next();
    assertEquals(3, stats.getCheckpointId());
    assertTrue(stats.getStatus().isInProgress());
    assertTrue(it.hasNext());
    stats = it.next();
    assertEquals(2, stats.getCheckpointId());
    assertTrue(stats.getStatus().isCompleted());
    assertTrue(it.hasNext());
    stats = it.next();
    assertEquals(1, stats.getCheckpointId());
    assertTrue(stats.getStatus().isFailed());
    assertTrue(it.hasNext());
    stats = it.next();
    assertEquals(0, stats.getCheckpointId());
    assertTrue(stats.getStatus().isCompleted());
    assertFalse(it.hasNext());
    // Latest checkpoints
    assertEquals(completed1.getCheckpointId(), snapshot.getHistory().getLatestCompletedCheckpoint().getCheckpointId());
    assertEquals(savepoint.getCheckpointId(), snapshot.getHistory().getLatestSavepoint().getCheckpointId());
    assertEquals(failed.getCheckpointId(), snapshot.getHistory().getLatestFailedCheckpoint().getCheckpointId());
    assertEquals(restored, snapshot.getLatestRestoredCheckpoint());
}

Example 65 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class CheckpointStatsTrackerTest method createTestTracker.

// ------------------------------------------------------------------------
/**
	 * Creates a "disabled" checkpoint tracker for tests.
	 */
static CheckpointStatsTracker createTestTracker() {
    ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
    when(jobVertex.getJobVertexId()).thenReturn(new JobVertexID());
    when(jobVertex.getParallelism()).thenReturn(1);
    return new CheckpointStatsTracker(0, Collections.singletonList(jobVertex), mock(JobSnapshottingSettings.class), new UnregisteredMetricsGroup());
}