Examples with OperatorID org.apache.flink.runtime.jobgraph.OperatorID used on opensource projects

Example 11 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class CheckpointMetadataLoadingTest method testAllStateRestored.

/**
 * Tests correct savepoint loading.
 */
@Test
public void testAllStateRestored() throws Exception {
    final JobID jobId = new JobID();
    final OperatorID operatorId = new OperatorID();
    final long checkpointId = Integer.MAX_VALUE + 123123L;
    final int parallelism = 128128;
    final CompletedCheckpointStorageLocation testSavepoint = createSavepointWithOperatorSubtaskState(checkpointId, operatorId, parallelism);
    final Map<JobVertexID, ExecutionJobVertex> tasks = createTasks(operatorId, parallelism, parallelism);
    final CompletedCheckpoint loaded = Checkpoints.loadAndValidateCheckpoint(jobId, tasks, testSavepoint, cl, false, CheckpointProperties.forSavepoint(false, SavepointFormatType.CANONICAL), RestoreMode.NO_CLAIM);
    assertEquals(jobId, loaded.getJobId());
    assertEquals(checkpointId, loaded.getCheckpointID());
}

Example 12 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class CompletedCheckpointStoreTest method createCheckpoint.

public static TestCompletedCheckpoint createCheckpoint(long id, SharedStateRegistry sharedStateRegistry, CheckpointProperties props) {
    int numberOfStates = 4;
    OperatorID operatorID = new OperatorID();
    Map<OperatorID, OperatorState> operatorGroupState = new HashMap<>();
    OperatorState operatorState = new OperatorState(operatorID, numberOfStates, numberOfStates);
    operatorGroupState.put(operatorID, operatorState);
    for (int i = 0; i < numberOfStates; i++) {
        OperatorSubtaskState subtaskState = new TestOperatorSubtaskState();
        operatorState.putState(i, subtaskState);
    }
    operatorState.registerSharedStates(sharedStateRegistry, id);
    return new TestCompletedCheckpoint(new JobID(), id, 0, operatorGroupState, props);
}

Example 13 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class MetadataV2Serializer method deserializeOperatorState.

@Override
protected OperatorState deserializeOperatorState(DataInputStream dis, @Nullable DeserializationContext context) throws IOException {
    final OperatorID jobVertexId = new OperatorID(dis.readLong(), dis.readLong());
    final int parallelism = dis.readInt();
    final int maxParallelism = dis.readInt();
    // this field was "chain length" before Flink 1.3, and it is still part
    // of the format, despite being unused
    dis.readInt();
    // Add task state
    final OperatorState taskState = new OperatorState(jobVertexId, parallelism, maxParallelism);
    // Sub task states
    final int numSubTaskStates = dis.readInt();
    for (int j = 0; j < numSubTaskStates; j++) {
        final int subtaskIndex = dis.readInt();
        final OperatorSubtaskState subtaskState = deserializeSubtaskState(dis, context);
        taskState.putState(subtaskIndex, subtaskState);
    }
    return taskState;
}

Example 14 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class DataSinkTask method initOutputFormat.

/**
 * Initializes the OutputFormat implementation and configuration.
 *
 * @throws RuntimeException Throws if instance of OutputFormat implementation can not be
 *     obtained.
 */
private void initOutputFormat() {
    ClassLoader userCodeClassLoader = getUserCodeClassLoader();
    // obtain task configuration (including stub parameters)
    Configuration taskConf = getTaskConfiguration();
    this.config = new TaskConfig(taskConf);
    final Pair<OperatorID, OutputFormat<IT>> operatorIDAndOutputFormat;
    InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(config, userCodeClassLoader);
    try {
        operatorIDAndOutputFormat = formatContainer.getUniqueOutputFormat();
        this.format = operatorIDAndOutputFormat.getValue();
        // check if the class is a subclass, if the check is required
        if (!OutputFormat.class.isAssignableFrom(this.format.getClass())) {
            throw new RuntimeException("The class '" + this.format.getClass().getName() + "' is not a subclass of '" + OutputFormat.class.getName() + "' as is required.");
        }
    } catch (ClassCastException ccex) {
        throw new RuntimeException("The stub class is not a proper subclass of " + OutputFormat.class.getName(), ccex);
    }
    Thread thread = Thread.currentThread();
    ClassLoader original = thread.getContextClassLoader();
    // user code
    try {
        thread.setContextClassLoader(userCodeClassLoader);
        this.format.configure(formatContainer.getParameters(operatorIDAndOutputFormat.getKey()));
    } catch (Throwable t) {
        throw new RuntimeException("The user defined 'configure()' method in the Output Format caused an error: " + t.getMessage(), t);
    } finally {
        thread.setContextClassLoader(original);
    }
}

Example 15 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class StateAssignmentOperationTest method testRepartitionBroadcastStateWithNullSubtaskState.

@Test
public void testRepartitionBroadcastStateWithNullSubtaskState() {
    OperatorID operatorID = new OperatorID();
    OperatorState operatorState = new OperatorState(operatorID, 2, 4);
    // Only the subtask 0 reports the states.
    Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(2);
    metaInfoMap1.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[] { 0, 10, 20 }, OperatorStateHandle.Mode.BROADCAST));
    metaInfoMap1.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[] { 30, 40, 50 }, OperatorStateHandle.Mode.BROADCAST));
    OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[60]));
    operatorState.putState(0, OperatorSubtaskState.builder().setManagedOperatorState(osh1).build());
    verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}