Examples with JobInformation org.apache.flink.runtime.executiongraph.JobInformation used on opensource projects

Example 16 with JobInformation

use of org.apache.flink.runtime.executiongraph.JobInformation in project flink by apache.

the class TaskExecutorSubmissionTest method createTaskDeploymentDescriptor.

static TaskDeploymentDescriptor createTaskDeploymentDescriptor(JobID jobId, String jobName, ExecutionAttemptID executionAttemptId, SerializedValue<ExecutionConfig> serializedExecutionConfig, String taskName, int maxNumberOfSubtasks, int subtaskIndex, int numberOfSubtasks, int attemptNumber, Configuration jobConfiguration, Configuration taskConfiguration, String invokableClassName, List<ResultPartitionDeploymentDescriptor> producedPartitions, List<InputGateDeploymentDescriptor> inputGates, Collection<PermanentBlobKey> requiredJarFiles, Collection<URL> requiredClasspaths) throws IOException {
    JobInformation jobInformation = new JobInformation(jobId, jobName, serializedExecutionConfig, jobConfiguration, requiredJarFiles, requiredClasspaths);
    TaskInformation taskInformation = new TaskInformation(new JobVertexID(), taskName, numberOfSubtasks, maxNumberOfSubtasks, invokableClassName, taskConfiguration);
    SerializedValue<JobInformation> serializedJobInformation = new SerializedValue<>(jobInformation);
    SerializedValue<TaskInformation> serializedJobVertexInformation = new SerializedValue<>(taskInformation);
    return new TaskDeploymentDescriptor(jobId, new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobInformation), new TaskDeploymentDescriptor.NonOffloaded<>(serializedJobVertexInformation), executionAttemptId, new AllocationID(), subtaskIndex, attemptNumber, null, producedPartitions, inputGates);
}

Example 17 with JobInformation

use of org.apache.flink.runtime.executiongraph.JobInformation in project flink by apache.

the class TestTaskBuilder method build.

public Task build() throws Exception {
    final JobVertexID jobVertexId = new JobVertexID();
    final SerializedValue<ExecutionConfig> serializedExecutionConfig = new SerializedValue<>(executionConfig);
    final JobInformation jobInformation = new JobInformation(jobId, "Test Job", serializedExecutionConfig, new Configuration(), requiredJarFileBlobKeys, Collections.emptyList());
    final TaskInformation taskInformation = new TaskInformation(jobVertexId, "Test Task", 1, 1, invokable.getName(), taskConfig);
    final TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();
    return new Task(jobInformation, taskInformation, executionAttemptId, allocationID, 0, 0, resultPartitions, inputGates, MemoryManagerBuilder.newBuilder().setMemorySize(1024 * 1024).build(), mock(IOManager.class), shuffleEnvironment, kvStateService, new BroadcastVariableManager(), new TaskEventDispatcher(), externalResourceInfoProvider, new TestTaskStateManager(), taskManagerActions, new MockInputSplitProvider(), testCheckpointResponder, new NoOpTaskOperatorEventGateway(), new TestGlobalAggregateManager(), classLoaderHandle, mock(FileCache.class), new TestingTaskManagerRuntimeInfo(taskManagerConfig), taskMetricGroup, consumableNotifier, partitionProducerStateChecker, executor);
}

Example 18 with JobInformation

use of org.apache.flink.runtime.executiongraph.JobInformation in project flink by apache.

the class StreamTaskSystemExitTest method createSystemExitTask.

private Task createSystemExitTask(final String invokableClassName, StreamOperator<?> operator) throws Exception {
    final Configuration taskConfiguration = new Configuration();
    final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
    streamConfig.setOperatorID(new OperatorID());
    streamConfig.setStreamOperator(operator);
    // for source run
    streamConfig.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    final JobInformation jobInformation = new JobInformation(new JobID(), "Test Job", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.emptyList(), Collections.emptyList());
    final TaskInformation taskInformation = new TaskInformation(new JobVertexID(), "Test Task", 1, 1, invokableClassName, taskConfiguration);
    final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();
    final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
    return new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), MemoryManagerBuilder.newBuilder().setMemorySize(32L * 1024L).build(), new IOManagerAsync(), shuffleEnvironment, new KvStateService(new KvStateRegistry(), null, null), mock(BroadcastVariableManager.class), new TaskEventDispatcher(), ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES, new TestTaskStateManager(), mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), new NoOpTaskOperatorEventGateway(), new TestGlobalAggregateManager(), TestingClassLoaderLease.newBuilder().build(), mock(FileCache.class), taskManagerRuntimeInfo, UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(), new NoOpResultPartitionConsumableNotifier(), mock(PartitionProducerStateChecker.class), Executors.directExecutor());
}

Example 19 with JobInformation

use of org.apache.flink.runtime.executiongraph.JobInformation in project flink by apache.

the class TaskDeploymentDescriptor method loadBigData.

/**
 * Loads externalized data from the BLOB store back to the object.
 *
 * @param blobService the blob store to use (may be <tt>null</tt> if {@link
 *     #serializedJobInformation} and {@link #serializedTaskInformation} are non-<tt>null</tt>)
 * @throws IOException during errors retrieving or reading the BLOBs
 * @throws ClassNotFoundException Class of a serialized object cannot be found.
 */
public void loadBigData(@Nullable PermanentBlobService blobService) throws IOException, ClassNotFoundException {
    // here, if this fails, we need to throw the exception as there is no backup path anymore
    if (serializedJobInformation instanceof Offloaded) {
        PermanentBlobKey jobInfoKey = ((Offloaded<JobInformation>) serializedJobInformation).serializedValueKey;
        Preconditions.checkNotNull(blobService);
        final File dataFile = blobService.getFile(jobId, jobInfoKey);
        // NOTE: Do not delete the job info BLOB since it may be needed again during recovery.
        // (it is deleted automatically on the BLOB server and cache when the job
        // enters a terminal state)
        SerializedValue<JobInformation> serializedValue = SerializedValue.fromBytes(FileUtils.readAllBytes(dataFile.toPath()));
        serializedJobInformation = new NonOffloaded<>(serializedValue);
    }
    // re-integrate offloaded task info from blob
    if (serializedTaskInformation instanceof Offloaded) {
        PermanentBlobKey taskInfoKey = ((Offloaded<TaskInformation>) serializedTaskInformation).serializedValueKey;
        Preconditions.checkNotNull(blobService);
        final File dataFile = blobService.getFile(jobId, taskInfoKey);
        // NOTE: Do not delete the task info BLOB since it may be needed again during recovery.
        // (it is deleted automatically on the BLOB server and cache when the job
        // enters a terminal state)
        SerializedValue<TaskInformation> serializedValue = SerializedValue.fromBytes(FileUtils.readAllBytes(dataFile.toPath()));
        serializedTaskInformation = new NonOffloaded<>(serializedValue);
    }
    for (InputGateDeploymentDescriptor inputGate : inputGates) {
        inputGate.loadBigData(blobService, jobId);
    }
    // make sure that the serialized job and task information fields are filled
    Preconditions.checkNotNull(serializedJobInformation);
    Preconditions.checkNotNull(serializedTaskInformation);
}