Examples with JobVertex org.apache.flink.runtime.jobgraph.JobVertex used on opensource projects

Example 51 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class DispatcherCleanupITCase method createJobGraph.

private JobGraph createJobGraph() {
    final JobVertex firstVertex = new JobVertex("first");
    firstVertex.setInvokableClass(NoOpInvokable.class);
    firstVertex.setParallelism(1);
    final JobVertex secondVertex = new JobVertex("second");
    secondVertex.setInvokableClass(NoOpInvokable.class);
    secondVertex.setParallelism(1);
    final CheckpointCoordinatorConfiguration checkpointCoordinatorConfiguration = CheckpointCoordinatorConfiguration.builder().setCheckpointInterval(20L).setMinPauseBetweenCheckpoints(20L).setCheckpointTimeout(10_000L).build();
    final JobCheckpointingSettings checkpointingSettings = new JobCheckpointingSettings(checkpointCoordinatorConfiguration, null);
    return JobGraphBuilder.newStreamingJobGraphBuilder().addJobVertex(firstVertex).addJobVertex(secondVertex).setJobCheckpointingSettings(checkpointingSettings).build();
}

Example 52 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class DefaultExecutionGraphBuilder method buildGraph.

public static DefaultExecutionGraph buildGraph(JobGraph jobGraph, Configuration jobManagerConfig, ScheduledExecutorService futureExecutor, Executor ioExecutor, ClassLoader classLoader, CompletedCheckpointStore completedCheckpointStore, CheckpointsCleaner checkpointsCleaner, CheckpointIDCounter checkpointIdCounter, Time rpcTimeout, BlobWriter blobWriter, Logger log, ShuffleMaster<?> shuffleMaster, JobMasterPartitionTracker partitionTracker, TaskDeploymentDescriptorFactory.PartitionLocationConstraint partitionLocationConstraint, ExecutionDeploymentListener executionDeploymentListener, ExecutionStateUpdateListener executionStateUpdateListener, long initializationTimestamp, VertexAttemptNumberStore vertexAttemptNumberStore, VertexParallelismStore vertexParallelismStore, Supplier<CheckpointStatsTracker> checkpointStatsTrackerFactory, boolean isDynamicGraph) throws JobExecutionException, JobException {
    checkNotNull(jobGraph, "job graph cannot be null");
    final String jobName = jobGraph.getName();
    final JobID jobId = jobGraph.getJobID();
    final JobInformation jobInformation = new JobInformation(jobId, jobName, jobGraph.getSerializedExecutionConfig(), jobGraph.getJobConfiguration(), jobGraph.getUserJarBlobKeys(), jobGraph.getClasspaths());
    final int maxPriorAttemptsHistoryLength = jobManagerConfig.getInteger(JobManagerOptions.MAX_ATTEMPTS_HISTORY_SIZE);
    final PartitionGroupReleaseStrategy.Factory partitionGroupReleaseStrategyFactory = PartitionGroupReleaseStrategyFactoryLoader.loadPartitionGroupReleaseStrategyFactory(jobManagerConfig);
    // create a new execution graph, if none exists so far
    final DefaultExecutionGraph executionGraph;
    try {
        executionGraph = new DefaultExecutionGraph(jobInformation, futureExecutor, ioExecutor, rpcTimeout, maxPriorAttemptsHistoryLength, classLoader, blobWriter, partitionGroupReleaseStrategyFactory, shuffleMaster, partitionTracker, partitionLocationConstraint, executionDeploymentListener, executionStateUpdateListener, initializationTimestamp, vertexAttemptNumberStore, vertexParallelismStore, isDynamicGraph);
    } catch (IOException e) {
        throw new JobException("Could not create the ExecutionGraph.", e);
    }
    try {
        executionGraph.setJsonPlan(JsonPlanGenerator.generatePlan(jobGraph));
    } catch (Throwable t) {
        log.warn("Cannot create JSON plan for job", t);
        // give the graph an empty plan
        executionGraph.setJsonPlan("{}");
    }
    // initialize the vertices that have a master initialization hook
    // file output formats create directories here, input formats create splits
    final long initMasterStart = System.nanoTime();
    log.info("Running initialization on master for job {} ({}).", jobName, jobId);
    for (JobVertex vertex : jobGraph.getVertices()) {
        String executableClass = vertex.getInvokableClassName();
        if (executableClass == null || executableClass.isEmpty()) {
            throw new JobSubmissionException(jobId, "The vertex " + vertex.getID() + " (" + vertex.getName() + ") has no invokable class.");
        }
        try {
            vertex.initializeOnMaster(classLoader);
        } catch (Throwable t) {
            throw new JobExecutionException(jobId, "Cannot initialize task '" + vertex.getName() + "': " + t.getMessage(), t);
        }
    }
    log.info("Successfully ran initialization on master in {} ms.", (System.nanoTime() - initMasterStart) / 1_000_000);
    // topologically sort the job vertices and attach the graph to the existing one
    List<JobVertex> sortedTopology = jobGraph.getVerticesSortedTopologicallyFromSources();
    if (log.isDebugEnabled()) {
        log.debug("Adding {} vertices from job graph {} ({}).", sortedTopology.size(), jobName, jobId);
    }
    executionGraph.attachJobGraph(sortedTopology);
    if (log.isDebugEnabled()) {
        log.debug("Successfully created execution graph from job graph {} ({}).", jobName, jobId);
    }
    // configure the state checkpointing
    if (isDynamicGraph) {
        // dynamic graph does not support checkpointing so we skip it
        log.warn("Skip setting up checkpointing for a job with dynamic graph.");
    } else if (isCheckpointingEnabled(jobGraph)) {
        JobCheckpointingSettings snapshotSettings = jobGraph.getCheckpointingSettings();
        // load the state backend from the application settings
        final StateBackend applicationConfiguredBackend;
        final SerializedValue<StateBackend> serializedAppConfigured = snapshotSettings.getDefaultStateBackend();
        if (serializedAppConfigured == null) {
            applicationConfiguredBackend = null;
        } else {
            try {
                applicationConfiguredBackend = serializedAppConfigured.deserializeValue(classLoader);
            } catch (IOException | ClassNotFoundException e) {
                throw new JobExecutionException(jobId, "Could not deserialize application-defined state backend.", e);
            }
        }
        final StateBackend rootBackend;
        try {
            rootBackend = StateBackendLoader.fromApplicationOrConfigOrDefault(applicationConfiguredBackend, snapshotSettings.isChangelogStateBackendEnabled(), jobManagerConfig, classLoader, log);
        } catch (IllegalConfigurationException | IOException | DynamicCodeLoadingException e) {
            throw new JobExecutionException(jobId, "Could not instantiate configured state backend", e);
        }
        // load the checkpoint storage from the application settings
        final CheckpointStorage applicationConfiguredStorage;
        final SerializedValue<CheckpointStorage> serializedAppConfiguredStorage = snapshotSettings.getDefaultCheckpointStorage();
        if (serializedAppConfiguredStorage == null) {
            applicationConfiguredStorage = null;
        } else {
            try {
                applicationConfiguredStorage = serializedAppConfiguredStorage.deserializeValue(classLoader);
            } catch (IOException | ClassNotFoundException e) {
                throw new JobExecutionException(jobId, "Could not deserialize application-defined checkpoint storage.", e);
            }
        }
        final CheckpointStorage rootStorage;
        try {
            rootStorage = CheckpointStorageLoader.load(applicationConfiguredStorage, null, rootBackend, jobManagerConfig, classLoader, log);
        } catch (IllegalConfigurationException | DynamicCodeLoadingException e) {
            throw new JobExecutionException(jobId, "Could not instantiate configured checkpoint storage", e);
        }
        // instantiate the user-defined checkpoint hooks
        final SerializedValue<MasterTriggerRestoreHook.Factory[]> serializedHooks = snapshotSettings.getMasterHooks();
        final List<MasterTriggerRestoreHook<?>> hooks;
        if (serializedHooks == null) {
            hooks = Collections.emptyList();
        } else {
            final MasterTriggerRestoreHook.Factory[] hookFactories;
            try {
                hookFactories = serializedHooks.deserializeValue(classLoader);
            } catch (IOException | ClassNotFoundException e) {
                throw new JobExecutionException(jobId, "Could not instantiate user-defined checkpoint hooks", e);
            }
            final Thread thread = Thread.currentThread();
            final ClassLoader originalClassLoader = thread.getContextClassLoader();
            thread.setContextClassLoader(classLoader);
            try {
                hooks = new ArrayList<>(hookFactories.length);
                for (MasterTriggerRestoreHook.Factory factory : hookFactories) {
                    hooks.add(MasterHooks.wrapHook(factory.create(), classLoader));
                }
            } finally {
                thread.setContextClassLoader(originalClassLoader);
            }
        }
        final CheckpointCoordinatorConfiguration chkConfig = snapshotSettings.getCheckpointCoordinatorConfiguration();
        executionGraph.enableCheckpointing(chkConfig, hooks, checkpointIdCounter, completedCheckpointStore, rootBackend, rootStorage, checkpointStatsTrackerFactory.get(), checkpointsCleaner);
    }
    return executionGraph;
}

Example 53 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class DefaultExecutionGraph method initializeJobVertices.

private void initializeJobVertices(List<JobVertex> topologicallySorted) throws JobException {
    final long createTimestamp = System.currentTimeMillis();
    for (JobVertex jobVertex : topologicallySorted) {
        final ExecutionJobVertex ejv = tasks.get(jobVertex.getID());
        initializeJobVertex(ejv, createTimestamp);
    }
}

Example 54 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class DefaultCheckpointPlanCalculatorTest method createExecutionGraph.

private ExecutionGraph createExecutionGraph(List<VertexDeclaration> vertexDeclarations, List<EdgeDeclaration> edgeDeclarations) throws Exception {
    JobVertex[] jobVertices = new JobVertex[vertexDeclarations.size()];
    for (int i = 0; i < vertexDeclarations.size(); ++i) {
        jobVertices[i] = ExecutionGraphTestUtils.createJobVertex(vertexName(i), vertexDeclarations.get(i).parallelism, NoOpInvokable.class);
    }
    for (EdgeDeclaration edgeDeclaration : edgeDeclarations) {
        jobVertices[edgeDeclaration.target].connectNewDataSetAsInput(jobVertices[edgeDeclaration.source], edgeDeclaration.distributionPattern, ResultPartitionType.PIPELINED);
    }
    ExecutionGraph graph = ExecutionGraphTestUtils.createSimpleTestGraph(jobVertices);
    graph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
    graph.transitionToRunning();
    graph.getAllExecutionVertices().forEach(task -> task.getCurrentExecutionAttempt().transitionState(ExecutionState.RUNNING));
    for (int i = 0; i < vertexDeclarations.size(); ++i) {
        JobVertexID jobVertexId = jobVertices[i].getID();
        vertexDeclarations.get(i).finishedSubtaskIndices.forEach(index -> {
            graph.getJobVertex(jobVertexId).getTaskVertices()[index].getCurrentExecutionAttempt().markFinished();
        });
    }
    return graph;
}

Example 55 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class JobGraphGenerator method finalizeWorksetIteration.

private void finalizeWorksetIteration(IterationDescriptor descr) {
    final WorksetIterationPlanNode iterNode = (WorksetIterationPlanNode) descr.getIterationNode();
    final JobVertex headVertex = descr.getHeadTask();
    final TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
    final TaskConfig headFinalOutputConfig = descr.getHeadFinalResultConfig();
    // ------------ finalize the head config with the final outputs and the sync gate
    // ------------
    {
        final int numStepFunctionOuts = headConfig.getNumOutputs();
        final int numFinalOuts = headFinalOutputConfig.getNumOutputs();
        if (numStepFunctionOuts == 0) {
            throw new CompilerException("The workset iteration has no operation on the workset inside the step function.");
        }
        headConfig.setIterationHeadFinalOutputConfig(headFinalOutputConfig);
        headConfig.setIterationHeadIndexOfSyncOutput(numStepFunctionOuts + numFinalOuts);
        final double relativeMemory = iterNode.getRelativeMemoryPerSubTask();
        if (relativeMemory <= 0) {
            throw new CompilerException("Bug: No memory has been assigned to the workset iteration.");
        }
        headConfig.setIsWorksetIteration();
        headConfig.setRelativeBackChannelMemory(relativeMemory / 2);
        headConfig.setRelativeSolutionSetMemory(relativeMemory / 2);
        // set the solution set serializer and comparator
        headConfig.setSolutionSetSerializer(iterNode.getSolutionSetSerializer());
        headConfig.setSolutionSetComparator(iterNode.getSolutionSetComparator());
    }
    // --------------------------- create the sync task ---------------------------
    final TaskConfig syncConfig;
    {
        final JobVertex sync = new JobVertex("Sync (" + iterNode.getNodeName() + ")");
        sync.setResources(iterNode.getMinResources(), iterNode.getPreferredResources());
        sync.setInvokableClass(IterationSynchronizationSinkTask.class);
        sync.setParallelism(1);
        sync.setMaxParallelism(1);
        this.auxVertices.add(sync);
        syncConfig = new TaskConfig(sync.getConfiguration());
        syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getParallelism());
        // set the number of iteration / convergence criterion for the sync
        final int maxNumIterations = iterNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
        if (maxNumIterations < 1) {
            throw new CompilerException("Cannot create workset iteration with unspecified maximum number of iterations.");
        }
        syncConfig.setNumberOfIterations(maxNumIterations);
        // connect the sync task
        sync.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
    }
    // ----------------------------- create the iteration tails -----------------------------
    // ----------------------- for next workset and solution set delta-----------------------
    {
        // we have three possible cases:
        // 1) Two tails, one for workset update, one for solution set update
        // 2) One tail for workset update, solution set update happens in an intermediate task
        // 3) One tail for solution set update, workset update happens in an intermediate task
        final PlanNode nextWorksetNode = iterNode.getNextWorkSetPlanNode();
        final PlanNode solutionDeltaNode = iterNode.getSolutionSetDeltaPlanNode();
        final boolean hasWorksetTail = nextWorksetNode.getOutgoingChannels().isEmpty();
        final boolean hasSolutionSetTail = (!iterNode.isImmediateSolutionSetUpdate()) || (!hasWorksetTail);
        {
            // get the vertex for the workset update
            final TaskConfig worksetTailConfig;
            JobVertex nextWorksetVertex = this.vertices.get(nextWorksetNode);
            if (nextWorksetVertex == null) {
                // nextWorksetVertex is chained
                TaskInChain taskInChain = this.chainedTasks.get(nextWorksetNode);
                if (taskInChain == null) {
                    throw new CompilerException("Bug: Next workset node not found as vertex or chained task.");
                }
                nextWorksetVertex = taskInChain.getContainingVertex();
                worksetTailConfig = taskInChain.getTaskConfig();
            } else {
                worksetTailConfig = new TaskConfig(nextWorksetVertex.getConfiguration());
            }
            // mark the node to perform workset updates
            worksetTailConfig.setIsWorksetIteration();
            worksetTailConfig.setIsWorksetUpdate();
            if (hasWorksetTail) {
                nextWorksetVertex.setInvokableClass(IterationTailTask.class);
                worksetTailConfig.setOutputSerializer(iterNode.getWorksetSerializer());
            }
        }
        {
            final TaskConfig solutionDeltaConfig;
            JobVertex solutionDeltaVertex = this.vertices.get(solutionDeltaNode);
            if (solutionDeltaVertex == null) {
                // last op is chained
                TaskInChain taskInChain = this.chainedTasks.get(solutionDeltaNode);
                if (taskInChain == null) {
                    throw new CompilerException("Bug: Solution Set Delta not found as vertex or chained task.");
                }
                solutionDeltaVertex = taskInChain.getContainingVertex();
                solutionDeltaConfig = taskInChain.getTaskConfig();
            } else {
                solutionDeltaConfig = new TaskConfig(solutionDeltaVertex.getConfiguration());
            }
            solutionDeltaConfig.setIsWorksetIteration();
            solutionDeltaConfig.setIsSolutionSetUpdate();
            if (hasSolutionSetTail) {
                solutionDeltaVertex.setInvokableClass(IterationTailTask.class);
                solutionDeltaConfig.setOutputSerializer(iterNode.getSolutionSetSerializer());
                // tell the head that it needs to wait for the solution set updates
                headConfig.setWaitForSolutionSetUpdate();
            } else {
                // no tail, intermediate update. must be immediate update
                if (!iterNode.isImmediateSolutionSetUpdate()) {
                    throw new CompilerException("A solution set update without dedicated tail is not set to perform immediate updates.");
                }
                solutionDeltaConfig.setIsSolutionSetUpdateWithoutReprobe();
            }
        }
    }
    // ------------------- register the aggregators -------------------
    AggregatorRegistry aggs = iterNode.getIterationNode().getIterationContract().getAggregators();
    Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();
    for (AggregatorWithName<?> agg : allAggregators) {
        if (agg.getName().equals(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME)) {
            throw new CompilerException("User defined aggregator used the same name as built-in workset " + "termination check aggregator: " + WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME);
        }
    }
    headConfig.addIterationAggregators(allAggregators);
    syncConfig.addIterationAggregators(allAggregators);
    String convAggName = aggs.getConvergenceCriterionAggregatorName();
    ConvergenceCriterion<?> convCriterion = aggs.getConvergenceCriterion();
    if (convCriterion != null || convAggName != null) {
        if (convCriterion == null) {
            throw new CompilerException("Error: Convergence criterion aggregator set, but criterion is null.");
        }
        if (convAggName == null) {
            throw new CompilerException("Error: Aggregator convergence criterion set, but aggregator is null.");
        }
        syncConfig.setConvergenceCriterion(convAggName, convCriterion);
    }
    headConfig.addIterationAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new LongSumAggregator());
    syncConfig.addIterationAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new LongSumAggregator());
    syncConfig.setImplicitConvergenceCriterion(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new WorksetEmptyConvergenceCriterion());
}