Examples with JobVertex org.apache.flink.runtime.jobgraph.JobVertex used on opensource projects

Example 41 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class CoLocationConstraintTest method testCreateConstraints.

@Test
public void testCreateConstraints() {
    try {
        JobVertexID id1 = new JobVertexID();
        JobVertexID id2 = new JobVertexID();
        JobVertex vertex1 = new JobVertex("vertex1", id1);
        vertex1.setParallelism(2);
        JobVertex vertex2 = new JobVertex("vertex2", id2);
        vertex2.setParallelism(3);
        CoLocationGroup group = new CoLocationGroup(vertex1, vertex2);
        AbstractID groupId = group.getId();
        assertNotNull(groupId);
        CoLocationConstraint constraint1 = group.getLocationConstraint(0);
        CoLocationConstraint constraint2 = group.getLocationConstraint(1);
        CoLocationConstraint constraint3 = group.getLocationConstraint(2);
        assertFalse(constraint1 == constraint2);
        assertFalse(constraint1 == constraint3);
        assertFalse(constraint2 == constraint3);
        assertEquals(groupId, constraint1.getGroupId());
        assertEquals(groupId, constraint2.getGroupId());
        assertEquals(groupId, constraint3.getGroupId());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 42 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class MiniClusterITCase method getSimpleJob.

private static JobGraph getSimpleJob() throws IOException {
    JobVertex task = new JobVertex("Test task");
    task.setParallelism(1);
    task.setMaxParallelism(1);
    task.setInvokableClass(NoOpInvokable.class);
    JobGraph jg = new JobGraph(new JobID(), "Test Job", task);
    jg.setAllowQueuedScheduling(true);
    jg.setScheduleMode(ScheduleMode.EAGER);
    ExecutionConfig executionConfig = new ExecutionConfig();
    executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000));
    jg.setExecutionConfig(executionConfig);
    return jg;
}

Example 43 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class ExecutionGraphBuilder method buildGraph.

/**
	 * Builds the ExecutionGraph from the JobGraph.
	 * If a prior execution graph exists, the JobGraph will be attached. If no prior execution
	 * graph exists, then the JobGraph will become attach to a new empty execution graph.
	 */
public static ExecutionGraph buildGraph(@Nullable ExecutionGraph prior, JobGraph jobGraph, Configuration jobManagerConfig, ScheduledExecutorService futureExecutor, Executor ioExecutor, SlotProvider slotProvider, ClassLoader classLoader, CheckpointRecoveryFactory recoveryFactory, Time timeout, RestartStrategy restartStrategy, MetricGroup metrics, int parallelismForAutoMax, Logger log) throws JobExecutionException, JobException {
    checkNotNull(jobGraph, "job graph cannot be null");
    final String jobName = jobGraph.getName();
    final JobID jobId = jobGraph.getJobID();
    // create a new execution graph, if none exists so far
    final ExecutionGraph executionGraph;
    try {
        executionGraph = (prior != null) ? prior : new ExecutionGraph(futureExecutor, ioExecutor, jobId, jobName, jobGraph.getJobConfiguration(), jobGraph.getSerializedExecutionConfig(), timeout, restartStrategy, jobGraph.getUserJarBlobKeys(), jobGraph.getClasspaths(), slotProvider, classLoader, metrics);
    } catch (IOException e) {
        throw new JobException("Could not create the execution graph.", e);
    }
    // set the basic properties
    executionGraph.setScheduleMode(jobGraph.getScheduleMode());
    executionGraph.setQueuedSchedulingAllowed(jobGraph.getAllowQueuedScheduling());
    try {
        executionGraph.setJsonPlan(JsonPlanGenerator.generatePlan(jobGraph));
    } catch (Throwable t) {
        log.warn("Cannot create JSON plan for job", t);
        // give the graph an empty plan
        executionGraph.setJsonPlan("{}");
    }
    // initialize the vertices that have a master initialization hook
    // file output formats create directories here, input formats create splits
    final long initMasterStart = System.nanoTime();
    log.info("Running initialization on master for job {} ({}).", jobName, jobId);
    for (JobVertex vertex : jobGraph.getVertices()) {
        String executableClass = vertex.getInvokableClassName();
        if (executableClass == null || executableClass.isEmpty()) {
            throw new JobSubmissionException(jobId, "The vertex " + vertex.getID() + " (" + vertex.getName() + ") has no invokable class.");
        }
        if (vertex.getParallelism() == ExecutionConfig.PARALLELISM_AUTO_MAX) {
            vertex.setParallelism(parallelismForAutoMax);
        }
        try {
            vertex.initializeOnMaster(classLoader);
        } catch (Throwable t) {
            throw new JobExecutionException(jobId, "Cannot initialize task '" + vertex.getName() + "': " + t.getMessage(), t);
        }
    }
    log.info("Successfully ran initialization on master in {} ms.", (System.nanoTime() - initMasterStart) / 1_000_000);
    // topologically sort the job vertices and attach the graph to the existing one
    List<JobVertex> sortedTopology = jobGraph.getVerticesSortedTopologicallyFromSources();
    if (log.isDebugEnabled()) {
        log.debug("Adding {} vertices from job graph {} ({}).", sortedTopology.size(), jobName, jobId);
    }
    executionGraph.attachJobGraph(sortedTopology);
    if (log.isDebugEnabled()) {
        log.debug("Successfully created execution graph from job graph {} ({}).", jobName, jobId);
    }
    // configure the state checkpointing
    JobSnapshottingSettings snapshotSettings = jobGraph.getSnapshotSettings();
    if (snapshotSettings != null) {
        List<ExecutionJobVertex> triggerVertices = idToVertex(snapshotSettings.getVerticesToTrigger(), executionGraph);
        List<ExecutionJobVertex> ackVertices = idToVertex(snapshotSettings.getVerticesToAcknowledge(), executionGraph);
        List<ExecutionJobVertex> confirmVertices = idToVertex(snapshotSettings.getVerticesToConfirm(), executionGraph);
        CompletedCheckpointStore completedCheckpoints;
        CheckpointIDCounter checkpointIdCounter;
        try {
            int maxNumberOfCheckpointsToRetain = jobManagerConfig.getInteger(CoreOptions.MAX_RETAINED_CHECKPOINTS);
            if (maxNumberOfCheckpointsToRetain <= 0) {
                // warning and use 1 as the default value if the setting in
                // state.checkpoints.max-retained-checkpoints is not greater than 0.
                log.warn("The setting for '{} : {}' is invalid. Using default value of {}", CoreOptions.MAX_RETAINED_CHECKPOINTS.key(), maxNumberOfCheckpointsToRetain, CoreOptions.MAX_RETAINED_CHECKPOINTS.defaultValue());
                maxNumberOfCheckpointsToRetain = CoreOptions.MAX_RETAINED_CHECKPOINTS.defaultValue();
            }
            completedCheckpoints = recoveryFactory.createCheckpointStore(jobId, maxNumberOfCheckpointsToRetain, classLoader);
            checkpointIdCounter = recoveryFactory.createCheckpointIDCounter(jobId);
        } catch (Exception e) {
            throw new JobExecutionException(jobId, "Failed to initialize high-availability checkpoint handler", e);
        }
        // Maximum number of remembered checkpoints
        int historySize = jobManagerConfig.getInteger(ConfigConstants.JOB_MANAGER_WEB_CHECKPOINTS_HISTORY_SIZE, ConfigConstants.DEFAULT_JOB_MANAGER_WEB_CHECKPOINTS_HISTORY_SIZE);
        CheckpointStatsTracker checkpointStatsTracker = new CheckpointStatsTracker(historySize, ackVertices, snapshotSettings, metrics);
        // The default directory for externalized checkpoints
        String externalizedCheckpointsDir = jobManagerConfig.getString(ConfigConstants.CHECKPOINTS_DIRECTORY_KEY, null);
        // load the state backend for checkpoint metadata.
        // if specified in the application, use from there, otherwise load from configuration
        final StateBackend metadataBackend;
        final StateBackend applicationConfiguredBackend = snapshotSettings.getDefaultStateBackend();
        if (applicationConfiguredBackend != null) {
            metadataBackend = applicationConfiguredBackend;
            log.info("Using application-defined state backend for checkpoint/savepoint metadata: {}.", applicationConfiguredBackend);
        } else {
            try {
                metadataBackend = AbstractStateBackend.loadStateBackendFromConfigOrCreateDefault(jobManagerConfig, classLoader, log);
            } catch (IllegalConfigurationException | IOException | DynamicCodeLoadingException e) {
                throw new JobExecutionException(jobId, "Could not instantiate configured state backend", e);
            }
        }
        executionGraph.enableCheckpointing(snapshotSettings.getCheckpointInterval(), snapshotSettings.getCheckpointTimeout(), snapshotSettings.getMinPauseBetweenCheckpoints(), snapshotSettings.getMaxConcurrentCheckpoints(), snapshotSettings.getExternalizedCheckpointSettings(), triggerVertices, ackVertices, confirmVertices, checkpointIdCounter, completedCheckpoints, externalizedCheckpointsDir, metadataBackend, checkpointStatsTracker);
    }
    return executionGraph;
}

Example 44 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class JobManagerHAJobGraphRecoveryITCase method createBlockingJobGraph.

// ---------------------------------------------------------------------------------------------
/**
	 * Creates a simple blocking JobGraph.
	 */
private static JobGraph createBlockingJobGraph() {
    JobGraph jobGraph = new JobGraph("Blocking program");
    JobVertex jobVertex = new JobVertex("Blocking Vertex");
    jobVertex.setInvokableClass(BlockingNoOpInvokable.class);
    jobGraph.addVertex(jobVertex);
    return jobGraph;
}

Example 45 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class WebFrontendITCase method testStopYarn.

@Test
public void testStopYarn() throws Exception {
    // this only works if there is no active job at this point
    assertTrue(cluster.getCurrentlyRunningJobsJava().isEmpty());
    // Create a task
    final JobVertex sender = new JobVertex("Sender");
    sender.setParallelism(2);
    sender.setInvokableClass(StoppableInvokable.class);
    final JobGraph jobGraph = new JobGraph("Stoppable streaming test job", sender);
    final JobID jid = jobGraph.getJobID();
    cluster.submitJobDetached(jobGraph);
    // wait for job to show up
    while (cluster.getCurrentlyRunningJobsJava().isEmpty()) {
        Thread.sleep(10);
    }
    final FiniteDuration testTimeout = new FiniteDuration(2, TimeUnit.MINUTES);
    final Deadline deadline = testTimeout.fromNow();
    while (!cluster.getCurrentlyRunningJobsJava().isEmpty()) {
        try (HttpTestClient client = new HttpTestClient("localhost", port)) {
            // Request the file from the web server
            client.sendGetRequest("/jobs/" + jid + "/yarn-stop", deadline.timeLeft());
            HttpTestClient.SimpleHttpResponse response = client.getNextResponse(deadline.timeLeft());
            assertEquals(HttpResponseStatus.OK, response.getStatus());
            assertEquals(response.getType(), MimeTypes.getMimeTypeForExtension("json"));
            assertEquals("{}", response.getContent());
        }
        Thread.sleep(20);
    }
}