Examples with JobExecutionPlanDagFactory org.apache.gobblin.service.modules.spec.JobExecutionPlanDagFactory used on opensource projects

Example 1 with JobExecutionPlanDagFactory

use of org.apache.gobblin.service.modules.spec.JobExecutionPlanDagFactory in project incubator-gobblin by apache.

the class DagTestUtils method buildDag.

/**
 * Create a {@link Dag < JobExecutionPlan >} with one parent and one child.
 * @return a Dag.
 */
public static Dag<JobExecutionPlan> buildDag(String id, Long flowExecutionId) throws URISyntaxException {
    List<JobExecutionPlan> jobExecutionPlans = new ArrayList<>();
    for (int i = 0; i < 2; i++) {
        String suffix = Integer.toString(i);
        Config jobConfig = ConfigBuilder.create().addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, "group" + id).addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, "flow" + id).addPrimitive(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, flowExecutionId).addPrimitive(ConfigurationKeys.JOB_NAME_KEY, "job" + suffix).build();
        if (i > 0) {
            jobConfig = jobConfig.withValue(ConfigurationKeys.JOB_DEPENDENCIES, ConfigValueFactory.fromAnyRef("job" + (i - 1)));
        }
        JobSpec js = JobSpec.builder("test_job" + suffix).withVersion(suffix).withConfig(jobConfig).withTemplate(new URI("job" + suffix)).build();
        SpecExecutor specExecutor = buildNaiveTopologySpec("mySpecExecutor").getSpecExecutor();
        JobExecutionPlan jobExecutionPlan = new JobExecutionPlan(js, specExecutor);
        jobExecutionPlan.setExecutionStatus(ExecutionStatus.RUNNING);
        // Future of type CompletedFuture is used because in tests InMemorySpecProducer is used and that responds with CompletedFuture
        CompletedFuture future = new CompletedFuture<>(Boolean.TRUE, null);
        jobExecutionPlan.setJobFuture(Optional.of(future));
        jobExecutionPlans.add(jobExecutionPlan);
    }
    return new JobExecutionPlanDagFactory().createDag(jobExecutionPlans);
}

Example 2 with JobExecutionPlanDagFactory

use of org.apache.gobblin.service.modules.spec.JobExecutionPlanDagFactory in project incubator-gobblin by apache.

the class FlowGraphPathTest method buildDag.

/**
 * A method to create a {@link Dag <JobExecutionPlan>}.
 * @return a Dag.
 */
public Dag<JobExecutionPlan> buildDag(int numNodes, int startNodeId, boolean isForkable) throws URISyntaxException {
    List<JobExecutionPlan> jobExecutionPlans = new ArrayList<>();
    Config baseConfig = ConfigBuilder.create().addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, "group0").addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, "flow0").addPrimitive(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, System.currentTimeMillis()).addPrimitive(ConfigurationKeys.JOB_GROUP_KEY, "group0").build();
    for (int i = startNodeId; i < startNodeId + numNodes; i++) {
        String suffix = Integer.toString(i);
        Config jobConfig = baseConfig.withValue(ConfigurationKeys.JOB_NAME_KEY, ConfigValueFactory.fromAnyRef("job" + suffix));
        if (isForkable && (i == startNodeId + numNodes - 1)) {
            jobConfig = jobConfig.withValue(ConfigurationKeys.JOB_FORK_ON_CONCAT, ConfigValueFactory.fromAnyRef(true));
        }
        if (i > startNodeId) {
            jobConfig = jobConfig.withValue(ConfigurationKeys.JOB_DEPENDENCIES, ConfigValueFactory.fromAnyRef("job" + (i - 1)));
        }
        JobSpec js = JobSpec.builder("test_job" + suffix).withVersion(suffix).withConfig(jobConfig).withTemplate(new URI("job" + suffix)).build();
        SpecExecutor specExecutor = InMemorySpecExecutor.createDummySpecExecutor(new URI("job" + i));
        JobExecutionPlan jobExecutionPlan = new JobExecutionPlan(js, specExecutor);
        jobExecutionPlans.add(jobExecutionPlan);
    }
    return new JobExecutionPlanDagFactory().createDag(jobExecutionPlans);
}

Example 3 with JobExecutionPlanDagFactory

use of org.apache.gobblin.service.modules.spec.JobExecutionPlanDagFactory in project incubator-gobblin by apache.

the class IdentityFlowToJobSpecCompiler method compileFlow.

@Override
public Dag<JobExecutionPlan> compileFlow(Spec spec) {
    Preconditions.checkNotNull(spec);
    Preconditions.checkArgument(spec instanceof FlowSpec, "IdentityFlowToJobSpecCompiler only converts FlowSpec to JobSpec");
    long startTime = System.nanoTime();
    FlowSpec flowSpec = (FlowSpec) spec;
    String source = flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY);
    String destination = flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY);
    log.info(String.format("Compiling flow for source: %s and destination: %s", source, destination));
    JobSpec jobSpec = jobSpecGenerator(flowSpec);
    Instrumented.markMeter(this.flowCompilationSuccessFulMeter);
    Instrumented.updateTimer(this.flowCompilationTimer, System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
    List<JobExecutionPlan> jobExecutionPlans;
    try {
        jobExecutionPlans = getJobExecutionPlans(source, destination, jobSpec);
    } catch (InterruptedException | ExecutionException e) {
        Instrumented.markMeter(this.flowCompilationFailedMeter);
        throw new RuntimeException("Cannot determine topology capabilities", e);
    }
    return new JobExecutionPlanDagFactory().createDag(jobExecutionPlans);
}

Example 4 with JobExecutionPlanDagFactory

use of org.apache.gobblin.service.modules.spec.JobExecutionPlanDagFactory in project incubator-gobblin by apache.

the class DagManagerTest method buildDag.

static Dag<JobExecutionPlan> buildDag(String id, Long flowExecutionId, String flowFailureOption, int numNodes) throws URISyntaxException {
    List<JobExecutionPlan> jobExecutionPlans = new ArrayList<>();
    for (int i = 0; i < numNodes; i++) {
        String suffix = Integer.toString(i);
        Config jobConfig = ConfigBuilder.create().addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, "group" + id).addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, "flow" + id).addPrimitive(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, flowExecutionId).addPrimitive(ConfigurationKeys.JOB_GROUP_KEY, "group" + id).addPrimitive(ConfigurationKeys.JOB_NAME_KEY, "job" + suffix).addPrimitive(ConfigurationKeys.FLOW_FAILURE_OPTION, flowFailureOption).build();
        if ((i == 1) || (i == 2)) {
            jobConfig = jobConfig.withValue(ConfigurationKeys.JOB_DEPENDENCIES, ConfigValueFactory.fromAnyRef("job0"));
        } else if (i == 3) {
            jobConfig = jobConfig.withValue(ConfigurationKeys.JOB_DEPENDENCIES, ConfigValueFactory.fromAnyRef("job1"));
        } else if (i == 4) {
            jobConfig = jobConfig.withValue(ConfigurationKeys.JOB_DEPENDENCIES, ConfigValueFactory.fromAnyRef("job2"));
        }
        JobSpec js = JobSpec.builder("test_job" + suffix).withVersion(suffix).withConfig(jobConfig).withTemplate(new URI("job" + suffix)).build();
        SpecExecutor specExecutor = MockedSpecExecutor.createDummySpecExecutor(new URI("job" + i));
        JobExecutionPlan jobExecutionPlan = new JobExecutionPlan(js, specExecutor);
        jobExecutionPlans.add(jobExecutionPlan);
    }
    return new JobExecutionPlanDagFactory().createDag(jobExecutionPlans);
}

Example 5 with JobExecutionPlanDagFactory

use of org.apache.gobblin.service.modules.spec.JobExecutionPlanDagFactory in project incubator-gobblin by apache.

the class DagManagerTest method testDagManagerWithBadFlowSLAConfig.

@Test(dependsOnMethods = "testJobStartSLAKilledDag")
public void testDagManagerWithBadFlowSLAConfig() throws URISyntaxException, IOException {
    long flowExecutionId = System.currentTimeMillis();
    String flowGroup = "group0";
    String flowName = "flow0";
    String jobName = "job0";
    // Create a config with an improperly formatted Flow SLA time e.g. "1h"
    Config jobConfig = ConfigBuilder.create().addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, "group" + flowGroup).addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, "flow" + flowName).addPrimitive(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, flowExecutionId).addPrimitive(ConfigurationKeys.JOB_GROUP_KEY, flowGroup).addPrimitive(ConfigurationKeys.JOB_NAME_KEY, jobName).addPrimitive(ConfigurationKeys.FLOW_FAILURE_OPTION, "FINISH_RUNNING").addPrimitive(ConfigurationKeys.GOBBLIN_FLOW_SLA_TIME, "1h").build();
    List<JobExecutionPlan> jobExecutionPlans = new ArrayList<>();
    JobSpec js = JobSpec.builder("test_job" + jobName).withVersion("0").withConfig(jobConfig).withTemplate(new URI(jobName)).build();
    SpecExecutor specExecutor = MockedSpecExecutor.createDummySpecExecutor(new URI(jobName));
    JobExecutionPlan jobExecutionPlan = new JobExecutionPlan(js, specExecutor);
    jobExecutionPlans.add(jobExecutionPlan);
    Dag<JobExecutionPlan> dag = (new JobExecutionPlanDagFactory()).createDag(jobExecutionPlans);
    String dagId = DagManagerUtils.generateDagId(dag);
    // Add a dag to the queue of dags
    this.queue.offer(dag);
    // Job should have been run normally without breaking on SLA check, so we can just mark as completed for status
    Iterator<JobStatus> jobStatusIterator1 = getMockJobStatus(flowName, flowGroup, flowExecutionId + 1, jobName, flowGroup, String.valueOf(ExecutionStatus.COMPLETE));
    Mockito.when(_jobStatusRetriever.getJobStatusesForFlowExecution(Mockito.anyString(), Mockito.anyString(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString())).thenReturn(jobStatusIterator1);
    // Run the thread once. Job should run without crashing thread on SLA check and cleanup
    this._dagManagerThread.run();
    Assert.assertEquals(this.dags.size(), 0);
    Assert.assertEquals(this.jobToDag.size(), 0);
    Assert.assertEquals(this.dagToJobs.size(), 0);
}