Examples with JobExecutionPlanDagFactory org.apache.gobblin.service.modules.spec.JobExecutionPlanDagFactory used on opensource projects

Example 6 with JobExecutionPlanDagFactory

use of org.apache.gobblin.service.modules.spec.JobExecutionPlanDagFactory in project incubator-gobblin by apache.

the class FlowGraphPath method convertHopToDag.

/**
 * Given an instance of {@link FlowEdge}, this method returns a {@link Dag < JobExecutionPlan >} that moves data
 * from the source of the {@link FlowEdge} to the destination of the {@link FlowEdge}.
 * @param flowEdgeContext an instance of {@link FlowEdgeContext}.
 * @param sysConfig environment config.
 * @return a {@link Dag} of {@link JobExecutionPlan}s associated with the {@link FlowEdge}.
 */
private Dag<JobExecutionPlan> convertHopToDag(FlowEdgeContext flowEdgeContext, Config sysConfig) throws SpecNotFoundException, JobTemplate.TemplateException, URISyntaxException {
    FlowTemplate flowTemplate = flowEdgeContext.getEdge().getFlowTemplate();
    DatasetDescriptor inputDatasetDescriptor = flowEdgeContext.getInputDatasetDescriptor();
    DatasetDescriptor outputDatasetDescriptor = flowEdgeContext.getOutputDatasetDescriptor();
    Config mergedConfig = flowEdgeContext.getMergedConfig();
    SpecExecutor specExecutor = flowEdgeContext.getSpecExecutor();
    // Get resolved job configs from the flow template
    List<Config> resolvedJobConfigs = flowTemplate.getResolvedJobConfigs(mergedConfig, inputDatasetDescriptor, outputDatasetDescriptor);
    List<JobExecutionPlan> jobExecutionPlans = new ArrayList<>(resolvedJobConfigs.size());
    Map<String, String> templateToJobNameMap = Maps.newHashMapWithExpectedSize(resolvedJobConfigs.size());
    // Iterate over each resolved job config and convert the config to a JobSpec.
    for (Config resolvedJobConfig : resolvedJobConfigs) {
        JobExecutionPlan jobExecutionPlan = new JobExecutionPlan.Factory().createPlan(flowSpec, resolvedJobConfig, specExecutor, flowExecutionId, sysConfig);
        jobExecutionPlans.add(jobExecutionPlan);
        templateToJobNameMap.put(getJobTemplateName(jobExecutionPlan), jobExecutionPlan.getJobSpec().getConfig().getString(ConfigurationKeys.JOB_NAME_KEY));
    }
    updateJobDependencies(jobExecutionPlans, templateToJobNameMap);
    return new JobExecutionPlanDagFactory().createDag(jobExecutionPlans);
}

Example 7 with JobExecutionPlanDagFactory

use of org.apache.gobblin.service.modules.spec.JobExecutionPlanDagFactory in project incubator-gobblin by apache.

the class MockedSpecCompiler method compileFlow.

@Override
public Dag<JobExecutionPlan> compileFlow(Spec spec) {
    String flowName = (String) ((FlowSpec) spec).getConfigAsProperties().get(ConfigurationKeys.FLOW_NAME_KEY);
    if (flowName.equalsIgnoreCase(UNCOMPILABLE_FLOW)) {
        return null;
    }
    List<JobExecutionPlan> jobExecutionPlans = new ArrayList<>();
    long flowExecutionId = System.currentTimeMillis();
    int i = 0;
    while (i++ < NUMBER_OF_JOBS) {
        String specUri = "/foo/bar/spec/" + i;
        Properties properties = new Properties();
        properties.put(ConfigurationKeys.FLOW_NAME_KEY, flowName);
        properties.put(ConfigurationKeys.FLOW_GROUP_KEY, ((FlowSpec) spec).getConfigAsProperties().get(ConfigurationKeys.FLOW_GROUP_KEY));
        properties.put(ConfigurationKeys.JOB_NAME_KEY, ((FlowSpec) spec).getConfigAsProperties().get(ConfigurationKeys.FLOW_NAME_KEY) + "_" + i);
        properties.put(ConfigurationKeys.JOB_GROUP_KEY, ((FlowSpec) spec).getConfigAsProperties().get(ConfigurationKeys.FLOW_GROUP_KEY) + "_" + i);
        properties.put(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, flowExecutionId);
        JobSpec jobSpec = JobSpec.builder(specUri).withConfig(ConfigUtils.propertiesToConfig(properties)).withVersion("1").withDescription("Spec Description").build();
        jobExecutionPlans.add(new JobExecutionPlan(jobSpec, new InMemorySpecExecutor(ConfigFactory.empty())));
    }
    return new JobExecutionPlanDagFactory().createDag(jobExecutionPlans);
}