Examples with JobConfig org.apache.samza.config.JobConfig used on opensource projects

Example 11 with JobConfig

use of org.apache.samza.config.JobConfig in project samza by apache.

the class TestExecutionPlanner method testTriggerIntervalForJoins.

@Test
public void testTriggerIntervalForJoins() throws Exception {
    Map<String, String> map = new HashMap<>(config);
    map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS(), String.valueOf(DEFAULT_PARTITIONS));
    Config cfg = new MapConfig(map);
    ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager);
    StreamGraphImpl streamGraph = createStreamGraphWithJoin();
    ExecutionPlan plan = planner.plan(streamGraph);
    List<JobConfig> jobConfigs = plan.getJobConfigs();
    for (JobConfig config : jobConfigs) {
        System.out.println(config);
    }
}

Example 12 with JobConfig

use of org.apache.samza.config.JobConfig in project samza by apache.

the class JobNode method generateConfig.

/**
   * Generate the configs for a job
   * @param executionPlanJson JSON representation of the execution plan
   * @return config of the job
   */
public JobConfig generateConfig(String executionPlanJson) {
    Map<String, String> configs = new HashMap<>();
    configs.put(JobConfig.JOB_NAME(), jobName);
    List<String> inputs = inEdges.stream().map(edge -> edge.getFormattedSystemStream()).collect(Collectors.toList());
    configs.put(TaskConfig.INPUT_STREAMS(), Joiner.on(',').join(inputs));
    // set triggering interval if a window or join is defined
    if (streamGraph.hasWindowOrJoins()) {
        if ("-1".equals(config.get(TaskConfig.WINDOW_MS(), "-1"))) {
            long triggerInterval = computeTriggerInterval();
            log.info("Using triggering interval: {} for jobName: {}", triggerInterval, jobName);
            configs.put(TaskConfig.WINDOW_MS(), String.valueOf(triggerInterval));
        }
    }
    log.info("Job {} has generated configs {}", jobName, configs);
    configs.put(CONFIG_INTERNAL_EXECUTION_PLAN, executionPlanJson);
    String configPrefix = String.format(CONFIG_JOB_PREFIX, jobName);
    // TODO: Disallow user specifying job inputs/outputs. This info comes strictly from the pipeline.
    return new JobConfig(Util.rewriteConfig(extractScopedConfig(config, new MapConfig(configs), configPrefix)));
}

Example 13 with JobConfig

use of org.apache.samza.config.JobConfig in project samza by apache.

the class LocalContainerRunner method main.

public static void main(String[] args) throws Exception {
    Thread.setDefaultUncaughtExceptionHandler(new SamzaContainerExceptionHandler(() -> {
        log.info("Exiting process now.");
        System.exit(1);
    }));
    String containerId = System.getenv(ShellCommandConfig.ENV_CONTAINER_ID());
    log.info(String.format("Got container ID: %s", containerId));
    String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL());
    log.info(String.format("Got coordinator URL: %s", coordinatorUrl));
    int delay = new Random().nextInt(SamzaContainer.DEFAULT_READ_JOBMODEL_DELAY_MS()) + 1;
    JobModel jobModel = SamzaContainer.readJobModel(coordinatorUrl, delay);
    Config config = jobModel.getConfig();
    JobConfig jobConfig = new JobConfig(config);
    if (jobConfig.getName().isEmpty()) {
        throw new SamzaException("can not find the job name");
    }
    String jobName = jobConfig.getName().get();
    String jobId = jobConfig.getJobId().getOrElse(ScalaToJavaUtils.defaultValue("1"));
    MDC.put("containerName", "samza-container-" + containerId);
    MDC.put("jobName", jobName);
    MDC.put("jobId", jobId);
    StreamApplication streamApp = TaskFactoryUtil.createStreamApplication(config);
    LocalContainerRunner localContainerRunner = new LocalContainerRunner(jobModel, containerId);
    localContainerRunner.run(streamApp);
}