Examples with Config org.apache.samza.config.Config used on opensource projects

Example 1 with Config

use of org.apache.samza.config.Config in project samza by apache.

the class ConfigManager method main.

/**
   * Main function for using the Config Manager. The main function starts a Config Manager, and reacts to all messages thereafter
   * In order for this module to run, you have to add the following configurations to the config file:
   * yarn.rm.address=localhost //the ip of the resource manager in yarn
   * yarn.rm.port=8088 //the port of the resource manager http server
   * Additionally, the config manger will periodically poll the coordinator stream to see if there are any new messages.
   * This period is set to 100 ms by default. However, it can be configured by adding the following property to the input config file.
   * configManager.polling.interval= < polling interval >
   * To run the code use the following command:
   * {path to samza deployment}/samza/bin/run-config-manager.sh  --config-factory={config-factory} --config-path={path to config file of a job}
   *
   * @param args input arguments for running ConfigManager.
   */
public static void main(String[] args) {
    CommandLine cmdline = new CommandLine();
    OptionSet options = cmdline.parser().parse(args);
    Config config = cmdline.loadConfig(options);
    ConfigManager configManager = new ConfigManager(config);
    configManager.run();
}

Example 2 with Config

use of org.apache.samza.config.Config in project samza by apache.

the class ClusterBasedJobCoordinator method main.

/**
   * The entry point for the {@link ClusterBasedJobCoordinator}
   * @param args args
   */
public static void main(String[] args) {
    Config coordinatorSystemConfig = null;
    final String coordinatorSystemEnv = System.getenv(ShellCommandConfig.ENV_COORDINATOR_SYSTEM_CONFIG());
    try {
        //Read and parse the coordinator system config.
        log.info("Parsing coordinator system config {}", coordinatorSystemEnv);
        coordinatorSystemConfig = new MapConfig(SamzaObjectMapper.getObjectMapper().readValue(coordinatorSystemEnv, Config.class));
    } catch (IOException e) {
        log.error("Exception while reading coordinator stream config {}", e);
        throw new SamzaException(e);
    }
    ClusterBasedJobCoordinator jc = new ClusterBasedJobCoordinator(coordinatorSystemConfig);
    jc.run();
}

Example 3 with Config

use of org.apache.samza.config.Config in project samza by apache.

the class TestExecutionPlanner method testTriggerIntervalWithInvalidWindowMs.

@Test
public void testTriggerIntervalWithInvalidWindowMs() throws Exception {
    Map<String, String> map = new HashMap<>(config);
    map.put(TaskConfig.WINDOW_MS(), "-1");
    map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS(), String.valueOf(DEFAULT_PARTITIONS));
    Config cfg = new MapConfig(map);
    ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager);
    StreamGraphImpl streamGraph = createStreamGraphWithJoinAndWindow();
    ExecutionPlan plan = planner.plan(streamGraph);
    List<JobConfig> jobConfigs = plan.getJobConfigs();
    assertEquals(jobConfigs.size(), 1);
    // GCD of 8, 16, 1600 and 252 is 4
    assertEquals(jobConfigs.get(0).get(TaskConfig.WINDOW_MS()), "4");
}

Example 4 with Config

use of org.apache.samza.config.Config in project samza by apache.

the class TestExecutionPlanner method createSimpleGraph.

private StreamGraphImpl createSimpleGraph() {
    /**
     * a simple graph of partitionBy and map
     *
     * input1 -> partitionBy -> map -> output1
     *
     */
    StreamGraphImpl streamGraph = new StreamGraphImpl(runner, config);
    Function mockFn = mock(Function.class);
    OutputStream<Object, Object, Object> output1 = streamGraph.getOutputStream("output1", mockFn, mockFn);
    BiFunction mockBuilder = mock(BiFunction.class);
    streamGraph.getInputStream("input1", mockBuilder).partitionBy(m -> "yes!!!").map(m -> m).sendTo(output1);
    return streamGraph;
}

Example 5 with Config

use of org.apache.samza.config.Config in project samza by apache.

the class TestContainerProcessManager method testAppMasterWithFwk.

@Test
public void testAppMasterWithFwk() {
    ContainerProcessManager taskManager = new ContainerProcessManager(new MapConfig(config), state, new MetricsRegistryMap(), manager);
    taskManager.start();
    SamzaResource container2 = new SamzaResource(1, 1024, "", "id0");
    assertFalse(taskManager.shouldShutdown());
    taskManager.onResourceAllocated(container2);
    configVals.put(JobConfig.SAMZA_FWK_PATH(), "/export/content/whatever");
    Config config1 = new MapConfig(configVals);
    ContainerProcessManager taskManager1 = new ContainerProcessManager(new MapConfig(config), state, new MetricsRegistryMap(), manager);
    taskManager1.start();
    taskManager1.onResourceAllocated(container2);
}