Examples with MapConfig org.apache.samza.config.MapConfig used on opensource projects

Example 76 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestTasksResource method configure.

@Override
protected Application configure() {
    Map<String, String> configMap = ImmutableMap.of(TaskResourceConfig.CONFIG_TASK_PROXY_FACTORY, MockTaskProxyFactory.class.getName(), SamzaRestConfig.CONFIG_REST_RESOURCE_FACTORIES, MockResourceFactory.class.getName());
    SamzaRestConfig config = new SamzaRestConfig(new MapConfig(configMap));
    return new SamzaRestApplication(config);
}

Example 77 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestJobsResource method configure.

@Override
protected Application configure() {
    Map<String, String> configMap = ImmutableMap.of(JobsResourceConfig.CONFIG_JOB_PROXY_FACTORY, MockJobProxyFactory.class.getName(), SamzaRestConfig.CONFIG_REST_RESOURCE_FACTORIES, MockResourceFactory.class.getName());
    SamzaRestConfig config = new SamzaRestConfig(new MapConfig(configMap));
    return new SamzaRestApplication(config);
}

Example 78 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class JobNode method extractScopedConfig.

/**
   * This function extract the subset of configs from the full config, and use it to override the generated configs
   * from the job.
   * @param fullConfig full config
   * @param generatedConfig config generated for the job
   * @param configPrefix prefix to extract the subset of the config overrides
   * @return config that merges the generated configs and overrides
   */
private static Config extractScopedConfig(Config fullConfig, Config generatedConfig, String configPrefix) {
    Config scopedConfig = fullConfig.subset(configPrefix);
    Config[] configPrecedence = new Config[] { fullConfig, generatedConfig, scopedConfig };
    // Strip empty configs so they don't override the configs before them.
    Map<String, String> mergedConfig = new HashMap<>();
    for (Map<String, String> config : configPrecedence) {
        for (Map.Entry<String, String> property : config.entrySet()) {
            String value = property.getValue();
            if (!(value == null || value.isEmpty())) {
                mergedConfig.put(property.getKey(), property.getValue());
            }
        }
    }
    scopedConfig = new MapConfig(mergedConfig);
    log.debug("Prefix '{}' has merged config {}", configPrefix, scopedConfig);
    return scopedConfig;
}

Example 79 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class JobNode method generateConfig.

/**
   * Generate the configs for a job
   * @param executionPlanJson JSON representation of the execution plan
   * @return config of the job
   */
public JobConfig generateConfig(String executionPlanJson) {
    Map<String, String> configs = new HashMap<>();
    configs.put(JobConfig.JOB_NAME(), jobName);
    List<String> inputs = inEdges.stream().map(edge -> edge.getFormattedSystemStream()).collect(Collectors.toList());
    configs.put(TaskConfig.INPUT_STREAMS(), Joiner.on(',').join(inputs));
    // set triggering interval if a window or join is defined
    if (streamGraph.hasWindowOrJoins()) {
        if ("-1".equals(config.get(TaskConfig.WINDOW_MS(), "-1"))) {
            long triggerInterval = computeTriggerInterval();
            log.info("Using triggering interval: {} for jobName: {}", triggerInterval, jobName);
            configs.put(TaskConfig.WINDOW_MS(), String.valueOf(triggerInterval));
        }
    }
    log.info("Job {} has generated configs {}", jobName, configs);
    configs.put(CONFIG_INTERNAL_EXECUTION_PLAN, executionPlanJson);
    String configPrefix = String.format(CONFIG_JOB_PREFIX, jobName);
    // TODO: Disallow user specifying job inputs/outputs. This info comes strictly from the pipeline.
    return new JobConfig(Util.rewriteConfig(extractScopedConfig(config, new MapConfig(configs), configPrefix)));
}

Example 80 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class StateStorageTool method main.

public static void main(String[] args) {
    StateStorageTool tool = new StateStorageTool();
    OptionSet options = tool.parser().parse(args);
    MapConfig config = tool.loadConfig(options);
    String path = tool.getPath();
    StorageRecovery storageRecovery = new StorageRecovery(config, path);
    storageRecovery.run();
}