Examples with ConfigLoader com.linkedin.databus.core.util.ConfigLoader used on opensource projects

Example 11 with ConfigLoader

use of com.linkedin.databus.core.util.ConfigLoader in project databus by linkedin.

the class DbusKeyCompositeFilterConfig method main.

public static void main(String[] args) throws Exception {
    String configFile = "/Users/bvaradar/Documents/workspace/BR_DDS_SERVER_SIDE_FILTERING/integration-test/config/server-filter-test.properties";
    Properties startupProps = new Properties();
    startupProps.load(new FileInputStream(configFile));
    Config staticConfigBuilder = new Config();
    ConfigLoader<StaticConfig> staticConfigLoader = new ConfigLoader<StaticConfig>("serversidefilter.", staticConfigBuilder);
    StaticConfig staticConfig = staticConfigLoader.loadConfig(startupProps);
    DbusKeyCompositeFilterConfig filterConf = new DbusKeyCompositeFilterConfig(staticConfig);
    System.out.println("FilterConf :" + filterConf);
}

Example 12 with ConfigLoader

use of com.linkedin.databus.core.util.ConfigLoader in project databus by linkedin.

the class KeyFilterConfigHolder method main.

public static void main(String[] args) throws Exception {
    Properties props = new Properties();
    props.setProperty("dummy.type", "MOD");
    props.setProperty("dummy.mod.numBuckets", "10");
    props.setProperty("dummy.mod.buckets", "[1,2,4-7]");
    Config config = new Config();
    ConfigLoader<StaticConfig> configLoader = new ConfigLoader<StaticConfig>("dummy.", config);
    StaticConfig staticConfig = configLoader.loadConfig(props);
    KeyFilterConfigHolder holder = new KeyFilterConfigHolder(staticConfig);
    System.out.println("Holder is :" + holder.toString());
    ObjectMapper mapper = new ObjectMapper();
    String s = mapper.writeValueAsString(holder);
    System.out.println("Config Holder JSON:" + s);
    JSONObject obj = new JSONObject(s);
    String type = obj.getString("partitionType");
    String configStr = obj.getString("filterConfig");
    KeyFilterConfigHolder configHolder = new KeyFilterConfigHolder();
    configHolder.setPartitionType(PartitionType.valueOf(type));
    KeyFilterConfig conf = null;
    if (PartitionType.MOD == configHolder.getPartitionType()) {
        conf = mapper.readValue(configStr, KeyModFilterConfig.class);
    } else if (PartitionType.RANGE == configHolder.getPartitionType()) {
        conf = mapper.readValue(configStr, KeyRangeFilterConfig.class);
    }
    configHolder.setFilterConfig(conf);
    System.out.println("Holder2 is :" + configHolder.toString());
}

Example 13 with ConfigLoader

use of com.linkedin.databus.core.util.ConfigLoader in project databus by linkedin.

the class ClusterFileLoggingClient method mainFunction.

public void mainFunction(String[] args) throws Exception {
    String[] leftOverArgs = processLocalArgs(args);
    Properties startupProps = DatabusHttpClientImpl.processCommandLineArgs(leftOverArgs);
    DatabusHttpClientImpl.Config clientConfigBuilder = new DatabusHttpClientImpl.Config();
    clientConfigBuilder.getContainer().setIdFromName(MODULE + ".localhost");
    if (_enableBootStrap) {
        clientConfigBuilder.getRuntime().getBootstrap().setEnabled(true);
    }
    ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>("databus.client.", clientConfigBuilder);
    String[] sources = getSources();
    StringBuilder sourcesString = new StringBuilder();
    boolean firstSrc = true;
    for (String source : sources) {
        if (!firstSrc)
            sourcesString.append(",");
        firstSrc = false;
        sourcesString.append(source);
    }
    if (_httpPort != null) {
        startupProps.put("databus.client.container.httpPort", _httpPort);
    }
    if (_jmxServicePort != null) {
        startupProps.put("databus.client.container.jmx.jmxServicePort", _jmxServicePort);
    }
    if (_checkpointFileRootDir != null) {
        startupProps.put("databus.client.checkpointPersistence.fileSystem.rootDirectory", _checkpointFileRootDir);
    }
    DatabusHttpClientImpl.StaticConfig clientConfig = configLoader.loadConfig(startupProps);
    // set up relay
    ServerInfoBuilder relayBuilder = clientConfig.getRuntime().getRelay("1");
    relayBuilder.setName("DefaultRelay");
    if (_relayHost != null) {
        relayBuilder.setHost(_relayHost);
    }
    if (_relayPort != null) {
        relayBuilder.setPort(Integer.parseInt(_relayPort));
    }
    relayBuilder.setSources(sourcesString.toString());
    // set up bootstrap
    if (_enableBootStrap) {
        ServerInfoBuilder bootstrapBuilder = clientConfig.getRuntime().getBootstrap().getService("2");
        bootstrapBuilder.setName("DefaultBootstrapServices");
        if (_bootstrapHost != null) {
            bootstrapBuilder.setHost(_bootstrapHost);
        }
        if (_bootstrapPort != null) {
            bootstrapBuilder.setPort(Integer.parseInt(_bootstrapPort));
        }
        bootstrapBuilder.setSources(sourcesString.toString());
    }
    // set up listeners
    DatabusHttpClientImpl client = new DatabusHttpClientImpl(clientConfig);
    List<DatabusRegistration> regs = new ArrayList<DatabusRegistration>();
    for (String cluster : _clusters) {
        DatabusRegistration reg = client.registerCluster(cluster, createConsumerFactory(cluster, _valueDumpFile, _eventDumpFile), createServerSideFactory(cluster), createPartitionListener(cluster), sources);
        regs.add(reg);
    }
    // add pause processor
    try {
        client.getProcessorRegistry().register(ContainerOperationProcessor.COMMAND_NAME, new ContainerOperationProcessor(null, client));
    } catch (ProcessorRegistrationConflictException e) {
        LOG.error("Failed to register " + ConsumerPauseRequestProcessor.COMMAND_NAME);
    }
    DatabusClientShutdownThread shutdownThread = new DatabusClientShutdownThread(client);
    Runtime.getRuntime().addShutdownHook(shutdownThread);
    client.startAndBlock();
}

Example 14 with ConfigLoader

use of com.linkedin.databus.core.util.ConfigLoader in project databus by linkedin.

the class SimpleFileLoggingConsumer method createServerSideFilterConfig.

private DbusKeyCompositeFilterConfig createServerSideFilterConfig(String filterConfFile, Properties startupProps) throws IOException, InvalidConfigException {
    Boolean cmdLineHasFilterConfig = hasServerSideFilterConfig(startupProps);
    if (filterConfFile != null || cmdLineHasFilterConfig) {
        Properties props;
        // cmdline override the config file
        if (cmdLineHasFilterConfig) {
            props = startupProps;
        } else {
            LOG.info("filterConfFile = " + filterConfFile);
            props = new Properties();
            FileInputStream filterConfStream = new FileInputStream(filterConfFile);
            try {
                props.load(filterConfStream);
            } finally {
                filterConfStream.close();
            }
        }
        DbusKeyCompositeFilterConfig.Config conf = new DbusKeyCompositeFilterConfig.Config();
        ConfigLoader<DbusKeyCompositeFilterConfig.StaticConfig> filterConfigLoader = new ConfigLoader<DbusKeyCompositeFilterConfig.StaticConfig>(SERVER_SIDE_FILTER_PREFIX, conf);
        DbusKeyCompositeFilterConfig.StaticConfig sConf = filterConfigLoader.loadConfig(props);
        return new DbusKeyCompositeFilterConfig(sConf);
    } else {
        return null;
    }
}

Example 15 with ConfigLoader

use of com.linkedin.databus.core.util.ConfigLoader in project databus by linkedin.

the class SimpleFileLoggingConsumer method mainFunction.

public void mainFunction(String[] args) throws Exception {
    String[] leftOverArgs = processLocalArgs(args);
    Properties startupProps = DatabusHttpClientImpl.processCommandLineArgs(leftOverArgs);
    DatabusHttpClientImpl.Config clientConfigBuilder = new DatabusHttpClientImpl.Config();
    clientConfigBuilder.getContainer().setIdFromName(MODULE + ".localhost");
    if (_enableBootStrap) {
        clientConfigBuilder.getRuntime().getBootstrap().setEnabled(true);
    }
    ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>("databus.client.", clientConfigBuilder);
    String[] sources = addSources();
    StringBuilder sourcesString = new StringBuilder();
    boolean firstSrc = true;
    for (String source : sources) {
        if (!firstSrc)
            sourcesString.append(",");
        firstSrc = false;
        sourcesString.append(source);
    }
    if (_httpPort != null) {
        startupProps.put("databus.client.container.httpPort", _httpPort);
    }
    if (_jmxServicePort != null) {
        startupProps.put("databus.client.container.jmx.jmxServicePort", _jmxServicePort);
    }
    if (_checkpointFileRootDir != null) {
        startupProps.put("databus.client.checkpointPersistence.fileSystem.rootDirectory", _checkpointFileRootDir);
    }
    DatabusHttpClientImpl.StaticConfig clientConfig = configLoader.loadConfig(startupProps);
    // set up relay
    ServerInfoBuilder relayBuilder = clientConfig.getRuntime().getRelay("1");
    relayBuilder.setName("DefaultRelay");
    if (_relayHost != null) {
        relayBuilder.setHost(_relayHost);
    }
    if (_relayPort != null) {
        relayBuilder.setPort(Integer.parseInt(_relayPort));
    }
    relayBuilder.setSources(sourcesString.toString());
    // set up bootstrap
    if (_enableBootStrap) {
        ServerInfoBuilder bootstrapBuilder = clientConfig.getRuntime().getBootstrap().getService("2");
        bootstrapBuilder.setName("DefaultBootstrapServices");
        if (_bootstrapHost != null) {
            bootstrapBuilder.setHost(_bootstrapHost);
        }
        if (_bootstrapPort != null) {
            bootstrapBuilder.setPort(Integer.parseInt(_bootstrapPort));
        }
        bootstrapBuilder.setSources(sourcesString.toString());
    }
    // handler server side filtering, can either pass a config file or set from command line
    DbusKeyCompositeFilterConfig filterConfig = createServerSideFilterConfig(_filterConfFile, startupProps);
    // set up listeners
    DatabusHttpClientImpl client = new DatabusHttpClientImpl(clientConfig);
    // dump decoded payload values and raw (undecoded) events
    DatabusFileLoggingConsumer consumer = createTypedConsumer(_valueDumpFile, _eventDumpFile);
    if (_eventPattern != null) {
        consumer.setEventPattern(_eventPattern);
    }
    DatabusRegistration reg = client.register(consumer, sources);
    if (!(reg instanceof DatabusV2RegistrationImpl)) {
        throw new RuntimeException("Unexpected type for registration Object !!");
    }
    if (null != filterConfig)
        reg.withServerSideFilter(filterConfig);
    // add pause processor
    try {
        client.getProcessorRegistry().register(ConsumerPauseRequestProcessor.COMMAND_NAME, new ConsumerPauseRequestProcessor(null, consumer));
        client.getProcessorRegistry().register(ContainerOperationProcessor.COMMAND_NAME, new ContainerOperationProcessor(null, client));
    } catch (ProcessorRegistrationConflictException e) {
        LOG.error("Failed to register " + ConsumerPauseRequestProcessor.COMMAND_NAME);
    }
    DatabusClientShutdownThread shutdownThread = new DatabusClientShutdownThread(client);
    Runtime.getRuntime().addShutdownHook(shutdownThread);
    //this should automatically start the registration
    client.startAndBlock();
}