Examples with BootstrapConfig com.linkedin.databus.bootstrap.common.BootstrapConfig used on opensource projects

Example 1 with BootstrapConfig

use of com.linkedin.databus.bootstrap.common.BootstrapConfig in project databus by linkedin.

the class TestBootstrap method testBootstrapService.

@Test
public void testBootstrapService() throws InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException, IOException, BootstrapProcessingException, DatabusException, BootstrapDatabaseTooOldException, BootstrapDBException {
    final Logger log = Logger.getLogger("TestBootstrap.testBootstrapService");
    EventProcessor processorCallback = new EventProcessor();
    BootstrapConfig config = new BootstrapConfig();
    BootstrapReadOnlyConfig staticConfig = config.build();
    String[] sources = new String[4];
    sources[0] = "TestBootstrap.testBootstrapService.event";
    sources[1] = "TestBootstrap.testBootstrapService.event1";
    sources[2] = "TestBootstrap.testBootstrapService.event2";
    sources[3] = "TestBootstrap.testBootstrapService.event3";
    // Create the tables for all the sources before starting up the threads
    BootstrapConn _bootstrapConn = new BootstrapConn();
    final boolean autoCommit = true;
    _bootstrapConn.initBootstrapConn(autoCommit, staticConfig.getBootstrapDBUsername(), staticConfig.getBootstrapDBPassword(), staticConfig.getBootstrapDBHostname(), staticConfig.getBootstrapDBName());
    BootstrapDBMetaDataDAO dao = new BootstrapDBMetaDataDAO(_bootstrapConn, staticConfig.getBootstrapDBHostname(), staticConfig.getBootstrapDBUsername(), staticConfig.getBootstrapDBPassword(), staticConfig.getBootstrapDBName(), autoCommit);
    for (String source : sources) {
        SourceStatusInfo srcStatusInfo = dao.getSrcIdStatusFromDB(source, false);
        if (srcStatusInfo.getSrcId() >= 0) {
            dao.dropSourceInDB(srcStatusInfo.getSrcId());
        }
        dao.addNewSourceInDB(source, BootstrapProducerStatus.ACTIVE);
    }
    setBootstrapLoginfoTable(_bootstrapConn, 1, 1);
    DatabusBootstrapClient s = new DatabusBootstrapClient(sources);
    Checkpoint cp;
    while ((cp = s.getNextBatch(10, processorCallback)).getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION) {
        log.debug(cp);
    }
}

Example 2 with BootstrapConfig

use of com.linkedin.databus.bootstrap.common.BootstrapConfig in project databus by linkedin.

the class BootstrapDBCleanerMain method main.

/**
 * @param args
 */
public static void main(String[] args) throws Exception {
    parseArgs(args);
    BootstrapCleanerConfig config = new BootstrapCleanerConfig();
    BootstrapConfig bsConfig = new BootstrapConfig();
    ConfigLoader<BootstrapCleanerStaticConfig> configLoader = new ConfigLoader<BootstrapCleanerStaticConfig>("databus.bootstrap.cleaner.", config);
    ConfigLoader<BootstrapReadOnlyConfig> configLoader2 = new ConfigLoader<BootstrapReadOnlyConfig>("databus.bootstrap.db.", bsConfig);
    _sCleanerConfig = configLoader.loadConfig(_sBootstrapConfigProps);
    _sBootstrapConfig = configLoader2.loadConfig(_sBootstrapConfigProps);
    BootstrapDBCleaner cleaner = new BootstrapDBCleaner("StandAloneCleaner", _sCleanerConfig, _sBootstrapConfig, null, _sSources);
    cleaner.doClean();
}

Example 3 with BootstrapConfig

use of com.linkedin.databus.bootstrap.common.BootstrapConfig in project databus by linkedin.

the class BootstrapDbControlMain method main.

public static void main(String[] args) throws IOException, DatabusException, InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException {
    Cli cli = new Cli("java " + BootstrapDbControlMain.class.getSimpleName() + " [options]");
    cli.processCommandLineArgs(args);
    Properties cfgProps = cli.getConfigProps();
    BootstrapConfig cfgBuilder = new BootstrapConfig();
    ConfigLoader<BootstrapReadOnlyConfig> configLoad = new ConfigLoader<BootstrapReadOnlyConfig>("databus.bootstrap.", cfgBuilder);
    configLoad.loadConfig(cfgProps);
    BootstrapReadOnlyConfig cfg = cfgBuilder.build();
    BootstrapConn conn = new BootstrapConn();
    try {
        conn.initBootstrapConn(true, cfg.getBootstrapDBUsername(), cfg.getBootstrapDBPassword(), cfg.getBootstrapDBHostname(), cfg.getBootstrapDBName());
        BootstrapDBMetaDataDAO dao = new BootstrapDBMetaDataDAO(conn, cfg.getBootstrapDBHostname(), cfg.getBootstrapDBUsername(), cfg.getBootstrapDBPassword(), cfg.getBootstrapDBName(), false);
        switch(cli.getAction()) {
            case CREATE_LOG_TABLE:
                dao.createNewLogTable(cli.getSrcId());
                break;
            case CREATE_TAB_TABLE:
                dao.createNewSrcTable(cli.getSrcId());
                break;
            default:
                throw new RuntimeException("unknown action: " + cli.getAction());
        }
    } finally {
        conn.close();
    }
}

Example 4 with BootstrapConfig

use of com.linkedin.databus.bootstrap.common.BootstrapConfig in project databus by linkedin.

the class TestBootstrap method testBootstrapProcessor.

@Test
public void testBootstrapProcessor() throws InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException, IOException, BootstrapProcessingException, DatabusException, BootstrapDatabaseTooOldException, BootstrapDBException {
    EventProcessor processorCallback = new EventProcessor();
    BootstrapConfig config = new BootstrapConfig();
    BootstrapReadOnlyConfig staticConfig = config.build();
    BootstrapServerConfig configBuilder = new BootstrapServerConfig();
    configBuilder.setEnableMinScnCheck(false);
    BootstrapServerStaticConfig staticServerConfig = configBuilder.build();
    BootstrapProcessor processor = new BootstrapProcessor(staticServerConfig, null);
    String sourceName = "TestBootstrap.testBootstrapProcessor.events";
    // Create the tables for all the sources before starting up the threads
    BootstrapConn _bootstrapConn = new BootstrapConn();
    boolean autoCommit = true;
    _bootstrapConn.initBootstrapConn(autoCommit, staticConfig.getBootstrapDBUsername(), staticConfig.getBootstrapDBPassword(), staticConfig.getBootstrapDBHostname(), staticConfig.getBootstrapDBName());
    BootstrapDBMetaDataDAO dao = new BootstrapDBMetaDataDAO(_bootstrapConn, staticConfig.getBootstrapDBHostname(), staticConfig.getBootstrapDBUsername(), staticConfig.getBootstrapDBPassword(), staticConfig.getBootstrapDBName(), autoCommit);
    SourceStatusInfo srcStatusInfo = dao.getSrcIdStatusFromDB(sourceName, false);
    if (srcStatusInfo.getSrcId() >= 0) {
        dao.dropSourceInDB(srcStatusInfo.getSrcId());
    }
    dao.addNewSourceInDB(sourceName, BootstrapProducerStatus.ACTIVE);
    srcStatusInfo = dao.getSrcIdStatusFromDB(sourceName, false);
    setBootstrapLoginfoTable(_bootstrapConn, 0, Long.MAX_VALUE);
    // insert one row
    insertOneSnapshotEvent(dao, srcStatusInfo.getSrcId(), Long.MAX_VALUE - 10, Long.MAX_VALUE - 100, "check", "test_payload_data");
    BootstrapCheckpointHandler bstCheckpointHandler = new BootstrapCheckpointHandler(sourceName);
    Checkpoint c = bstCheckpointHandler.createInitialBootstrapCheckpoint(null, 0L);
    c.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    c.setBootstrapStartScn(Long.MAX_VALUE - 10);
    c.setSnapshotOffset(Long.MAX_VALUE - 1000);
    // System.out.println("Snapshot");
    processor.streamSnapShotRows(c, processorCallback);
    Assert.assertEquals(processorCallback.getrIds().size(), 1);
    Assert.assertEquals(Long.MAX_VALUE - 10, processorCallback.getrIds().get(0).longValue());
    Assert.assertEquals(Long.MAX_VALUE - 100, processorCallback.getSequences().get(0).longValue());
    Assert.assertEquals("check", processorCallback.getSrcKeys().get(0));
    Assert.assertEquals("test_payload_data", new String(processorCallback.getValues().get(0)));
    processorCallback.reset();
    c.setSnapshotOffset(Checkpoint.FULLY_CONSUMED_WINDOW_OFFSET);
    c.setBootstrapTargetScn(c.getBootstrapStartScn().longValue());
    bstCheckpointHandler.advanceAfterTargetScn(c);
    bstCheckpointHandler.advanceAfterSnapshotPhase(c);
    // c.setConsumptionMode(DbusClientMode.BOOTSTRAP_CATCHUP);
    // c.setCatchupSource(sourceName);
    // c.setWindowScn(0L);
    // c.setWindowOffset(0);
    // System.out.println("Catchup");
    boolean phaseCompleted = processor.streamCatchupRows(c, processorCallback);
    Assert.assertEquals(true, phaseCompleted);
}

Example 5 with BootstrapConfig

use of com.linkedin.databus.bootstrap.common.BootstrapConfig in project databus by linkedin.

the class BootstrapAddSource method parseArgs.

@SuppressWarnings("static-access")
public static Config parseArgs(String[] args) throws Exception {
    CommandLineParser cliParser = new GnuParser();
    Option helpOption = OptionBuilder.withLongOpt(HELP_OPT_LONG_NAME).withDescription("Help screen").create(HELP_OPT_CHAR);
    Option sourceIdOption = OptionBuilder.withLongOpt(SOURCE_ID_OPT_LONG_NAME).withDescription("Source ID for which tables need to be added").hasArg().withArgName("Source ID").create(SOURCE_ID_OPT_CHAR);
    Option sourceNameOption = OptionBuilder.withLongOpt(SOURCE_NAME_OPT_LONG_NAME).withDescription("Source Name for which tables need to be added").hasArg().withArgName("Source ID").create(SOURCE_NAME_OPT_CHAR);
    Option dbOption = OptionBuilder.withLongOpt(BOOTSTRAP_DB_PROPS_OPT_LONG_NAME).withDescription("Bootstrap producer properties to use").hasArg().withArgName("property_file").create(BOOTSTRAP_DB_PROP_OPT_CHAR);
    Option cmdLinePropsOption = OptionBuilder.withLongOpt(CMD_LINE_PROPS_OPT_LONG_NAME).withDescription("Cmd line override of config properties. Semicolon separated.").hasArg().withArgName("Semicolon_separated_properties").create(CMD_LINE_PROPS_OPT_CHAR);
    Option log4jPropsOption = OptionBuilder.withLongOpt(LOG4J_PROPS_OPT_LONG_NAME).withDescription("Log4j properties to use").hasArg().withArgName("property_file").create(LOG4J_PROPS_OPT_CHAR);
    Options options = new Options();
    options.addOption(helpOption);
    options.addOption(sourceIdOption);
    options.addOption(sourceNameOption);
    options.addOption(dbOption);
    options.addOption(cmdLinePropsOption);
    options.addOption(log4jPropsOption);
    CommandLine cmd = null;
    try {
        cmd = cliParser.parse(options, args);
    } catch (ParseException pe) {
        LOG.error("Bootstrap Physical Config: failed to parse command-line options.", pe);
        throw new RuntimeException("Bootstrap Physical Config: failed to parse command-line options.", pe);
    }
    if (cmd.hasOption(LOG4J_PROPS_OPT_CHAR)) {
        String log4jPropFile = cmd.getOptionValue(LOG4J_PROPS_OPT_CHAR);
        PropertyConfigurator.configure(log4jPropFile);
        LOG.info("Using custom logging settings from file " + log4jPropFile);
    } else {
        PatternLayout defaultLayout = new PatternLayout("%d{ISO8601} +%r [%t] (%p) {%c} %m%n");
        ConsoleAppender defaultAppender = new ConsoleAppender(defaultLayout);
        Logger.getRootLogger().removeAllAppenders();
        Logger.getRootLogger().addAppender(defaultAppender);
        LOG.info("Using default logging settings");
    }
    if (cmd.hasOption(HELP_OPT_CHAR)) {
        printCliHelp(options);
        System.exit(0);
    }
    if (!cmd.hasOption(SOURCE_ID_OPT_CHAR))
        throw new RuntimeException("Source ID is not provided");
    if (!cmd.hasOption(SOURCE_NAME_OPT_CHAR))
        throw new RuntimeException("Source Name is not provided");
    if (!cmd.hasOption(BOOTSTRAP_DB_PROP_OPT_CHAR))
        throw new RuntimeException("Bootstrap config is not provided");
    String propFile = cmd.getOptionValue(BOOTSTRAP_DB_PROP_OPT_CHAR);
    LOG.info("Loading bootstrap DB config from properties file " + propFile);
    _sBootstrapConfigProps = new Properties();
    FileInputStream f = new FileInputStream(propFile);
    try {
        _sBootstrapConfigProps.load(f);
    } finally {
        f.close();
    }
    if (cmd.hasOption(CMD_LINE_PROPS_OPT_CHAR)) {
        String cmdLinePropString = cmd.getOptionValue(CMD_LINE_PROPS_OPT_CHAR);
        updatePropsFromCmdLine(_sBootstrapConfigProps, cmdLinePropString);
    }
    int srcId = Integer.parseInt(cmd.getOptionValue(SOURCE_ID_OPT_CHAR));
    String srcName = cmd.getOptionValue(SOURCE_NAME_OPT_CHAR);
    BootstrapConfig config = new BootstrapConfig();
    ConfigLoader<BootstrapReadOnlyConfig> configLoader = new ConfigLoader<BootstrapReadOnlyConfig>("bootstrap.", config);
    BootstrapReadOnlyConfig staticConfig = configLoader.loadConfig(_sBootstrapConfigProps);
    Config cfg = new Config();
    cfg.setDbConfig(staticConfig);
    cfg.setSrcId(srcId);
    cfg.setSrcName(srcName);
    return cfg;
}