Search in sources :

Example 1 with BootstrapServerConfig

use of com.linkedin.databus.bootstrap.server.BootstrapServerConfig in project databus by linkedin.

the class TestFilterToSQL method setUp.

@BeforeClass
public void setUp() throws IOException, DatabusException, InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException {
    Logger.getRootLogger().removeAllAppenders();
    Appender defApp = new ConsoleAppender(new SimpleLayout());
    Logger.getRootLogger().addAppender(defApp);
    Logger.getRootLogger().setLevel(Level.INFO);
    BootstrapServerConfig configBuilder = new BootstrapServerConfig();
    BootstrapServerStaticConfig staticConfig = configBuilder.build();
    processor = new BootstrapProcessor(staticConfig, null);
    partConf = new KeyFilterConfigHolder.Config();
}
Also used : Appender(org.apache.log4j.Appender) ConsoleAppender(org.apache.log4j.ConsoleAppender) ConsoleAppender(org.apache.log4j.ConsoleAppender) BootstrapServerConfig(com.linkedin.databus.bootstrap.server.BootstrapServerConfig) BootstrapProcessor(com.linkedin.databus.bootstrap.server.BootstrapProcessor) KeyFilterConfigHolder(com.linkedin.databus2.core.filter.KeyFilterConfigHolder) SimpleLayout(org.apache.log4j.SimpleLayout) BootstrapServerStaticConfig(com.linkedin.databus.bootstrap.server.BootstrapServerStaticConfig) BeforeClass(org.testng.annotations.BeforeClass)

Example 2 with BootstrapServerConfig

use of com.linkedin.databus.bootstrap.server.BootstrapServerConfig in project databus by linkedin.

the class DatabusBootstrapClient method init.

private void init() throws InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException, IOException, DatabusException {
    BootstrapServerConfig configBuilder = new BootstrapServerConfig();
    BootstrapServerStaticConfig staticConfig = configBuilder.build();
    _dbProcessor = new BootstrapProcessor(staticConfig, null);
}
Also used : BootstrapServerConfig(com.linkedin.databus.bootstrap.server.BootstrapServerConfig) BootstrapProcessor(com.linkedin.databus.bootstrap.server.BootstrapProcessor) BootstrapServerStaticConfig(com.linkedin.databus.bootstrap.server.BootstrapServerStaticConfig)

Example 3 with BootstrapServerConfig

use of com.linkedin.databus.bootstrap.server.BootstrapServerConfig in project databus by linkedin.

the class TestBootstrap method testBootstrapProcessor.

@Test
public void testBootstrapProcessor() throws InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException, IOException, BootstrapProcessingException, DatabusException, BootstrapDatabaseTooOldException, BootstrapDBException {
    EventProcessor processorCallback = new EventProcessor();
    BootstrapConfig config = new BootstrapConfig();
    BootstrapReadOnlyConfig staticConfig = config.build();
    BootstrapServerConfig configBuilder = new BootstrapServerConfig();
    configBuilder.setEnableMinScnCheck(false);
    BootstrapServerStaticConfig staticServerConfig = configBuilder.build();
    BootstrapProcessor processor = new BootstrapProcessor(staticServerConfig, null);
    String sourceName = "TestBootstrap.testBootstrapProcessor.events";
    // Create the tables for all the sources before starting up the threads
    BootstrapConn _bootstrapConn = new BootstrapConn();
    boolean autoCommit = true;
    _bootstrapConn.initBootstrapConn(autoCommit, staticConfig.getBootstrapDBUsername(), staticConfig.getBootstrapDBPassword(), staticConfig.getBootstrapDBHostname(), staticConfig.getBootstrapDBName());
    BootstrapDBMetaDataDAO dao = new BootstrapDBMetaDataDAO(_bootstrapConn, staticConfig.getBootstrapDBHostname(), staticConfig.getBootstrapDBUsername(), staticConfig.getBootstrapDBPassword(), staticConfig.getBootstrapDBName(), autoCommit);
    SourceStatusInfo srcStatusInfo = dao.getSrcIdStatusFromDB(sourceName, false);
    if (srcStatusInfo.getSrcId() >= 0) {
        dao.dropSourceInDB(srcStatusInfo.getSrcId());
    }
    dao.addNewSourceInDB(sourceName, BootstrapProducerStatus.ACTIVE);
    srcStatusInfo = dao.getSrcIdStatusFromDB(sourceName, false);
    setBootstrapLoginfoTable(_bootstrapConn, 0, Long.MAX_VALUE);
    //insert one row
    insertOneSnapshotEvent(dao, srcStatusInfo.getSrcId(), Long.MAX_VALUE - 10, Long.MAX_VALUE - 100, "check", "test_payload_data");
    BootstrapCheckpointHandler bstCheckpointHandler = new BootstrapCheckpointHandler(sourceName);
    Checkpoint c = bstCheckpointHandler.createInitialBootstrapCheckpoint(null, 0L);
    c.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    c.setBootstrapStartScn(Long.MAX_VALUE - 10);
    c.setSnapshotOffset(Long.MAX_VALUE - 1000);
    // System.out.println("Snapshot");
    processor.streamSnapShotRows(c, processorCallback);
    Assert.assertEquals(processorCallback.getrIds().size(), 1);
    Assert.assertEquals(Long.MAX_VALUE - 10, processorCallback.getrIds().get(0).longValue());
    Assert.assertEquals(Long.MAX_VALUE - 100, processorCallback.getSequences().get(0).longValue());
    Assert.assertEquals("check", processorCallback.getSrcKeys().get(0));
    Assert.assertEquals("test_payload_data", new String(processorCallback.getValues().get(0)));
    processorCallback.reset();
    c.setSnapshotOffset(Checkpoint.FULLY_CONSUMED_WINDOW_OFFSET);
    c.setBootstrapTargetScn(c.getBootstrapStartScn().longValue());
    bstCheckpointHandler.advanceAfterTargetScn(c);
    bstCheckpointHandler.advanceAfterSnapshotPhase(c);
    //		  c.setConsumptionMode(DbusClientMode.BOOTSTRAP_CATCHUP);
    //		  c.setCatchupSource(sourceName);
    //		  c.setWindowScn(0L);
    //		  c.setWindowOffset(0);
    // System.out.println("Catchup");
    boolean phaseCompleted = processor.streamCatchupRows(c, processorCallback);
    Assert.assertEquals(true, phaseCompleted);
}
Also used : BootstrapProcessor(com.linkedin.databus.bootstrap.server.BootstrapProcessor) BootstrapConn(com.linkedin.databus.bootstrap.common.BootstrapConn) BootstrapConfig(com.linkedin.databus.bootstrap.common.BootstrapConfig) BootstrapDBMetaDataDAO(com.linkedin.databus.bootstrap.common.BootstrapDBMetaDataDAO) BootstrapCheckpointHandler(com.linkedin.databus.core.BootstrapCheckpointHandler) BootstrapReadOnlyConfig(com.linkedin.databus.bootstrap.common.BootstrapReadOnlyConfig) BootstrapServerConfig(com.linkedin.databus.bootstrap.server.BootstrapServerConfig) Checkpoint(com.linkedin.databus.core.Checkpoint) BootstrapServerStaticConfig(com.linkedin.databus.bootstrap.server.BootstrapServerStaticConfig) SourceStatusInfo(com.linkedin.databus.bootstrap.common.BootstrapDBMetaDataDAO.SourceStatusInfo) Test(org.testng.annotations.Test)

Aggregations

BootstrapProcessor (com.linkedin.databus.bootstrap.server.BootstrapProcessor)3 BootstrapServerConfig (com.linkedin.databus.bootstrap.server.BootstrapServerConfig)3 BootstrapServerStaticConfig (com.linkedin.databus.bootstrap.server.BootstrapServerStaticConfig)3 BootstrapConfig (com.linkedin.databus.bootstrap.common.BootstrapConfig)1 BootstrapConn (com.linkedin.databus.bootstrap.common.BootstrapConn)1 BootstrapDBMetaDataDAO (com.linkedin.databus.bootstrap.common.BootstrapDBMetaDataDAO)1 SourceStatusInfo (com.linkedin.databus.bootstrap.common.BootstrapDBMetaDataDAO.SourceStatusInfo)1 BootstrapReadOnlyConfig (com.linkedin.databus.bootstrap.common.BootstrapReadOnlyConfig)1 BootstrapCheckpointHandler (com.linkedin.databus.core.BootstrapCheckpointHandler)1 Checkpoint (com.linkedin.databus.core.Checkpoint)1 KeyFilterConfigHolder (com.linkedin.databus2.core.filter.KeyFilterConfigHolder)1 Appender (org.apache.log4j.Appender)1 ConsoleAppender (org.apache.log4j.ConsoleAppender)1 SimpleLayout (org.apache.log4j.SimpleLayout)1 BeforeClass (org.testng.annotations.BeforeClass)1 Test (org.testng.annotations.Test)1