Examples with ConfigLoader com.linkedin.databus.core.util.ConfigLoader used on opensource projects

Example 6 with ConfigLoader

use of com.linkedin.databus.core.util.ConfigLoader in project databus by linkedin.

the class BootstrapDbControlMain method main.

public static void main(String[] args) throws IOException, DatabusException, InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException {
    Cli cli = new Cli("java " + BootstrapDbControlMain.class.getSimpleName() + " [options]");
    cli.processCommandLineArgs(args);
    Properties cfgProps = cli.getConfigProps();
    BootstrapConfig cfgBuilder = new BootstrapConfig();
    ConfigLoader<BootstrapReadOnlyConfig> configLoad = new ConfigLoader<BootstrapReadOnlyConfig>("databus.bootstrap.", cfgBuilder);
    configLoad.loadConfig(cfgProps);
    BootstrapReadOnlyConfig cfg = cfgBuilder.build();
    BootstrapConn conn = new BootstrapConn();
    try {
        conn.initBootstrapConn(true, cfg.getBootstrapDBUsername(), cfg.getBootstrapDBPassword(), cfg.getBootstrapDBHostname(), cfg.getBootstrapDBName());
        BootstrapDBMetaDataDAO dao = new BootstrapDBMetaDataDAO(conn, cfg.getBootstrapDBHostname(), cfg.getBootstrapDBUsername(), cfg.getBootstrapDBPassword(), cfg.getBootstrapDBName(), false);
        switch(cli.getAction()) {
            case CREATE_LOG_TABLE:
                dao.createNewLogTable(cli.getSrcId());
                break;
            case CREATE_TAB_TABLE:
                dao.createNewSrcTable(cli.getSrcId());
                break;
            default:
                throw new RuntimeException("unknown action: " + cli.getAction());
        }
    } finally {
        conn.close();
    }
}

Example 7 with ConfigLoader

use of com.linkedin.databus.core.util.ConfigLoader in project databus by linkedin.

the class DatabusBootstrapProducer method main.

public static void main(String[] args) throws Exception {
    // Properties startupProps = BootstrapConfig.loadConfigProperties(args);
    Properties startupProps = ServerContainer.processCommandLineArgs(args);
    BootstrapProducerConfig producerConfig = new BootstrapProducerConfig();
    ConfigLoader<BootstrapProducerStaticConfig> staticProducerConfigLoader = new ConfigLoader<BootstrapProducerStaticConfig>("databus.bootstrap.", producerConfig);
    BootstrapProducerStaticConfig staticProducerConfig = staticProducerConfigLoader.loadConfig(startupProps);
    DatabusBootstrapProducer bootstrapProducer = new DatabusBootstrapProducer(staticProducerConfig);
    bootstrapProducer.registerShutdownHook();
    bootstrapProducer.startAndBlock();
}

Example 8 with ConfigLoader

use of com.linkedin.databus.core.util.ConfigLoader in project databus by linkedin.

the class MockBootstrapConnection method testBootstrapPendingEvent.

// Make sure that we suspend on error when we get the x-dbus-pending-size header with a size that is
// larger than our dbusevent size.
@Test
public void testBootstrapPendingEvent() throws Exception {
    List<String> sources = Arrays.asList("source1");
    Properties clientProps = new Properties();
    clientProps.setProperty("client.container.httpPort", "0");
    clientProps.setProperty("client.container.jmx.rmiEnabled", "false");
    clientProps.setProperty("client.runtime.bootstrap.enabled", "true");
    clientProps.setProperty("client.runtime.bootstrap.service(1).name", "bs1");
    clientProps.setProperty("client.runtime.bootstrap.service(1).host", "localhost");
    clientProps.setProperty("client.runtime.bootstrap.service(1).port", "10001");
    clientProps.setProperty("client.runtime.bootstrap.service(1).sources", "source1");
    clientProps.setProperty("client.runtime.relay(1).name", "relay1");
    clientProps.setProperty("client.runtime.relay(1).port", "10001");
    clientProps.setProperty("client.runtime.relay(1).sources", "source1");
    clientProps.setProperty("client.connectionDefaults.eventBuffer.maxSize", "100000");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.maxRetryNum", "3");
    DatabusHttpClientImpl.Config clientConfBuilder = new DatabusHttpClientImpl.Config();
    ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>("client.", clientConfBuilder);
    configLoader.loadConfig(clientProps);
    DatabusHttpClientImpl.StaticConfig clientConf = clientConfBuilder.build();
    DatabusSourcesConnection.StaticConfig srcConnConf = clientConf.getConnectionDefaults();
    DatabusHttpClientImpl client = new DatabusHttpClientImpl(clientConf);
    client.registerDatabusBootstrapListener(new LoggingConsumer(), null, "source1");
    Assert.assertNotNull(client, "client instantiation failed");
    DatabusHttpClientImpl.RuntimeConfig clientRtConf = clientConf.getRuntime().build();
    //we keep the index of the next server we expect to see
    AtomicInteger serverIdx = new AtomicInteger(-1);
    List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
    sourcesResponse.add(new IdNamePair(1L, "source1"));
    Map<Long, List<RegisterResponseEntry>> registerResponse = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> regResponse = new ArrayList<RegisterResponseEntry>();
    regResponse.add(new RegisterResponseEntry(1L, (short) 1, SCHEMA$.toString()));
    registerResponse.put(1L, regResponse);
    ChunkedBodyReadableByteChannel channel = EasyMock.createMock(ChunkedBodyReadableByteChannel.class);
    // getting the pending-event-size header is called twice, once for checking and once for logging.
    EasyMock.expect(channel.getMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE)).andReturn("1000000").times(2);
    EasyMock.expect(channel.getMetadata("x-dbus-error-cause")).andReturn(null).times(2);
    EasyMock.expect(channel.getMetadata("x-dbus-error")).andReturn(null).times(2);
    EasyMock.replay(channel);
    DbusEventBuffer dbusBuffer = EasyMock.createMock(DbusEventBuffer.class);
    dbusBuffer.endEvents(false, -1, false, false, null);
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(dbusBuffer.injectEvent(EasyMock.<DbusEventInternalReadable>notNull())).andReturn(true).anyTimes();
    EasyMock.expect(dbusBuffer.getEventSerializationVersion()).andReturn(DbusEventFactory.DBUS_EVENT_V1).anyTimes();
    EasyMock.expect(dbusBuffer.getMaxReadBufferCapacity()).andReturn(600).times(2);
    EasyMock.expect(dbusBuffer.getBufferFreeReadSpace()).andReturn(600000).times(2);
    EasyMock.expect(dbusBuffer.readEvents(EasyMock.<ReadableByteChannel>notNull())).andReturn(1).times(1);
    EasyMock.expect(dbusBuffer.readEvents(EasyMock.<ReadableByteChannel>notNull(), EasyMock.<List<InternalDatabusEventsListener>>notNull(), EasyMock.<DbusEventsStatisticsCollector>isNull())).andReturn(0).times(1);
    EasyMock.replay(dbusBuffer);
    ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sources);
    //This guy succeeds on /sources but fails on /register
    MockBootstrapConnection mockSuccessConn = new MockBootstrapConnection(10, 10, channel, serverIdx, false);
    DatabusBootstrapConnectionFactory mockConnFactory = org.easymock.EasyMock.createMock("mockRelayFactory", DatabusBootstrapConnectionFactory.class);
    //each server should be tried MAX_RETRIES time until all retries are exhausted
    EasyMock.expect(mockConnFactory.createConnection(EasyMock.<ServerInfo>notNull(), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andReturn(mockSuccessConn).anyTimes();
    List<DatabusSubscription> sourcesSubList = DatabusSubscription.createSubscriptionList(sources);
    DatabusSourcesConnection sourcesConn2 = EasyMock.createMock(DatabusSourcesConnection.class);
    EasyMock.expect(sourcesConn2.getSourcesNames()).andReturn(Arrays.asList("source1")).anyTimes();
    EasyMock.expect(sourcesConn2.getSubscriptions()).andReturn(sourcesSubList).anyTimes();
    EasyMock.expect(sourcesConn2.getConnectionConfig()).andReturn(srcConnConf).anyTimes();
    EasyMock.expect(sourcesConn2.getConnectionStatus()).andReturn(new DatabusComponentStatus("dummy")).anyTimes();
    EasyMock.expect(sourcesConn2.getLocalRelayCallsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getRelayCallsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getUnifiedClientStats()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapConnFactory()).andReturn(mockConnFactory).anyTimes();
    EasyMock.expect(sourcesConn2.loadPersistentCheckpoint()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getDataEventsBuffer()).andReturn(dbusBuffer).anyTimes();
    EasyMock.expect(sourcesConn2.isBootstrapEnabled()).andReturn(true).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapRegistrations()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapServices()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapEventsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.makeThreadSafe(mockConnFactory, true);
    EasyMock.makeThreadSafe(sourcesConn2, true);
    EasyMock.replay(mockConnFactory);
    EasyMock.replay(sourcesConn2);
    BootstrapPullThread bsPuller = new BootstrapPullThread("RelayPuller", sourcesConn2, dbusBuffer, connStateFactory, clientRtConf.getBootstrap().getServicesSet(), new ArrayList<DbusKeyCompositeFilterConfig>(), clientConf.getPullerBufferUtilizationPct(), ManagementFactory.getPlatformMBeanServer(), new DbusEventV2Factory(), null, null);
    mockSuccessConn.setCallback(bsPuller);
    bsPuller.getComponentStatus().start();
    Checkpoint cp = _ckptHandlerSource1.createInitialBootstrapCheckpoint(null, 0L);
    //TODO remove
    //cp.setSnapshotSource("source1");
    //cp.setCatchupSource("source1");
    //cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    ConnectionState connState = bsPuller.getConnectionState();
    connState.switchToBootstrap(cp);
    testTransitionCase(bsPuller, StateId.BOOTSTRAP, StateId.REQUEST_START_SCN, cp);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.REQUEST_START_SCN, StateId.START_SCN_RESPONSE_SUCCESS, null);
    bsPuller.getMessageQueue().clear();
    Map<Long, List<RegisterResponseEntry>> entries = new HashMap<Long, List<RegisterResponseEntry>>();
    entries.put(1L, new ArrayList<RegisterResponseEntry>());
    connState.setSourcesSchemas(entries);
    connState.setCurrentBSServerInfo(bsPuller.getCurentServer());
    testTransitionCase(bsPuller, StateId.START_SCN_RESPONSE_SUCCESS, StateId.REQUEST_STREAM, null);
    bsPuller.getMessageQueue().clear();
    connState.getSourcesNameMap().put("source1", new IdNamePair(1L, "source1"));
    connState.getSourceIdMap().put(1L, new IdNamePair(1L, "source1"));
    testTransitionCase(bsPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS, null);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.STREAM_REQUEST_SUCCESS, "SUSPEND_ON_ERROR", null);
    EasyMock.verify(channel);
    EasyMock.verify(sourcesConn2);
    EasyMock.verify(dbusBuffer);
    EasyMock.verify(channel);
    EasyMock.verify(mockConnFactory);
}

Example 9 with ConfigLoader

use of com.linkedin.databus.core.util.ConfigLoader in project databus by linkedin.

the class TestMultitenantRelay method setUp.

@BeforeClass
public void setUp() throws InvalidConfigException {
    Properties clientProps = new Properties();
    clientProps.setProperty("client.runtime.relay(1).name", "relay");
    clientProps.setProperty("client.runtime.relay(1).port", "10001");
    clientProps.setProperty("client.runtime.relay(1).sources", "source1");
    clientProps.setProperty("client.runtime.relay(2).name", "relay");
    clientProps.setProperty("client.runtime.relay(2).port", "10001");
    clientProps.setProperty("client.runtime.relay(2).sources", "source2");
    clientProps.setProperty("client.runtime.relay(3).name", "relay2");
    clientProps.setProperty("client.runtime.relay(3).port", "10002");
    clientProps.setProperty("client.runtime.relay(3).sources", "source3");
    clientConfBuilder = new DatabusHttpClientImpl.Config();
    ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>("client.", clientConfBuilder);
    configLoader.loadConfig(clientProps);
}

Example 10 with ConfigLoader

use of com.linkedin.databus.core.util.ConfigLoader in project databus by linkedin.

the class CheckpointSerializerMain method main.

public static void main(String[] args) throws Exception {
    parseArgs(args);
    PatternLayout defaultLayout = new PatternLayout("%d{ISO8601} +%r [%t] (%p) {%c{1}} %m%n");
    ConsoleAppender defaultAppender = new ConsoleAppender(defaultLayout);
    Logger.getRootLogger().removeAllAppenders();
    Logger.getRootLogger().addAppender(defaultAppender);
    Logger.getRootLogger().setLevel(Level.INFO);
    Logger.getRootLogger().info("NOTE. This tool works only with V2/V1 checkpoints");
    CheckpointPersistenceProvider cp3 = null;
    if (null != _cp3Props) {
        CheckpointPersistenceStaticConfigBuilder cp3ConfBuilder = new CheckpointPersistenceStaticConfigBuilder();
        ConfigLoader<CheckpointPersistenceStaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.CheckpointPersistenceStaticConfig>(_propPrefix, cp3ConfBuilder);
        configLoader.loadConfig(_cp3Props);
        CheckpointPersistenceStaticConfig cp3Conf = cp3ConfBuilder.build();
        if (cp3Conf.getType() != CheckpointPersistenceStaticConfig.ProviderType.FILE_SYSTEM) {
            throw new RuntimeException("don't know what to do with cp3 type:" + cp3Conf.getType());
        }
        cp3 = new FileSystemCheckpointPersistenceProvider(cp3Conf.getFileSystem(), 2);
    } else if (null != _clientProps) {
        DatabusHttpClientImpl.Config clientConfBuilder = new DatabusHttpClientImpl.Config();
        ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>(_propPrefix, clientConfBuilder);
        configLoader.loadConfig(_clientProps);
        DatabusHttpClientImpl.StaticConfig clientConf = clientConfBuilder.build();
        if (clientConf.getCheckpointPersistence().getType() != CheckpointPersistenceStaticConfig.ProviderType.FILE_SYSTEM) {
            throw new RuntimeException("don't know what to do with cp3 type:" + clientConf.getCheckpointPersistence().getType());
        }
        cp3 = new FileSystemCheckpointPersistenceProvider(clientConf.getCheckpointPersistence().getFileSystem(), 2);
    }
    List<String> sourceList = Arrays.asList(_sources);
    Checkpoint cpOld = null != cp3 ? cp3.loadCheckpoint(sourceList) : new Checkpoint();
    Checkpoint cpNew;
    if (Action.PRINT == _action) {
        cpNew = updateCheckpoint(cpOld);
    } else if (Action.CHANGE == _action) {
        cpNew = updateCheckpoint(cpOld);
        cp3.storeCheckpoint(sourceList, cpNew);
        //reread as a sanity check
        cpNew = cp3.loadCheckpoint(sourceList);
    } else if (Action.DELETE == _action) {
        cp3.removeCheckpoint(sourceList);
        cpNew = cp3.loadCheckpoint(sourceList);
    } else {
        throw new RuntimeException("don't know what to do with action: " + _action);
    }
    if (null != cpOld)
        System.out.println("old: " + cpOld.toString());
    else
        System.out.println("old: null");
    if (null != cpNew)
        System.out.println("new: " + cpNew.toString());
    else
        System.out.println("new: null");
}