Examples with LogicalSourceConfig com.linkedin.databus2.relay.config.LogicalSourceConfig used on opensource projects

Example 6 with LogicalSourceConfig

use of com.linkedin.databus2.relay.config.LogicalSourceConfig in project databus by linkedin.

the class DatabusRelayTestUtil method createPhysicalConfigBuilder.

public static PhysicalSourceConfig createPhysicalConfigBuilder(short id, String name, String uri, long pollIntervalMs, int eventRatePerSec, long restartScnOffset, int largestEventSize, long largestWindowSize, LogicalSourceConfig[] lSourceConfigs) {
    PhysicalSourceConfig pConfig = new PhysicalSourceConfig();
    pConfig.setId(id);
    pConfig.setName(name);
    pConfig.setUri(uri);
    pConfig.setEventRatePerSec(eventRatePerSec);
    pConfig.setRestartScnOffset(restartScnOffset);
    pConfig.setLargestEventSizeInBytes(largestEventSize);
    pConfig.setLargestWindowSizeInBytes(largestWindowSize);
    BackoffTimerStaticConfigBuilder retriesConf = new BackoffTimerStaticConfigBuilder();
    retriesConf.setInitSleep(pollIntervalMs);
    pConfig.setRetries(retriesConf);
    for (LogicalSourceConfig lConf : lSourceConfigs) {
        pConfig.addSource(lConf);
    }
    return pConfig;
}

Example 7 with LogicalSourceConfig

use of com.linkedin.databus2.relay.config.LogicalSourceConfig in project databus by linkedin.

the class TestDatabusRelayMain method testPendingEventSize.

@Test
public /**
   * When the relay has no events, we should not get the x-dbus-pending-event-size even if we present a small buffer.
   * When the relay has events, we should see the header on a small buffer but see an event when the buffer
   * is large enough, and should not see the header in the large buffer case.
   */
void testPendingEventSize() throws Exception {
    DatabusRelayMain relay = null;
    try {
        final short srcId = 104;
        final String srcName = "foo";
        PhysicalSourceConfig pConfig = new PhysicalSourceConfig();
        pConfig.setId(srcId);
        pConfig.setName(srcName);
        pConfig.setUri("mock");
        short lid = (short) (srcId + 1);
        LogicalSourceConfig lConf = new LogicalSourceConfig();
        lConf.setId(lid);
        lConf.setName(srcName);
        // this is table name in the oracle source world
        lConf.setUri(srcName);
        lConf.setPartitionFunction("constant:1");
        pConfig.addSource(lConf);
        int relayPort = Utils.getAvailablePort(11994);
        final int relayId = 666;
        HttpRelay.Config httpRelayConfig = new HttpRelay.Config();
        ServerContainer.Config containerConfig = DatabusRelayTestUtil.createContainerConfig(relayId, relayPort);
        DbusEventBuffer.Config bufferConfig = DatabusRelayTestUtil.createBufferConfig(10000, 250, 100);
        httpRelayConfig.setContainer(containerConfig);
        httpRelayConfig.setEventBuffer(bufferConfig);
        httpRelayConfig.setStartDbPuller("true");
        PhysicalSourceStaticConfig[] pStaticConfigs = new PhysicalSourceStaticConfig[1];
        for (LogicalSourceConfig lsc : pConfig.getSources()) {
            httpRelayConfig.setSourceName("" + lsc.getId(), lsc.getName());
        }
        pStaticConfigs[0] = pConfig.build();
        relay = new DatabusRelayMain(httpRelayConfig.build(), pStaticConfigs);
        relay.start();
        // Insert one event into the relay.
        LogicalSource lsrc = new LogicalSource((int) lid, srcName);
        DbusEventBuffer buf = relay.getEventBuffer().getDbusEventBuffer(lsrc);
        byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
        final long prevScn = 99;
        final long eventScn = 101;
        buf.start(prevScn);
        buf.startEvents();
        Assert.assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, lid, schema, new byte[100], false, null));
        buf.endEvents(eventScn, null);
        HttpResponseHandler handler = new HttpResponseHandler();
        // On a good buffer length we should not see the extra header.
        testClient(relayPort, 1000, 100L, handler);
        Assert.assertNull(handler._pendingEventHeader, "Received pending event header on full buffer");
        // We should see the extra header when we get 0 events and the next event is too big to fit in
        testClient(relayPort, 10, 100L, handler);
        Assert.assertNotNull(handler._pendingEventHeader);
        Assert.assertEquals(Integer.valueOf(handler._pendingEventHeader).intValue(), 161);
        // But if there are no events, then we should not see the header even if buffer is very small
        handler._pendingEventHeader = null;
        testClient(relayPort, 10, 1005L, handler);
        Assert.assertNull(handler._pendingEventHeader, "Received pending event header on full buffer");
    } finally {
        relay.shutdownUninteruptibly();
    }
}

Example 8 with LogicalSourceConfig

use of com.linkedin.databus2.relay.config.LogicalSourceConfig in project databus by linkedin.

the class BootstrapAvroFileSeederMain method init.

public static void init(String[] args) throws Exception {
    parseArgs(args);
    File sourcesJson = new File(_sSourcesConfigFile);
    ObjectMapper mapper = new ObjectMapper();
    PhysicalSourceConfig physicalSourceConfig = mapper.readValue(sourcesJson, PhysicalSourceConfig.class);
    physicalSourceConfig.checkForNulls();
    Config config = new Config();
    ConfigLoader<StaticConfig> configLoader = new ConfigLoader<StaticConfig>("databus.seed.", config);
    _sStaticConfig = configLoader.loadConfig(_sBootstrapConfigProps);
    // Make sure the URI from the configuration file identifies an Oracle JDBC source.
    String uri = physicalSourceConfig.getUri();
    if (!uri.startsWith("jdbc:oracle")) {
        throw new InvalidConfigException("Invalid source URI (" + physicalSourceConfig.getUri() + "). Only jdbc:oracle: URIs are supported.");
    }
    OracleEventProducerFactory factory = new BootstrapSeederOracleEventProducerFactory(_sStaticConfig.getController().getPKeyNameMap());
    // Parse each one of the logical sources
    _sources = new ArrayList<OracleTriggerMonitoredSourceInfo>();
    FileSystemSchemaRegistryService schemaRegistryService = FileSystemSchemaRegistryService.build(_sStaticConfig.getSchemaRegistry().getFileSystem());
    for (LogicalSourceConfig sourceConfig : physicalSourceConfig.getSources()) {
        OracleTriggerMonitoredSourceInfo source = factory.buildOracleMonitoredSourceInfo(sourceConfig.build(), physicalSourceConfig.build(), schemaRegistryService);
        _sources.add(source);
    }
    _sSeeder = new BootstrapDBSeeder(_sStaticConfig.getBootstrap(), _sources);
    _sBootstrapBuffer = new BootstrapEventBuffer(_sStaticConfig.getController().getCommitInterval() * 2);
    _sWriterThread = new BootstrapSeederWriterThread(_sBootstrapBuffer, _sSeeder);
    _sReader = new BootstrapAvroFileEventReader(_sStaticConfig.getController(), _sources, _sSeeder.getLastRows(), _sBootstrapBuffer);
}

Example 9 with LogicalSourceConfig

use of com.linkedin.databus2.relay.config.LogicalSourceConfig in project databus by linkedin.

the class BootstrapSeederMain method init.

public static void init(String[] args) throws Exception {
    parseArgs(args);
    // Load the source configuration JSON file
    //File sourcesJson = new File("integration-test/config/sources-member2.json");
    File sourcesJson = new File(_sSourcesConfigFile);
    ObjectMapper mapper = new ObjectMapper();
    PhysicalSourceConfig physicalSourceConfig = mapper.readValue(sourcesJson, PhysicalSourceConfig.class);
    physicalSourceConfig.checkForNulls();
    Config config = new Config();
    ConfigLoader<StaticConfig> configLoader = new ConfigLoader<StaticConfig>("databus.seed.", config);
    _sStaticConfig = configLoader.loadConfig(_sBootstrapConfigProps);
    // Make sure the URI from the configuration file identifies an Oracle JDBC source.
    String uri = physicalSourceConfig.getUri();
    if (!uri.startsWith("jdbc:oracle")) {
        throw new InvalidConfigException("Invalid source URI (" + physicalSourceConfig.getUri() + "). Only jdbc:oracle: URIs are supported.");
    }
    String sourceTypeStr = physicalSourceConfig.getReplBitSetter().getSourceType();
    if (SourceType.TOKEN.toString().equalsIgnoreCase(sourceTypeStr))
        throw new InvalidConfigException("Token Source-type for Replication bit setter config cannot be set for trigger-based Databus relay !!");
    // Create the OracleDataSource used to get DB connection(s)
    try {
        Class oracleDataSourceClass = OracleJarUtils.loadClass("oracle.jdbc.pool.OracleDataSource");
        Object ods = oracleDataSourceClass.newInstance();
        Method setURLMethod = oracleDataSourceClass.getMethod("setURL", String.class);
        setURLMethod.invoke(ods, uri);
        _sDataStore = (DataSource) ods;
    } catch (Exception e) {
        String errMsg = "Error creating a data source object ";
        LOG.error(errMsg, e);
        throw e;
    }
    //TODO: Need a better way than relaying on RelayFactory for generating MonitoredSourceInfo
    OracleEventProducerFactory factory = new BootstrapSeederOracleEventProducerFactory(_sStaticConfig.getController().getPKeyNameMap());
    // Parse each one of the logical sources
    _sources = new ArrayList<OracleTriggerMonitoredSourceInfo>();
    FileSystemSchemaRegistryService schemaRegistryService = FileSystemSchemaRegistryService.build(_sStaticConfig.getSchemaRegistry().getFileSystem());
    Set<String> seenUris = new HashSet<String>();
    for (LogicalSourceConfig sourceConfig : physicalSourceConfig.getSources()) {
        String srcUri = sourceConfig.getUri();
        if (seenUris.contains(srcUri)) {
            String msg = "Uri (" + srcUri + ") is used for more than one sources. Currently Bootstrap Seeder cannot support seeding sources with the same URI together. Please have them run seperately !!";
            LOG.fatal(msg);
            throw new InvalidConfigException(msg);
        }
        seenUris.add(srcUri);
        OracleTriggerMonitoredSourceInfo source = factory.buildOracleMonitoredSourceInfo(sourceConfig.build(), physicalSourceConfig.build(), schemaRegistryService);
        _sources.add(source);
    }
    _sSeeder = new BootstrapDBSeeder(_sStaticConfig.getBootstrap(), _sources);
    _sBootstrapBuffer = new BootstrapEventBuffer(_sStaticConfig.getController().getCommitInterval() * 2);
    _sWriterThread = new BootstrapSeederWriterThread(_sBootstrapBuffer, _sSeeder);
    _sReader = new BootstrapSrcDBEventReader(_sDataStore, _sBootstrapBuffer, _sStaticConfig.getController(), _sources, _sSeeder.getLastRows(), _sSeeder.getLastKeys(), 0);
}

Example 10 with LogicalSourceConfig

use of com.linkedin.databus2.relay.config.LogicalSourceConfig in project databus by linkedin.

the class TestDevRelaySrcConfigGen method testDevRelaySrcConfigGen.

@Test
public /**
   *
   * Steps:
   * 1. Generate new Schema_registry with some example schemas.
   * 2. Construct schemaDataMAnager and generate RelayDevConfigs for the example DB.
   * 3. Open the newly generated json file containing physical-sources config and de-serialize to construct a physicalSourcesConfig object
   * 4. Ensure physicalSourcesConfig object have expected source configurations.
   *
   * @throws Exception
   */
void testDevRelaySrcConfigGen() throws Exception {
    // Schema
    String[] personSchema = { "{\"type\":\"record\",\"name\":\"Person\",\"namespace\":\"com.linkedin.events.example\",\"fields\":[{\"name\":\"txn\",\"type\":[\"long\",\"null\"],\"meta\":\"dbFieldName=TXN;dbFieldPosition=0;\"},{\"name\":\"key\",\"type\":[\"long\",\"null\"],\"meta\":\"dbFieldName=KEY;dbFieldPosition=1;\"},{\"name\":\"firstName\",\"type\":[\"string\",\"null\"],\"meta\":\"dbFieldName=FIRST_NAME;dbFieldPosition=2;\"},{\"name\":\"lastName\",\"type\":[\"string\",\"null\"],\"meta\":\"dbFieldName=LAST_NAME;dbFieldPosition=3;\"},{\"name\":\"birthDate\",\"type\":[\"long\",\"null\"],\"meta\":\"dbFieldName=BIRTH_DATE;dbFieldPosition=4;\"},{\"name\":\"deleted\",\"type\":[\"string\",\"null\"],\"meta\":\"dbFieldName=DELETED;dbFieldPosition=5;\"}],\"meta\":\"dbFieldName=sy$person;\"}" };
    String[] addressSchema = { "{\"type\":\"record\",\"name\":\"Address\",\"namespace\":\"com.linkedin.events.example\",\"fields\":[{\"name\":\"txn\",\"type\":[\"long\",\"null\"],\"meta\":\"dbFieldName=TXN;dbFieldPosition=0;\"},{\"name\":\"key\",\"type\":[\"long\",\"null\"],\"meta\":\"dbFieldName=KEY;dbFieldPosition=1;\"},{\"name\":\"addressLine1\",\"type\":[\"string\",\"null\"],\"meta\":\"dbFieldName=ADDRESS_LINE_1;dbFieldPosition=2;\"},{\"name\":\"addressLine2\",\"type\":[\"string\",\"null\"],\"meta\":\"dbFieldName=ADDRESS_LINE_2;dbFieldPosition=3;\"},{\"name\":\"zipCode\",\"type\":[\"long\",\"null\"],\"meta\":\"dbFieldName=ZIP_CODE;dbFieldPosition=4;\"},{\"name\":\"deleted\",\"type\":[\"string\",\"null\"],\"meta\":\"dbFieldName=DELETED;dbFieldPosition=5;\"}],\"meta\":\"dbFieldName=sy$address;\"}" };
    // Index Schema registry contents
    String[] indexContents = { "com.linkedin.events.example.person.avsc", "com.linkedin.events.example.address.avsc" };
    // Meta-Data contents
    String[] idNameMapContents = { "1:com.linkedin.events.example.person", "2:com.linkedin.events.example.address" };
    String[] dbToSrcMapContents = { "{", "  \"example\" : [ \"com.linkedin.events.example.Address\", \"com.linkedin.events.example.Person\" ]", "}" };
    File schemaDir = FileUtils.createTempDir("dir_testDevRelaySrcConfigGen");
    String dbToSrcFile1 = schemaDir.getAbsolutePath() + "/physical_logical_src_map.json";
    String idNameMapFile1 = schemaDir.getAbsolutePath() + "/idToName.map";
    String indexRegistry = schemaDir.getAbsolutePath() + "/index.schemas_registry";
    String personSchemaFile = schemaDir.getAbsolutePath() + "/com.linkedin.events.example.person.1.avsc";
    String addressSchemaFile = schemaDir.getAbsolutePath() + "/com.linkedin.events.example.address.1.avsc";
    FileUtils.storeLinesToTempFile(dbToSrcFile1, dbToSrcMapContents);
    FileUtils.storeLinesToTempFile(idNameMapFile1, idNameMapContents);
    FileUtils.storeLinesToTempFile(indexRegistry, indexContents);
    FileUtils.storeLinesToTempFile(personSchemaFile, personSchema);
    FileUtils.storeLinesToTempFile(addressSchemaFile, addressSchema);
    SchemaMetaDataManager mgr = new SchemaMetaDataManager(schemaDir.getAbsolutePath());
    String uri = "jdbc:oracle:thin:liar/liar@devdb:DB";
    List<String> exampleSrcs = Arrays.asList("com.linkedin.events.example.person", "com.linkedin.events.example.address");
    DevRelayConfigGenerator.generateRelayConfig(schemaDir.getAbsolutePath(), "example", uri, schemaDir.getAbsolutePath(), exampleSrcs, mgr);
    File f = new File(schemaDir.getAbsolutePath() + "/sources-example.json");
    Assert.assertTrue(f.exists(), "Physical Src Config file present?");
    BufferedReader r = new BufferedReader(new FileReader(f));
    String json = r.readLine();
    PhysicalSourceConfig config = PhysicalSourceConfig.fromString(json);
    config.checkForNulls();
    Assert.assertEquals(config.getUri(), uri);
    Assert.assertEquals(config.getName(), "example");
    List<LogicalSourceConfig> srcs = config.getSources();
    Assert.assertEquals(srcs.size(), 2);
    Assert.assertEquals(srcs.get(0).getId(), 1);
    Assert.assertEquals(srcs.get(0).getName(), "com.linkedin.events.example.person");
    Assert.assertEquals(srcs.get(0).getUri(), "example.sy$person");
    Assert.assertEquals(srcs.get(1).getId(), 2);
    Assert.assertEquals(srcs.get(1).getName(), "com.linkedin.events.example.address");
    Assert.assertEquals(srcs.get(1).getUri(), "example.sy$address");
}