Examples with PhysicalSourceConfig com.linkedin.databus2.relay.config.PhysicalSourceConfig used on opensource projects

Example 21 with PhysicalSourceConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.

the class BootstrapSeederMain method init.

public static void init(String[] args) throws Exception {
    parseArgs(args);
    // Load the source configuration JSON file
    //File sourcesJson = new File("integration-test/config/sources-member2.json");
    File sourcesJson = new File(_sSourcesConfigFile);
    ObjectMapper mapper = new ObjectMapper();
    PhysicalSourceConfig physicalSourceConfig = mapper.readValue(sourcesJson, PhysicalSourceConfig.class);
    physicalSourceConfig.checkForNulls();
    Config config = new Config();
    ConfigLoader<StaticConfig> configLoader = new ConfigLoader<StaticConfig>("databus.seed.", config);
    _sStaticConfig = configLoader.loadConfig(_sBootstrapConfigProps);
    // Make sure the URI from the configuration file identifies an Oracle JDBC source.
    String uri = physicalSourceConfig.getUri();
    if (!uri.startsWith("jdbc:oracle")) {
        throw new InvalidConfigException("Invalid source URI (" + physicalSourceConfig.getUri() + "). Only jdbc:oracle: URIs are supported.");
    }
    String sourceTypeStr = physicalSourceConfig.getReplBitSetter().getSourceType();
    if (SourceType.TOKEN.toString().equalsIgnoreCase(sourceTypeStr))
        throw new InvalidConfigException("Token Source-type for Replication bit setter config cannot be set for trigger-based Databus relay !!");
    // Create the OracleDataSource used to get DB connection(s)
    try {
        Class oracleDataSourceClass = OracleJarUtils.loadClass("oracle.jdbc.pool.OracleDataSource");
        Object ods = oracleDataSourceClass.newInstance();
        Method setURLMethod = oracleDataSourceClass.getMethod("setURL", String.class);
        setURLMethod.invoke(ods, uri);
        _sDataStore = (DataSource) ods;
    } catch (Exception e) {
        String errMsg = "Error creating a data source object ";
        LOG.error(errMsg, e);
        throw e;
    }
    //TODO: Need a better way than relaying on RelayFactory for generating MonitoredSourceInfo
    OracleEventProducerFactory factory = new BootstrapSeederOracleEventProducerFactory(_sStaticConfig.getController().getPKeyNameMap());
    // Parse each one of the logical sources
    _sources = new ArrayList<OracleTriggerMonitoredSourceInfo>();
    FileSystemSchemaRegistryService schemaRegistryService = FileSystemSchemaRegistryService.build(_sStaticConfig.getSchemaRegistry().getFileSystem());
    Set<String> seenUris = new HashSet<String>();
    for (LogicalSourceConfig sourceConfig : physicalSourceConfig.getSources()) {
        String srcUri = sourceConfig.getUri();
        if (seenUris.contains(srcUri)) {
            String msg = "Uri (" + srcUri + ") is used for more than one sources. Currently Bootstrap Seeder cannot support seeding sources with the same URI together. Please have them run seperately !!";
            LOG.fatal(msg);
            throw new InvalidConfigException(msg);
        }
        seenUris.add(srcUri);
        OracleTriggerMonitoredSourceInfo source = factory.buildOracleMonitoredSourceInfo(sourceConfig.build(), physicalSourceConfig.build(), schemaRegistryService);
        _sources.add(source);
    }
    _sSeeder = new BootstrapDBSeeder(_sStaticConfig.getBootstrap(), _sources);
    _sBootstrapBuffer = new BootstrapEventBuffer(_sStaticConfig.getController().getCommitInterval() * 2);
    _sWriterThread = new BootstrapSeederWriterThread(_sBootstrapBuffer, _sSeeder);
    _sReader = new BootstrapSrcDBEventReader(_sDataStore, _sBootstrapBuffer, _sStaticConfig.getController(), _sources, _sSeeder.getLastRows(), _sSeeder.getLastKeys(), 0);
}

Example 22 with PhysicalSourceConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.

the class TestDevRelaySrcConfigGen method testDevRelaySrcConfigGen.

@Test
public /**
   *
   * Steps:
   * 1. Generate new Schema_registry with some example schemas.
   * 2. Construct schemaDataMAnager and generate RelayDevConfigs for the example DB.
   * 3. Open the newly generated json file containing physical-sources config and de-serialize to construct a physicalSourcesConfig object
   * 4. Ensure physicalSourcesConfig object have expected source configurations.
   *
   * @throws Exception
   */
void testDevRelaySrcConfigGen() throws Exception {
    // Schema
    String[] personSchema = { "{\"type\":\"record\",\"name\":\"Person\",\"namespace\":\"com.linkedin.events.example\",\"fields\":[{\"name\":\"txn\",\"type\":[\"long\",\"null\"],\"meta\":\"dbFieldName=TXN;dbFieldPosition=0;\"},{\"name\":\"key\",\"type\":[\"long\",\"null\"],\"meta\":\"dbFieldName=KEY;dbFieldPosition=1;\"},{\"name\":\"firstName\",\"type\":[\"string\",\"null\"],\"meta\":\"dbFieldName=FIRST_NAME;dbFieldPosition=2;\"},{\"name\":\"lastName\",\"type\":[\"string\",\"null\"],\"meta\":\"dbFieldName=LAST_NAME;dbFieldPosition=3;\"},{\"name\":\"birthDate\",\"type\":[\"long\",\"null\"],\"meta\":\"dbFieldName=BIRTH_DATE;dbFieldPosition=4;\"},{\"name\":\"deleted\",\"type\":[\"string\",\"null\"],\"meta\":\"dbFieldName=DELETED;dbFieldPosition=5;\"}],\"meta\":\"dbFieldName=sy$person;\"}" };
    String[] addressSchema = { "{\"type\":\"record\",\"name\":\"Address\",\"namespace\":\"com.linkedin.events.example\",\"fields\":[{\"name\":\"txn\",\"type\":[\"long\",\"null\"],\"meta\":\"dbFieldName=TXN;dbFieldPosition=0;\"},{\"name\":\"key\",\"type\":[\"long\",\"null\"],\"meta\":\"dbFieldName=KEY;dbFieldPosition=1;\"},{\"name\":\"addressLine1\",\"type\":[\"string\",\"null\"],\"meta\":\"dbFieldName=ADDRESS_LINE_1;dbFieldPosition=2;\"},{\"name\":\"addressLine2\",\"type\":[\"string\",\"null\"],\"meta\":\"dbFieldName=ADDRESS_LINE_2;dbFieldPosition=3;\"},{\"name\":\"zipCode\",\"type\":[\"long\",\"null\"],\"meta\":\"dbFieldName=ZIP_CODE;dbFieldPosition=4;\"},{\"name\":\"deleted\",\"type\":[\"string\",\"null\"],\"meta\":\"dbFieldName=DELETED;dbFieldPosition=5;\"}],\"meta\":\"dbFieldName=sy$address;\"}" };
    // Index Schema registry contents
    String[] indexContents = { "com.linkedin.events.example.person.avsc", "com.linkedin.events.example.address.avsc" };
    // Meta-Data contents
    String[] idNameMapContents = { "1:com.linkedin.events.example.person", "2:com.linkedin.events.example.address" };
    String[] dbToSrcMapContents = { "{", "  \"example\" : [ \"com.linkedin.events.example.Address\", \"com.linkedin.events.example.Person\" ]", "}" };
    File schemaDir = FileUtils.createTempDir("dir_testDevRelaySrcConfigGen");
    String dbToSrcFile1 = schemaDir.getAbsolutePath() + "/physical_logical_src_map.json";
    String idNameMapFile1 = schemaDir.getAbsolutePath() + "/idToName.map";
    String indexRegistry = schemaDir.getAbsolutePath() + "/index.schemas_registry";
    String personSchemaFile = schemaDir.getAbsolutePath() + "/com.linkedin.events.example.person.1.avsc";
    String addressSchemaFile = schemaDir.getAbsolutePath() + "/com.linkedin.events.example.address.1.avsc";
    FileUtils.storeLinesToTempFile(dbToSrcFile1, dbToSrcMapContents);
    FileUtils.storeLinesToTempFile(idNameMapFile1, idNameMapContents);
    FileUtils.storeLinesToTempFile(indexRegistry, indexContents);
    FileUtils.storeLinesToTempFile(personSchemaFile, personSchema);
    FileUtils.storeLinesToTempFile(addressSchemaFile, addressSchema);
    SchemaMetaDataManager mgr = new SchemaMetaDataManager(schemaDir.getAbsolutePath());
    String uri = "jdbc:oracle:thin:liar/liar@devdb:DB";
    List<String> exampleSrcs = Arrays.asList("com.linkedin.events.example.person", "com.linkedin.events.example.address");
    DevRelayConfigGenerator.generateRelayConfig(schemaDir.getAbsolutePath(), "example", uri, schemaDir.getAbsolutePath(), exampleSrcs, mgr);
    File f = new File(schemaDir.getAbsolutePath() + "/sources-example.json");
    Assert.assertTrue(f.exists(), "Physical Src Config file present?");
    BufferedReader r = new BufferedReader(new FileReader(f));
    String json = r.readLine();
    PhysicalSourceConfig config = PhysicalSourceConfig.fromString(json);
    config.checkForNulls();
    Assert.assertEquals(config.getUri(), uri);
    Assert.assertEquals(config.getName(), "example");
    List<LogicalSourceConfig> srcs = config.getSources();
    Assert.assertEquals(srcs.size(), 2);
    Assert.assertEquals(srcs.get(0).getId(), 1);
    Assert.assertEquals(srcs.get(0).getName(), "com.linkedin.events.example.person");
    Assert.assertEquals(srcs.get(0).getUri(), "example.sy$person");
    Assert.assertEquals(srcs.get(1).getId(), 2);
    Assert.assertEquals(srcs.get(1).getName(), "com.linkedin.events.example.address");
    Assert.assertEquals(srcs.get(1).getUri(), "example.sy$address");
}

Example 23 with PhysicalSourceConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.

the class TestDatabusRelayMain method testRelayChainingSlowConsumer.

@Test
public /**
	 * Slow consumer; results in getting SCN not found exception ; as chained relay overwrites buffer .
	 * Makes sure chained relay overwrites buffer;just like main buffer
	 */
void testRelayChainingSlowConsumer() {
    DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
        // create main relay with random generator
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 10;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = 11993;
        DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1012, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        // create chained relay with only 1 MB buffer
        PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
        int j = 0;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
            chainedSrcConfigs[j++] = src1;
        }
        int chainedRelayPort = relayPort + 1;
        DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1013, chainedRelayPort, 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay2);
        resetSCN(relay2);
        r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + chainedRelayPort;
        // create slow consumer with 100ms delay
        CountingConsumer countingConsumer = new CountingConsumer(500);
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 1000, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        // async starts for all components;
        r1.start();
        // start chained relay
        r2.start();
        // start slow client
        cr.start();
        // let it run for 20 seconds
        Thread.sleep(20 * 1000);
        r1.pause();
        // wait until client got all events or for maxTimeout;
        long maxTimeOutMs = 100 * 1000;
        long startTime = System.currentTimeMillis();
        DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        Assert.assertTrue(stats.getNumSysEvents() > 0);
        while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
            Thread.sleep(500);
            if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
                break;
            }
        }
        DbusEventsTotalStats statsChained = relay2.getInboundEventStatisticsCollector().getTotalStats();
        LOG.info("Client stats=" + countingConsumer);
        LOG.info("Event windows generated=" + stats.getNumSysEvents());
        LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
        Assert.assertTrue(stats.getMinScn() < statsChained.getMinScn());
        Assert.assertTrue(stats.getNumDataEvents() == statsChained.getNumDataEvents());
        Assert.assertTrue(stats.getNumSysEvents() == statsChained.getNumSysEvents());
        Assert.assertTrue(countingConsumer.getNumErrors() > 0);
        Assert.assertTrue(stats.getNumDataEvents() > countingConsumer.getNumDataEvents());
        Assert.assertTrue(countingConsumer.getNumSources() == 2);
        Assert.assertTrue(stats.getNumSysEvents() > countingConsumer.getNumWindows());
    } catch (Exception e) {
        LOG.error("Exception: " + e);
        Assert.assertTrue(false);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
    }
}

Example 24 with PhysicalSourceConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.

the class TestDatabusRelayMain method testRelayChainingConnectFailure.

@Test
public /** Client and chained relay start before relay, simulating unavailable relay and retries at chained relay and client
	 *
	 */
void testRelayChainingConnectFailure() {
    DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.fake.FakeSchema", "com.linkedin.events.example.person.Person" } };
        // create main relay with random generator
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 10;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = 11993;
        DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1004, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        // create chained relay
        PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
        int j = 0;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
            chainedSrcConfigs[j++] = src1;
        }
        int chainedRelayPort = relayPort + 1;
        DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1005, chainedRelayPort, 10 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay2);
        r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
        resetSCN(relay2);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + chainedRelayPort;
        CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        // start chained relay
        r2.start();
        // start client
        cr.start();
        // Let them try for 5seconds
        Thread.sleep(5 * 1000);
        r1.start();
        // Let the relay run for 10s
        Thread.sleep(10 * 1000);
        r1.pause();
        // wait until client got all events or for maxTimeout;
        long maxTimeOutMs = 5 * 1000;
        long startTime = System.currentTimeMillis();
        DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
            Thread.sleep(500);
            // stats.getSizeDataEvents());
            if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
                break;
            }
        }
        LOG.info("Client stats=" + countingConsumer);
        LOG.info("Event windows generated=" + stats.getNumSysEvents());
        LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
        Assert.assertEquals(stats.getNumDataEvents(), countingConsumer.getNumDataEvents());
        Assert.assertEquals(countingConsumer.getNumSources(), 2);
        Assert.assertEquals(stats.getNumSysEvents(), countingConsumer.getNumWindows());
    } catch (Exception e) {
        LOG.error("Exception: " + e);
        Assert.assertTrue(false);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
    }
}

Example 25 with PhysicalSourceConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.

the class TestDatabusRelayMain method testRelayChainingPauseResume.

@Test
public /**
	 * Regular concurrent consumption amongst db-relay , chained relay and client. The only twist is that the chained relay
	 * will be paused and resumed
	 */
void testRelayChainingPauseResume() {
    DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
        // create main relay with random generator
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 10;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = 11993;
        DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1014, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        // create chained relay
        PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
        int j = 0;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
            chainedSrcConfigs[j++] = src1;
        }
        int chainedRelayPort = relayPort + 1;
        DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1015, chainedRelayPort, 1 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay2);
        resetSCN(relay2);
        r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + chainedRelayPort;
        CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        // async starts for all components;
        r1.start();
        // start chained relay
        r2.start();
        // start client
        cr.start();
        // let them run for 2 seconds
        Thread.sleep(2 * 1000);
        // pause chained relay
        r2.pause();
        // Sleep again for some time
        Thread.sleep(2 * 1000);
        // stop relay
        r1.pause();
        // resume chained relay
        r2.unpause();
        // wait until client got all events or for maxTimeout;
        long maxTimeOutMs = 5 * 1000;
        long startTime = System.currentTimeMillis();
        DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
            Thread.sleep(500);
            // stats.getSizeDataEvents());
            if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
                break;
            }
        }
        LOG.info("Client stats=" + countingConsumer);
        LOG.info("Event windows generated=" + stats.getNumSysEvents());
        LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
        Assert.assertTrue(stats.getNumDataEvents() == countingConsumer.getNumDataEvents());
        Assert.assertTrue(countingConsumer.getNumSources() == 2);
        Assert.assertTrue(stats.getNumSysEvents() == countingConsumer.getNumWindows());
    } catch (Exception e) {
        LOG.error("Exception: " + e);
        Assert.assertTrue(false);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
    }
}