Examples with PhysicalSourceConfig com.linkedin.databus2.relay.config.PhysicalSourceConfig used on opensource projects

Example 11 with PhysicalSourceConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.

the class DatabusRelayTestUtil method createPhysicalConfigBuilder.

public static PhysicalSourceConfig createPhysicalConfigBuilder(short id, String name, String uri, long pollIntervalMs, int eventRatePerSec, long restartScnOffset, int largestEventSize, long largestWindowSize, LogicalSourceConfig[] lSourceConfigs) {
    PhysicalSourceConfig pConfig = new PhysicalSourceConfig();
    pConfig.setId(id);
    pConfig.setName(name);
    pConfig.setUri(uri);
    pConfig.setEventRatePerSec(eventRatePerSec);
    pConfig.setRestartScnOffset(restartScnOffset);
    pConfig.setLargestEventSizeInBytes(largestEventSize);
    pConfig.setLargestWindowSizeInBytes(largestWindowSize);
    BackoffTimerStaticConfigBuilder retriesConf = new BackoffTimerStaticConfigBuilder();
    retriesConf.setInitSleep(pollIntervalMs);
    pConfig.setRetries(retriesConf);
    for (LogicalSourceConfig lConf : lSourceConfigs) {
        pConfig.addSource(lConf);
    }
    return pConfig;
}

Example 12 with PhysicalSourceConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.

the class TestDatabusRelayMain method testRelayChainingSCNRegress.

@Test
public /**
	 * Test regression of SCN  of chained relay
	 */
void testRelayChainingSCNRegress() throws InvalidConfigException, InterruptedException {
    final Logger log = Logger.getLogger("TestDatabusRelayMain.testRelayChainingSCNRegress");
    DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null, r3 = null;
    ClientRunner cr = null;
    //log.setLevel(Level.DEBUG);
    log.info("start");
    //DbusEventBuffer.LOG.setLevel(Level.DEBUG);
    try {
        String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
        // create main relay with random generator
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 10;
        int largestEventSize = 512 * 1024;
        long largestWindowSize = 1 * 1024 * 1024;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = 11993;
        final DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1019, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        final DatabusRelayMain relay3 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1020, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertNotNull(relay1);
        Assert.assertNotNull(relay3);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        final DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        final DbusEventsTotalStats stats3 = relay3.getInboundEventStatisticsCollector().getTotalStats();
        // create chained relay
        PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
        int j = 0;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, 500, eventRatePerSec, 0, largestEventSize, largestWindowSize, srcs);
            chainedSrcConfigs[j++] = src1;
        }
        int chainedRelayPort = relayPort + 1;
        final DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1021, chainedRelayPort, 10 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertNotNull(relay2);
        r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
        resetSCN(relay2);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + chainedRelayPort;
        CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, largestEventSize, 30 * 1000, 100, 15 * 1000, 1, true, largestEventSize / 10);
        cr = new ClientRunner(clientConn);
        // async starts for all components;
        r1.start();
        Thread.sleep(10 * 1000);
        // start chained relay
        r2.start();
        //start client
        cr.start();
        //Pause r1;
        r1.pause();
        Thread.sleep(1000);
        long firstGenDataEvents = stats.getNumDataEvents();
        long firstGenMinScn = stats.getMinScn();
        long firstGenWindows = stats.getNumSysEvents();
        Assert.assertTrue(stats.getNumSysEvents() > 0);
        log.warn("relay1:  numDataEvents=" + firstGenDataEvents + " numWindows=" + firstGenWindows + " minScn=" + firstGenMinScn + " maxScn=" + stats.getMaxScn());
        Thread.sleep(4 * 1000);
        //clear the relay
        boolean s = r1.shutdown(2000);
        Assert.assertTrue(s);
        DbusEventsTotalStats stats2 = relay2.getInboundEventStatisticsCollector().getTotalStats();
        long firstGenChainWindows = stats2.getNumSysEvents();
        log.warn("relay2:  numDataEvents=" + stats2.getNumDataEvents() + " numWindows=" + firstGenChainWindows + " minScn=" + stats2.getMinScn() + " maxScn=" + stats2.getMaxScn());
        Thread.sleep(2 * 1000);
        //restart relay
        r3 = new DatabusRelayTestUtil.RelayRunner(relay3);
        r3.start();
        Thread.sleep(15 * 1000);
        r3.pause();
        Thread.sleep(35 * 1000);
        log.warn("relay3:  numDataEvents=" + stats3.getNumDataEvents() + " numWindows=" + stats3.getNumSysEvents() + " minScn=" + stats3.getMinScn() + " maxScn=" + stats3.getMaxScn());
        stats2 = relay2.getInboundEventStatisticsCollector().getTotalStats();
        log.warn("relay2b: numDataEvents=" + stats2.getNumDataEvents() + " numWindows=" + stats2.getNumSysEvents() + " minScn=" + stats2.getMinScn() + " maxScn=" + stats2.getMaxScn());
        log.warn("consumer: " + countingConsumer);
        //compare chained relays with 2 gens of tier 0 relays
        Assert.assertEquals(stats2.getMinScn(), firstGenMinScn);
        Assert.assertEquals(stats2.getMaxScn(), stats3.getMaxScn());
        //the total event windows seen by the chained relay will be state of consumption at first failure of relay1 minus 1 overlap window
        Assert.assertEquals(stats2.getNumSysEvents(), (firstGenChainWindows - 1) + stats3.getNumSysEvents());
        Assert.assertTrue(stats2.getNumDataEvents() > stats3.getNumDataEvents());
        //compare source to final client
        Assert.assertEquals(countingConsumer.getNumSources(), 2);
        Assert.assertEquals(stats2.getNumSysEvents(), countingConsumer.getNumWindows());
        boolean sorted = true;
        long prev = -1;
        log.info(" scn seq on consumer=");
        for (Long l : countingConsumer.getEndScns()) {
            sorted = sorted && (l >= prev);
            prev = l;
            log.info(l + " ");
            if (!sorted)
                break;
        }
        Assert.assertTrue(sorted);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2, r3 }, cr);
        log.info("end");
    }
}

Example 13 with PhysicalSourceConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.

the class TestDatabusRelayMain method testRelayChainingDelayedConsumer.

@Test
public /**
	 * Chained relay overwrites buffer; consumer starts late; gets only the last n events that fit.
	 * Makes sure chained relay overwrites buffer;just like main buffer
	 */
void testRelayChainingDelayedConsumer() {
    DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
        // create main relay with random generator
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 10;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = 11993;
        DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1010, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        // create chained relay with only 1 MB buffer
        PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
        int j = 0;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
            chainedSrcConfigs[j++] = src1;
        }
        int chainedRelayPort = relayPort + 1;
        DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1011, chainedRelayPort, 1 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay2);
        resetSCN(relay2);
        r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + chainedRelayPort;
        CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        // async starts for all components;
        r1.start();
        // start chained relay
        r2.start();
        // let it run for 10 seconds
        Thread.sleep(10 * 1000);
        r1.pause();
        // start client after the relays have finished
        cr.start();
        // wait until client got all events or for maxTimeout;
        long maxTimeOutMs = 15 * 1000;
        long startTime = System.currentTimeMillis();
        DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
            Thread.sleep(500);
            // stats.getSizeDataEvents());
            if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
                break;
            }
        }
        DbusEventsTotalStats statsChained = relay2.getInboundEventStatisticsCollector().getTotalStats();
        LOG.info("Client stats=" + countingConsumer);
        LOG.info("Event windows generated=" + stats.getNumSysEvents());
        LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
        Assert.assertTrue(stats.getNumDataEvents() == statsChained.getNumDataEvents());
        Assert.assertTrue(stats.getNumSysEvents() == statsChained.getNumSysEvents());
        Assert.assertTrue(stats.getNumDataEvents() > countingConsumer.getNumDataEvents());
        Assert.assertTrue(countingConsumer.getNumSources() == 2);
        Assert.assertTrue(stats.getNumSysEvents() > countingConsumer.getNumWindows());
    } catch (Exception e) {
        LOG.error("Exception: " + e);
        Assert.assertTrue(false);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
    }
}

Example 14 with PhysicalSourceConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.

the class TestDatabusRelayMain method testRelalyChainingPartialSubscribe.

@Test
public /**
	 * Client consumes subset of sources from chained relay
	 */
void testRelalyChainingPartialSubscribe() {
    DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
        // create main relay with random generator
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 10;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = 11993;
        DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1006, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        // create chained relay
        PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
        int j = 0;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
            chainedSrcConfigs[j++] = src1;
        }
        int chainedRelayPort = relayPort + 1;
        DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1007, chainedRelayPort, 10 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay2);
        resetSCN(relay2);
        r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
        // now create client:
        String srcSubscriptionString = "com.linkedin.events.example.Settings";
        String serverName = "localhost:" + chainedRelayPort;
        CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, -1, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        // start relay r1
        r1.start();
        // start chained relay
        r2.start();
        // start client
        cr.start();
        // Let the relay run for 10s
        Thread.sleep(10 * 1000);
        r1.pause();
        // wait until client got all events or for maxTimeout;
        long maxTimeOutMs = 5 * 1000;
        long startTime = System.currentTimeMillis();
        DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
            Thread.sleep(500);
            // stats.getSizeDataEvents());
            if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
                break;
            }
        }
        LOG.info("Client stats=" + countingConsumer);
        LOG.info("Event windows generated=" + stats.getNumSysEvents());
        LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
        Assert.assertTrue(countingConsumer.getNumSources() == 1);
        Assert.assertTrue(stats.getNumSysEvents() == countingConsumer.getNumWindows());
        Assert.assertTrue(stats.getNumDataEvents() == 2 * countingConsumer.getNumDataEvents());
    } catch (Exception e) {
        LOG.error("Exception: " + e);
        Assert.assertTrue(false);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
    }
}

Example 15 with PhysicalSourceConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.

the class TestDatabusRelayMain method testRelayChainingRestartSCNOffset.

@Test
public /**
	 * Test relay chaining that utilizes restart SCN offset
	 */
void testRelayChainingRestartSCNOffset() throws InterruptedException, InvalidConfigException {
    final Logger log = Logger.getLogger("TestDatabusRelayMain.testRelayChainingRestartSCNOffset");
    log.info("start");
    DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null, r3 = null;
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
        // create main relay with random generator
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 2;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = 11993;
        DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1016, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        // create chained relay
        PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
        int j = 0;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
            chainedSrcConfigs[j++] = src1;
        }
        int chainedRelayPort = relayPort + 1;
        DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1017, chainedRelayPort, 1 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay2);
        r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
        resetSCN(relay2);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + chainedRelayPort;
        final CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        // async starts for all components;
        r1.start();
        // start chained relay
        r2.start();
        // start client
        Thread.sleep(500);
        cr.start();
        r1.pause();
        // let it run for 10 seconds
        Thread.sleep(10 * 1000);
        // wait until client got all events or for maxTimeout;
        final long maxTimeOutMs = 5 * 1000;
        final DbusEventsTotalStats dbRelayStats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return countingConsumer.getNumWindows() == dbRelayStats.getNumSysEvents();
            }
        }, "consumer caught up", maxTimeOutMs, log);
        log.info("Client stats=" + countingConsumer);
        log.info("Event windows generated=" + dbRelayStats.getNumSysEvents());
        log.info("numDataEvents=" + dbRelayStats.getNumDataEvents() + " numWindows=" + dbRelayStats.getNumSysEvents() + " size=" + dbRelayStats.getSizeDataEvents());
        Assert.assertEquals(dbRelayStats.getNumDataEvents(), countingConsumer.getNumDataEvents());
        Assert.assertEquals(countingConsumer.getNumSources(), 2);
        Assert.assertEquals(dbRelayStats.getNumSysEvents(), countingConsumer.getNumWindows());
        cr.shutdown();
        boolean s2 = r2.shutdown(2000);
        Assert.assertTrue(s2);
        Assert.assertTrue(!r2.isAlive());
        //start r3; new chained relay with restart SCN offset; we get some data;
        chainedSrcConfigs[0].setRestartScnOffset(dbRelayStats.getMaxScn() - dbRelayStats.getPrevScn());
        DatabusRelayMain relay3 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1018, chainedRelayPort, 1 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        r3 = new DatabusRelayTestUtil.RelayRunner(relay3);
        r3.start();
        final DbusEventsTotalStats newChainedRlyStats = relay3.getInboundEventStatisticsCollector().getTotalStats();
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return newChainedRlyStats.getNumDataEvents() > 0;
            }
        }, "new chained relay running", 5000, log);
        log.info("Stats3= numDataEvents=" + newChainedRlyStats.getNumDataEvents() + " numWindows=" + newChainedRlyStats.getNumSysEvents() + " size=" + newChainedRlyStats.getSizeDataEvents());
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2, r3 }, cr);
        log.info("end");
    }
}