Examples with DatabusRelayMain com.linkedin.databus2.relay.DatabusRelayMain used on opensource projects

Example 11 with DatabusRelayMain

use of com.linkedin.databus2.relay.DatabusRelayMain in project databus by linkedin.

the class TestDatabusRelayEvents method testEventConversion.

@Test
public /**
   * append event V2 to the buffer and stream it to the client
   * which only accepts events V1. Make sure it got converted
   */
void testEventConversion() throws InterruptedException, IOException, DatabusException {
    final Logger log = Logger.getLogger("TestDatabusRelayEvents.testEventConversion");
    log.setLevel(Level.INFO);
    DatabusRelayTestUtil.RelayRunner r1 = null;
    ClientRunner cr = null;
    try {
        String[] srcs = { "com.linkedin.events.example.fake.FakeSchema" };
        int pId = 1;
        int srcId = 2;
        int relayPort = Utils.getAvailablePort(11994);
        ;
        final DatabusRelayMain relay1 = createRelay(relayPort, pId, srcs);
        Assert.assertNotNull(relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        log.info("Relay created");
        DbusEventBufferMult bufMult = relay1.getEventBuffer();
        String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);
        PhysicalPartition pPartition = new PhysicalPartition(pId, pSourceName);
        DbusEventBufferAppendable buf = bufMult.getDbusEventBufferAppendable(pPartition);
        DbusEventKey key = new DbusEventKey(123L);
        byte[] schemaId = relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray();
        byte[] payload = RngUtils.randomString(100).getBytes(Charset.defaultCharset());
        DbusEventInfo eventInfo = new DbusEventInfo(DbusOpcode.UPSERT, 100L, (short) pId, (short) pId, 897L, (short) srcId, schemaId, payload, false, true);
        eventInfo.setEventSerializationVersion(DbusEventFactory.DBUS_EVENT_V2);
        buf.startEvents();
        buf.appendEvent(key, eventInfo, null);
        buf.endEvents(100L, null);
        r1.start();
        log.info("Relay started");
        // wait until relay comes up
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return relay1.isRunningStatus();
            }
        }, "Relay hasn't come up completely ", 7000, LOG);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcs, ",");
        String serverName = "localhost:" + relayPort;
        final EventsCountingConsumer countingConsumer = new EventsCountingConsumer();
        int id = (RngUtils.randomPositiveInt() % 10000) + 1;
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V1, 0);
        cr = new ClientRunner(clientConn);
        cr.start();
        log.info("Consumer started");
        // wait till client gets the event
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return countingConsumer.getNumDataEvents() == 1;
            }
        }, "Consumer didn't get any events ", 64 * 1024, LOG);
        // asserts
        Assert.assertEquals(1, countingConsumer.getNumDataEvents());
        Assert.assertEquals(1, countingConsumer.getNumWindows());
        Assert.assertEquals(1, countingConsumer.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1));
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}

Example 12 with DatabusRelayMain

use of com.linkedin.databus2.relay.DatabusRelayMain in project databus by linkedin.

the class TestDatabusRelayMain method testRelayEventGenerator.

@Test
public void testRelayEventGenerator() throws InterruptedException, InvalidConfigException {
    DatabusRelayTestUtil.RelayRunner r1 = null;
    //Logger.getRootLogger().setLevel(Level.INFO);
    final Logger log = Logger.getLogger("TestDatabusRelayMain.testRelayEventGenerator");
    //log.setLevel(Level.DEBUG);
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.fake.FakeSchema", "com.linkedin.events.example.person.Person" } };
        log.info("create main relay with random generator");
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 20;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = Utils.getAvailablePort(11993);
        final DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1001, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertNotEquals(relay1, null);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        log.info("async starts");
        r1.start();
        log.info("start client in parallel");
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + relayPort;
        final CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        cr.start();
        log.info("terminating conditions");
        final DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        long totalRunTime = 5000;
        long startTime = System.currentTimeMillis();
        do {
            log.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
            Thread.sleep(1000);
        } while ((System.currentTimeMillis() - startTime) < totalRunTime);
        r1.pause();
        log.info("Sending pause to relay!");
        log.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                boolean success = true;
                for (EventProducer p : relay1.getProducers()) {
                    if (!(success = success && p.isPaused()))
                        break;
                }
                return success;
            }
        }, "waiting for producers to pause", 4000, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("countingConsumer.getNumWindows()=" + countingConsumer.getNumWindows());
                return countingConsumer.getNumWindows() == stats.getNumSysEvents();
            }
        }, "wait until client got all events or for maxTimeout", 64 * 1024, log);
        log.info("Client stats=" + countingConsumer);
        log.info("Event windows generated=" + stats.getNumSysEvents());
        cr.shutdown(2000, log);
        log.info("Client cr stopped");
        Assert.assertEquals(countingConsumer.getNumDataEvents(), stats.getNumDataEvents());
        boolean stopped = r1.shutdown(2000);
        Assert.assertTrue(stopped);
        log.info("Relay r1 stopped");
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}

Example 13 with DatabusRelayMain

use of com.linkedin.databus2.relay.DatabusRelayMain in project databus by linkedin.

the class TestDatabusRelayMain method testPendingEventSize.

@Test
public /**
   * When the relay has no events, we should not get the x-dbus-pending-event-size even if we present a small buffer.
   * When the relay has events, we should see the header on a small buffer but see an event when the buffer
   * is large enough, and should not see the header in the large buffer case.
   */
void testPendingEventSize() throws Exception {
    DatabusRelayMain relay = null;
    try {
        final short srcId = 104;
        final String srcName = "foo";
        PhysicalSourceConfig pConfig = new PhysicalSourceConfig();
        pConfig.setId(srcId);
        pConfig.setName(srcName);
        pConfig.setUri("mock");
        short lid = (short) (srcId + 1);
        LogicalSourceConfig lConf = new LogicalSourceConfig();
        lConf.setId(lid);
        lConf.setName(srcName);
        // this is table name in the oracle source world
        lConf.setUri(srcName);
        lConf.setPartitionFunction("constant:1");
        pConfig.addSource(lConf);
        int relayPort = Utils.getAvailablePort(11994);
        final int relayId = 666;
        HttpRelay.Config httpRelayConfig = new HttpRelay.Config();
        ServerContainer.Config containerConfig = DatabusRelayTestUtil.createContainerConfig(relayId, relayPort);
        DbusEventBuffer.Config bufferConfig = DatabusRelayTestUtil.createBufferConfig(10000, 250, 100);
        httpRelayConfig.setContainer(containerConfig);
        httpRelayConfig.setEventBuffer(bufferConfig);
        httpRelayConfig.setStartDbPuller("true");
        PhysicalSourceStaticConfig[] pStaticConfigs = new PhysicalSourceStaticConfig[1];
        for (LogicalSourceConfig lsc : pConfig.getSources()) {
            httpRelayConfig.setSourceName("" + lsc.getId(), lsc.getName());
        }
        pStaticConfigs[0] = pConfig.build();
        relay = new DatabusRelayMain(httpRelayConfig.build(), pStaticConfigs);
        relay.start();
        // Insert one event into the relay.
        LogicalSource lsrc = new LogicalSource((int) lid, srcName);
        DbusEventBuffer buf = relay.getEventBuffer().getDbusEventBuffer(lsrc);
        byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
        final long prevScn = 99;
        final long eventScn = 101;
        buf.start(prevScn);
        buf.startEvents();
        Assert.assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, lid, schema, new byte[100], false, null));
        buf.endEvents(eventScn, null);
        HttpResponseHandler handler = new HttpResponseHandler();
        // On a good buffer length we should not see the extra header.
        testClient(relayPort, 1000, 100L, handler);
        Assert.assertNull(handler._pendingEventHeader, "Received pending event header on full buffer");
        // We should see the extra header when we get 0 events and the next event is too big to fit in
        testClient(relayPort, 10, 100L, handler);
        Assert.assertNotNull(handler._pendingEventHeader);
        Assert.assertEquals(Integer.valueOf(handler._pendingEventHeader).intValue(), 161);
        // But if there are no events, then we should not see the header even if buffer is very small
        handler._pendingEventHeader = null;
        testClient(relayPort, 10, 1005L, handler);
        Assert.assertNull(handler._pendingEventHeader, "Received pending event header on full buffer");
    } finally {
        relay.shutdownUninteruptibly();
    }
}

Example 14 with DatabusRelayMain

use of com.linkedin.databus2.relay.DatabusRelayMain in project databus by linkedin.

the class TestDatabusRelayMain method testRelayChainingSlowConsumer.

@Test
public /**
	 * Slow consumer; results in getting SCN not found exception ; as chained relay overwrites buffer .
	 * Makes sure chained relay overwrites buffer;just like main buffer
	 */
void testRelayChainingSlowConsumer() {
    DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
        // create main relay with random generator
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 10;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = 11993;
        DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1012, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        // create chained relay with only 1 MB buffer
        PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
        int j = 0;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
            chainedSrcConfigs[j++] = src1;
        }
        int chainedRelayPort = relayPort + 1;
        DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1013, chainedRelayPort, 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay2);
        resetSCN(relay2);
        r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + chainedRelayPort;
        // create slow consumer with 100ms delay
        CountingConsumer countingConsumer = new CountingConsumer(500);
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 1000, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        // async starts for all components;
        r1.start();
        // start chained relay
        r2.start();
        // start slow client
        cr.start();
        // let it run for 20 seconds
        Thread.sleep(20 * 1000);
        r1.pause();
        // wait until client got all events or for maxTimeout;
        long maxTimeOutMs = 100 * 1000;
        long startTime = System.currentTimeMillis();
        DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        Assert.assertTrue(stats.getNumSysEvents() > 0);
        while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
            Thread.sleep(500);
            if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
                break;
            }
        }
        DbusEventsTotalStats statsChained = relay2.getInboundEventStatisticsCollector().getTotalStats();
        LOG.info("Client stats=" + countingConsumer);
        LOG.info("Event windows generated=" + stats.getNumSysEvents());
        LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
        Assert.assertTrue(stats.getMinScn() < statsChained.getMinScn());
        Assert.assertTrue(stats.getNumDataEvents() == statsChained.getNumDataEvents());
        Assert.assertTrue(stats.getNumSysEvents() == statsChained.getNumSysEvents());
        Assert.assertTrue(countingConsumer.getNumErrors() > 0);
        Assert.assertTrue(stats.getNumDataEvents() > countingConsumer.getNumDataEvents());
        Assert.assertTrue(countingConsumer.getNumSources() == 2);
        Assert.assertTrue(stats.getNumSysEvents() > countingConsumer.getNumWindows());
    } catch (Exception e) {
        LOG.error("Exception: " + e);
        Assert.assertTrue(false);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
    }
}

Example 15 with DatabusRelayMain

use of com.linkedin.databus2.relay.DatabusRelayMain in project databus by linkedin.

the class TestDatabusRelayMain method testRelayChainingConnectFailure.

@Test
public /** Client and chained relay start before relay, simulating unavailable relay and retries at chained relay and client
	 *
	 */
void testRelayChainingConnectFailure() {
    DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.fake.FakeSchema", "com.linkedin.events.example.person.Person" } };
        // create main relay with random generator
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 10;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = 11993;
        DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1004, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        // create chained relay
        PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
        int j = 0;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
            chainedSrcConfigs[j++] = src1;
        }
        int chainedRelayPort = relayPort + 1;
        DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1005, chainedRelayPort, 10 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay2);
        r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
        resetSCN(relay2);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + chainedRelayPort;
        CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        // start chained relay
        r2.start();
        // start client
        cr.start();
        // Let them try for 5seconds
        Thread.sleep(5 * 1000);
        r1.start();
        // Let the relay run for 10s
        Thread.sleep(10 * 1000);
        r1.pause();
        // wait until client got all events or for maxTimeout;
        long maxTimeOutMs = 5 * 1000;
        long startTime = System.currentTimeMillis();
        DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
            Thread.sleep(500);
            // stats.getSizeDataEvents());
            if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
                break;
            }
        }
        LOG.info("Client stats=" + countingConsumer);
        LOG.info("Event windows generated=" + stats.getNumSysEvents());
        LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
        Assert.assertEquals(stats.getNumDataEvents(), countingConsumer.getNumDataEvents());
        Assert.assertEquals(countingConsumer.getNumSources(), 2);
        Assert.assertEquals(stats.getNumSysEvents(), countingConsumer.getNumWindows());
    } catch (Exception e) {
        LOG.error("Exception: " + e);
        Assert.assertTrue(false);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
    }
}