Examples with PhysicalSourceConfig com.linkedin.databus2.relay.config.PhysicalSourceConfig used on opensource projects

Example 16 with PhysicalSourceConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.

the class TestDatabusRelayMain method testClientNoEventsResetConnection.

@Test
public /**
	 * test resetting connection when there is no events for some period of time
	 * @throws InterruptedException
	 * @throws InvalidConfigException
	 */
void testClientNoEventsResetConnection() throws InterruptedException, InvalidConfigException {
    LOG.setLevel(Level.ALL);
    DatabusRelayTestUtil.RelayRunner r1 = null;
    ClientRunner cr = null;
    try {
        String srcName = "com.linkedin.events.example.Settings";
        // create main relay with random generator
        int eventRatePerSec = 10;
        PhysicalSourceConfig srcConfig = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) 1, DatabusRelayTestUtil.getPhysicalSrcName(srcName), "mock", 500, eventRatePerSec, new String[] { srcName });
        int relayPort = 11995;
        DatabusRelayMain relay = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1001, relayPort, 10 * 1024 * 1024, new PhysicalSourceConfig[] { srcConfig }, SCHEMA_REGISTRY_DIR);
        Assert.assertNotEquals(relay, null);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay);
        // async starts
        r1.start();
        DbusEventsTotalStats stats = relay.getInboundEventStatisticsCollector().getTotalStats();
        // start client in parallel
        String srcSubscriptionString = srcName;
        String serverName = "localhost:" + relayPort;
        ResetsCountingConsumer countingConsumer = new ResetsCountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        cr.start();
        // generate some events
        TestUtil.sleep(1000);
        // pause event generator
        // and wait untill all the events are consumed
        // but since it is less then timeout the connection should NOT reset
        LOG.info("Sending pause to relay!");
        r1.pause();
        TestUtil.sleep(4000);
        LOG.info("no events, time less then threshold. Events=" + countingConsumer.getNumDataEvents() + "; resets = " + countingConsumer.getNumResets());
        Assert.assertEquals(countingConsumer.getNumResets(), 0);
        // generate more events, more time elapsed then the threshold, but since there are
        // events - NO reset
        r1.unpause();
        Thread.sleep(8000);
        LOG.info("some events, more time then timeout. Events=" + countingConsumer.getNumDataEvents() + "; resets = " + countingConsumer.getNumResets());
        Assert.assertEquals(countingConsumer.getNumResets(), 0);
        // stop events
        r1.pause();
        //set threshold to 0 completely disabling the feature
        clientConn.getRelayPullThread().setNoEventsConnectionResetTimeSec(0);
        Thread.sleep(8000);
        LOG.info("no events, more time then timeout, but feature disabled. Events=" + countingConsumer.getNumDataEvents() + "; resets = " + countingConsumer.getNumResets());
        Assert.assertEquals(countingConsumer.getNumResets(), 0);
        // enable the feature, and sleep for timeout
        clientConn.getRelayPullThread().setNoEventsConnectionResetTimeSec(5);
        // now wait with no events
        LOG.info("pause the producer. sleep for 6 sec, should reset");
        TestUtil.sleep(6000);
        LOG.info("Client stats=" + countingConsumer);
        LOG.info("Num resets=" + countingConsumer.getNumResets());
        LOG.info("Event windows generated=" + stats.getNumSysEvents());
        Assert.assertEquals(countingConsumer.getNumResets(), 0, "0 resets");
        Assert.assertEquals(countingConsumer.getNumDataEvents(), stats.getNumDataEvents());
        boolean stopped = r1.shutdown(2000);
        Assert.assertTrue(stopped);
        LOG.info("Relay r1 stopped");
        cr.shutdown();
        LOG.info("Client cr stopped");
        Assert.assertEquals(countingConsumer.getNumDataEvents(), stats.getNumDataEvents());
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}

Example 17 with PhysicalSourceConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.

the class TestDatabusRelayEvents method createRelay.

/**
   * create a test relay with event producer turned off
   */
private DatabusRelayMain createRelay(int relayPort, int pId, String[] srcs) throws IOException, DatabusException {
    // create main relay with random generator
    PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcs.length];
    String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);
    PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) pId, pSourceName, "mock", 500, 0, srcs);
    srcConfigs[0] = src1;
    HttpRelay.Config httpRelayConfig = DatabusRelayTestUtil.createHttpRelayConfig(1002, relayPort, 3024);
    // do not produce any events
    httpRelayConfig.setStartDbPuller("false");
    httpRelayConfig.getSchemaRegistry().getFileSystem().setSchemaDir("TestDatabusRelayEvents_schemas");
    final DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelay(srcConfigs, httpRelayConfig);
    Assert.assertNotNull(relay1);
    return relay1;
}

Example 18 with PhysicalSourceConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.

the class TestDatabusRelayMain method testRelayEventGenerator.

@Test
public void testRelayEventGenerator() throws InterruptedException, InvalidConfigException {
    DatabusRelayTestUtil.RelayRunner r1 = null;
    //Logger.getRootLogger().setLevel(Level.INFO);
    final Logger log = Logger.getLogger("TestDatabusRelayMain.testRelayEventGenerator");
    //log.setLevel(Level.DEBUG);
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.fake.FakeSchema", "com.linkedin.events.example.person.Person" } };
        log.info("create main relay with random generator");
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 20;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = Utils.getAvailablePort(11993);
        final DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1001, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertNotEquals(relay1, null);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        log.info("async starts");
        r1.start();
        log.info("start client in parallel");
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + relayPort;
        final CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        cr.start();
        log.info("terminating conditions");
        final DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        long totalRunTime = 5000;
        long startTime = System.currentTimeMillis();
        do {
            log.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
            Thread.sleep(1000);
        } while ((System.currentTimeMillis() - startTime) < totalRunTime);
        r1.pause();
        log.info("Sending pause to relay!");
        log.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                boolean success = true;
                for (EventProducer p : relay1.getProducers()) {
                    if (!(success = success && p.isPaused()))
                        break;
                }
                return success;
            }
        }, "waiting for producers to pause", 4000, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("countingConsumer.getNumWindows()=" + countingConsumer.getNumWindows());
                return countingConsumer.getNumWindows() == stats.getNumSysEvents();
            }
        }, "wait until client got all events or for maxTimeout", 64 * 1024, log);
        log.info("Client stats=" + countingConsumer);
        log.info("Event windows generated=" + stats.getNumSysEvents());
        cr.shutdown(2000, log);
        log.info("Client cr stopped");
        Assert.assertEquals(countingConsumer.getNumDataEvents(), stats.getNumDataEvents());
        boolean stopped = r1.shutdown(2000);
        Assert.assertTrue(stopped);
        log.info("Relay r1 stopped");
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}

Example 19 with PhysicalSourceConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.

the class TestDatabusRelayMain method testPendingEventSize.

@Test
public /**
   * When the relay has no events, we should not get the x-dbus-pending-event-size even if we present a small buffer.
   * When the relay has events, we should see the header on a small buffer but see an event when the buffer
   * is large enough, and should not see the header in the large buffer case.
   */
void testPendingEventSize() throws Exception {
    DatabusRelayMain relay = null;
    try {
        final short srcId = 104;
        final String srcName = "foo";
        PhysicalSourceConfig pConfig = new PhysicalSourceConfig();
        pConfig.setId(srcId);
        pConfig.setName(srcName);
        pConfig.setUri("mock");
        short lid = (short) (srcId + 1);
        LogicalSourceConfig lConf = new LogicalSourceConfig();
        lConf.setId(lid);
        lConf.setName(srcName);
        // this is table name in the oracle source world
        lConf.setUri(srcName);
        lConf.setPartitionFunction("constant:1");
        pConfig.addSource(lConf);
        int relayPort = Utils.getAvailablePort(11994);
        final int relayId = 666;
        HttpRelay.Config httpRelayConfig = new HttpRelay.Config();
        ServerContainer.Config containerConfig = DatabusRelayTestUtil.createContainerConfig(relayId, relayPort);
        DbusEventBuffer.Config bufferConfig = DatabusRelayTestUtil.createBufferConfig(10000, 250, 100);
        httpRelayConfig.setContainer(containerConfig);
        httpRelayConfig.setEventBuffer(bufferConfig);
        httpRelayConfig.setStartDbPuller("true");
        PhysicalSourceStaticConfig[] pStaticConfigs = new PhysicalSourceStaticConfig[1];
        for (LogicalSourceConfig lsc : pConfig.getSources()) {
            httpRelayConfig.setSourceName("" + lsc.getId(), lsc.getName());
        }
        pStaticConfigs[0] = pConfig.build();
        relay = new DatabusRelayMain(httpRelayConfig.build(), pStaticConfigs);
        relay.start();
        // Insert one event into the relay.
        LogicalSource lsrc = new LogicalSource((int) lid, srcName);
        DbusEventBuffer buf = relay.getEventBuffer().getDbusEventBuffer(lsrc);
        byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
        final long prevScn = 99;
        final long eventScn = 101;
        buf.start(prevScn);
        buf.startEvents();
        Assert.assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, lid, schema, new byte[100], false, null));
        buf.endEvents(eventScn, null);
        HttpResponseHandler handler = new HttpResponseHandler();
        // On a good buffer length we should not see the extra header.
        testClient(relayPort, 1000, 100L, handler);
        Assert.assertNull(handler._pendingEventHeader, "Received pending event header on full buffer");
        // We should see the extra header when we get 0 events and the next event is too big to fit in
        testClient(relayPort, 10, 100L, handler);
        Assert.assertNotNull(handler._pendingEventHeader);
        Assert.assertEquals(Integer.valueOf(handler._pendingEventHeader).intValue(), 161);
        // But if there are no events, then we should not see the header even if buffer is very small
        handler._pendingEventHeader = null;
        testClient(relayPort, 10, 1005L, handler);
        Assert.assertNull(handler._pendingEventHeader, "Received pending event header on full buffer");
    } finally {
        relay.shutdownUninteruptibly();
    }
}

Example 20 with PhysicalSourceConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.

the class BootstrapAvroFileSeederMain method init.

public static void init(String[] args) throws Exception {
    parseArgs(args);
    File sourcesJson = new File(_sSourcesConfigFile);
    ObjectMapper mapper = new ObjectMapper();
    PhysicalSourceConfig physicalSourceConfig = mapper.readValue(sourcesJson, PhysicalSourceConfig.class);
    physicalSourceConfig.checkForNulls();
    Config config = new Config();
    ConfigLoader<StaticConfig> configLoader = new ConfigLoader<StaticConfig>("databus.seed.", config);
    _sStaticConfig = configLoader.loadConfig(_sBootstrapConfigProps);
    // Make sure the URI from the configuration file identifies an Oracle JDBC source.
    String uri = physicalSourceConfig.getUri();
    if (!uri.startsWith("jdbc:oracle")) {
        throw new InvalidConfigException("Invalid source URI (" + physicalSourceConfig.getUri() + "). Only jdbc:oracle: URIs are supported.");
    }
    OracleEventProducerFactory factory = new BootstrapSeederOracleEventProducerFactory(_sStaticConfig.getController().getPKeyNameMap());
    // Parse each one of the logical sources
    _sources = new ArrayList<OracleTriggerMonitoredSourceInfo>();
    FileSystemSchemaRegistryService schemaRegistryService = FileSystemSchemaRegistryService.build(_sStaticConfig.getSchemaRegistry().getFileSystem());
    for (LogicalSourceConfig sourceConfig : physicalSourceConfig.getSources()) {
        OracleTriggerMonitoredSourceInfo source = factory.buildOracleMonitoredSourceInfo(sourceConfig.build(), physicalSourceConfig.build(), schemaRegistryService);
        _sources.add(source);
    }
    _sSeeder = new BootstrapDBSeeder(_sStaticConfig.getBootstrap(), _sources);
    _sBootstrapBuffer = new BootstrapEventBuffer(_sStaticConfig.getController().getCommitInterval() * 2);
    _sWriterThread = new BootstrapSeederWriterThread(_sBootstrapBuffer, _sSeeder);
    _sReader = new BootstrapAvroFileEventReader(_sStaticConfig.getController(), _sources, _sSeeder.getLastRows(), _sBootstrapBuffer);
}