use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.
the class TestDatabusRelayMain method testDynamicBufferGrowthClient.
@Test
public /**
* Basic consumption client and by chained relay from regular relay
* but start with small readBuffer (20K) - Set up large events
*/
void testDynamicBufferGrowthClient() throws InterruptedException, InvalidConfigException {
DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
final Logger log = Logger.getLogger("TestDatabusRelayMain.testDynamicBufferGrowth");
Logger.getRootLogger().setLevel(Level.INFO);
ClientRunner cr = null;
try {
String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
// create main relay with random generator
PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
int i = 0;
int eventRatePerSec = 20;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 100, eventRatePerSec, srcs);
srcConfigs[i++] = src1;
}
int relayPort = Utils.getAvailablePort(11994);
DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1002, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertNotNull(relay1);
r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
log.info("now create client");
String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
String serverName = "localhost:" + relayPort;
final CountingConsumer countingConsumer = new CountingConsumer();
//Set maxSize to 100K, maxEventSize to 15k, set init readBufferSize to maxEventSize/2
int maxEventSize = 15 * 1024;
int initReadBufferSize = maxEventSize / 2;
DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 100 * 1024, maxEventSize, 30 * 1000, 100, 30 * 1000, 1, true, initReadBufferSize);
cr = new ClientRunner(clientConn);
log.info("async starts for all components");
r1.start();
assertRelayRunning(r1.getRelay(), 5000, log);
Thread.sleep(5 * 1000);
r1.pause();
final DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
log.info("start the client");
cr.start();
// wait until client got all events or for maxTimeout;
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("countingConsumer.getNumWindows()=" + countingConsumer.getNumWindows());
return countingConsumer.getNumWindows() == stats.getNumSysEvents();
}
}, "wait until client got all events", 10000, log);
LOG.info("Client stats=" + countingConsumer);
LOG.info("Event windows generated=" + stats.getNumSysEvents());
LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
Assert.assertEquals(stats.getNumDataEvents(), countingConsumer.getNumDataEvents());
Assert.assertEquals(countingConsumer.getNumSources(), 2);
Assert.assertEquals(stats.getNumSysEvents(), countingConsumer.getNumWindows());
} finally {
cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
}
}
use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.
the class TestGoldenGateEventProducer method buildSimplePssc.
private PhysicalSourceStaticConfig buildSimplePssc(short[] sourceIds, String[] sourceNames, String uri) throws InvalidConfigException {
String partitionFunction = "constant:1";
String pPartName = "part1";
short pPartId = 0;
PhysicalSourceConfig pc = new PhysicalSourceConfig(pPartName, uri, pPartId);
for (int i = 0; i < sourceIds.length; i++) {
LogicalSourceConfig lc = new LogicalSourceConfig();
lc.setId(sourceIds[i]);
lc.setName(sourceNames[i]);
lc.setPartitionFunction(partitionFunction);
// this format is expected by GG
lc.setUri(pPartName + "." + sourceNames[i]);
pc.addSource(lc);
}
return pc.build();
}
use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.
the class TestDatabusRelayMain method testRelayChainingBasic.
@Test
public /**
* Concurrent consumption of chained relay by client and by chained relay from regular relay
*/
void testRelayChainingBasic() throws InterruptedException, InvalidConfigException {
DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
final Logger log = Logger.getLogger("TestDatabusRelayMain.testRelayChainingBasic");
Logger.getRootLogger().setLevel(Level.INFO);
log.setLevel(Level.DEBUG);
log.debug("available processors:" + Runtime.getRuntime().availableProcessors());
log.debug("available memory:" + Runtime.getRuntime().freeMemory());
log.debug("total memory:" + Runtime.getRuntime().totalMemory());
ClientRunner cr = null;
try {
String[][] srcNames = { { "com.linkedin.events.example.fake.FakeSchema", "com.linkedin.events.example.person.Person" } };
// create main relay with random generator
PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
int i = 0;
int eventRatePerSec = 20;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 100, eventRatePerSec, srcs);
srcConfigs[i++] = src1;
}
int relayPort = Utils.getAvailablePort(11994);
DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1002, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertNotNull(relay1);
r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
log.info("create chained relay");
PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
int j = 0;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
chainedSrcConfigs[j++] = src1;
}
int chainedRelayPort = relayPort + 1000;
DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1003, chainedRelayPort, 1 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertNotNull(relay2);
r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
resetSCN(relay2);
log.info("now create client");
String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
String serverName = "localhost:" + chainedRelayPort;
final CountingConsumer countingConsumer = new CountingConsumer();
//Set maxSize to 100K, maxEventSize to 50K
DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 100 * 1024, 10000, 30 * 1000, 100, 30 * 1000, 1, true);
cr = new ClientRunner(clientConn);
log.info("async starts for all components");
r1.start();
assertRelayRunning(r1.getRelay(), 5000, log);
log.info("start chained relay");
r2.start();
assertRelayRunning(r2.getRelay(), 5000, log);
Thread.sleep(5 * 1000);
r1.pause();
// wait for r2 to catchup with r1
final DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
final DbusEventsTotalStats stats2 = relay2.getInboundEventStatisticsCollector().getTotalStats();
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("stats2.getNumSysEvents()=" + stats2.getNumSysEvents());
return stats2.getNumSysEvents() == stats.getNumSysEvents();
}
}, "wait for chained relay to catchup", 60000, log);
log.info("start the client");
cr.start();
// wait until client got all events or for maxTimeout;
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("countingConsumer.getNumWindows()=" + countingConsumer.getNumWindows());
return countingConsumer.getNumWindows() == stats.getNumSysEvents();
}
}, "wait until client got all events", 10000, log);
LOG.info("Client stats=" + countingConsumer);
LOG.info("Event windows generated=" + stats.getNumSysEvents());
LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
LOG.info("numDataEvents2=" + stats2.getNumDataEvents() + " numWindows2=" + stats2.getNumSysEvents() + " size2=" + stats2.getSizeDataEvents());
Assert.assertEquals(stats.getNumDataEvents(), countingConsumer.getNumDataEvents());
Assert.assertEquals(countingConsumer.getNumSources(), 2);
Assert.assertEquals(stats.getNumSysEvents(), countingConsumer.getNumWindows());
} finally {
cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
}
}
use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.
the class TestDatabusRelayMain method testRelayChainingPartialSubscribeRelay.
@Test
public /**
* Chained relay consumes a subset of parent relay
*/
void testRelayChainingPartialSubscribeRelay() {
DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
ClientRunner cr = null;
try {
String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
// create main relay with random generator
PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
int i = 0;
int eventRatePerSec = 10;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
srcConfigs[i++] = src1;
}
int relayPort = 11993;
DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1008, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay1);
r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
// create chained relay
PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
int j = 0;
for (String[] srcs : srcNames) {
String[] partialSrcs = new String[1];
partialSrcs[0] = srcs[srcs.length - 1];
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (srcs.length), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, partialSrcs);
chainedSrcConfigs[j++] = src1;
}
int chainedRelayPort = relayPort + 1;
DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1009, chainedRelayPort, 10 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay2);
resetSCN(relay2);
r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
// now create client:
String srcSubscriptionString = "com.linkedin.events.example.Settings";
String serverName = "localhost:" + chainedRelayPort;
CountingConsumer countingConsumer = new CountingConsumer();
DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
cr = new ClientRunner(clientConn);
// start relay r1
r1.start();
// start chained relay
r2.start();
// start client
cr.start();
// Let the relay run for 10s
Thread.sleep(10 * 1000);
r1.pause();
// wait until client got all events or for maxTimeout;
long maxTimeOutMs = 5 * 1000;
long startTime = System.currentTimeMillis();
DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
DbusEventsTotalStats statsChained = relay2.getInboundEventStatisticsCollector().getTotalStats();
while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
Thread.sleep(500);
// stats.getSizeDataEvents());
if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
break;
}
}
LOG.info("Client stats=" + countingConsumer);
LOG.info("Chained stats=" + statsChained.getNumSysEvents());
LOG.info("Event windows generated=" + stats.getNumSysEvents());
LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
Assert.assertTrue(stats.getNumSysEvents() == statsChained.getNumSysEvents());
Assert.assertTrue(statsChained.getNumDataEvents() == countingConsumer.getNumDataEvents());
Assert.assertTrue(stats.getNumDataEvents() == 2 * countingConsumer.getNumDataEvents());
Assert.assertTrue(countingConsumer.getNumSources() == 1);
Assert.assertTrue(stats.getNumSysEvents() == countingConsumer.getNumWindows());
} catch (Exception e) {
LOG.error("Exception: " + e);
Assert.assertTrue(false);
} finally {
cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
}
}
use of com.linkedin.databus2.relay.config.PhysicalSourceConfig in project databus by linkedin.
the class PhysicalSourceConfigBuilder method build.
@Override
public PhysicalSourceStaticConfig[] build() throws InvalidConfigException {
ObjectMapper mapper = new ObjectMapper();
PhysicalSourceStaticConfig[] list = new PhysicalSourceStaticConfig[null == _fileNames ? 0 : _fileNames.length];
if (_fileNames == null)
return list;
for (int i = 0; i < _fileNames.length; ++i) {
File sourceJson = _fileNames[i];
PhysicalSourceConfig pConfig = null;
Exception e = null;
try {
pConfig = mapper.readValue(sourceJson, PhysicalSourceConfig.class);
} catch (JsonParseException jpe) {
e = jpe;
} catch (JsonMappingException jme) {
e = jme;
} catch (IOException ioe) {
e = ioe;
}
if (e != null || pConfig == null) {
throw new InvalidConfigException(e);
}
pConfig.checkForNulls();
LOG.info("Generated Physical source config: name= " + pConfig.getId());
list[i] = pConfig.build();
}
/*
for(PhysicalSourceStaticConfig pCfg : pConfigs) {
for(LogicalSourceStaticConfig lSC : pCfg.getSources()) {
config.setSourceName("" + lSC.getId(), lSC.getName());
}
}
*/
return list;
}
Aggregations