use of com.linkedin.databus2.relay.GoldenGateEventProducer in project databus by linkedin.
the class RelayStatsRequestProcessor method processInboundGGStats.
private void processInboundGGStats(DatabusRequest request, String category) throws IOException, RequestProcessingException {
if (!(_relay instanceof DatabusRelayMain)) {
throw new IllegalArgumentException(category + " for relay which is not DatabusRelayMain");
}
String psourceName = null;
if (category.startsWith(INBOUND_GG_PSOURCE_PREFIX)) {
psourceName = category.substring(INBOUND_GG_PSOURCE_PREFIX.length());
if (psourceName == null || psourceName.length() <= 0) {
throw new InvalidRequestParamValueException(request.getName(), INBOUND_GG_PSOURCE_PREFIX, null);
}
LOG.info("get parser stats for source " + psourceName);
}
List<String> phSourceNames = new ArrayList<String>();
EventProducer[] prods = ((DatabusRelayMain) _relay).getProducers();
GGParserStatistics stat = null;
for (EventProducer prod : prods) {
if (prod != null && (prod instanceof GoldenGateEventProducer)) {
GoldenGateEventProducer ggProducer = (GoldenGateEventProducer) prod;
String pSrcName = ggProducer.getParserStats().getPhysicalSourceName();
phSourceNames.add(pSrcName);
if (// remember the stats object
psourceName != null && psourceName.equals(pSrcName))
stat = ggProducer.getParserStats();
}
}
if (psourceName != null) {
if (stat == null)
throw new InvalidRequestParamValueException(request.getName(), INBOUND_GG_PSOURCE_PREFIX, psourceName);
writeJsonObjectToResponse(stat, request);
} else {
writeJsonObjectToResponse(phSourceNames, request);
}
}
use of com.linkedin.databus2.relay.GoldenGateEventProducer in project databus by linkedin.
the class DatabusRelayMain method addOneProducer.
/** overrides HTTP relay method */
@Override
public void addOneProducer(PhysicalSourceStaticConfig pConfig) throws DatabusException, EventCreationException, UnsupportedKeyException, SQLException, InvalidConfigException {
// Register a command to allow start/stop/status of the relay
List<EventProducer> plist = new ArrayList<EventProducer>();
PhysicalPartition pPartition = pConfig.getPhysicalPartition();
MaxSCNReaderWriter maxScnReaderWriters = _maxScnReaderWriters.getOrCreateHandler(pPartition);
LOG.info("Starting server container with maxScnReaderWriter:" + maxScnReaderWriters);
// Get the event buffer
DbusEventBufferAppendable dbusEventBuffer = getEventBuffer().getDbusEventBufferAppendable(pPartition);
// Get the schema registry service
SchemaRegistryService schemaRegistryService = getSchemaRegistryService();
// Get a stats collector per physical source
addPhysicalPartitionCollectors(pPartition);
String statsCollectorName = pPartition.toSimpleString();
/*
* _inBoundStatsCollectors.addStatsCollector(statsCollectorName, new
* DbusEventsStatisticsCollector(getContainerStaticConfig().getId(),
* statsCollectorName+".inbound", true, false, getMbeanServer()));
*
* _outBoundStatsCollectors.addStatsCollector(statsCollectorName, new
* DbusEventsStatisticsCollector(getContainerStaticConfig().getId(),
* statsCollectorName+".outbound", true, false, getMbeanServer()));
*/
// Create the event producer
String uri = pConfig.getUri();
if (uri == null)
throw new DatabusException("Uri is required to start the relay");
uri = uri.trim();
EventProducer producer = null;
if (uri.startsWith("jdbc:")) {
SourceType sourceType = pConfig.getReplBitSetter().getSourceType();
if (SourceType.TOKEN.equals(sourceType))
throw new DatabusException("Token Source-type for Replication bit setter config cannot be set for trigger-based Databus relay !!");
// if a buffer for this partiton exists - we are overwri
producer = new OracleEventProducerFactory().buildEventProducer(pConfig, schemaRegistryService, dbusEventBuffer, getMbeanServer(), _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
} else if (uri.startsWith("mock")) {
// Get all relevant pConfig attributes
//TODO add real instantiation
EventProducerServiceProvider mockProvider = _producersRegistry.getEventProducerServiceProvider("mock");
if (null == mockProvider) {
throw new DatabusRuntimeException("relay event producer not available: " + "mock");
}
producer = mockProvider.createProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
} else if (uri.startsWith("gg:")) {
producer = new GoldenGateEventProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
} else if (uri.startsWith("mysql:")) {
LOG.info("Adding OpenReplicatorEventProducer for uri :" + uri);
final String serviceName = "or";
EventProducerServiceProvider orProvider = _producersRegistry.getEventProducerServiceProvider(serviceName);
if (null == orProvider) {
throw new DatabusRuntimeException("relay event producer not available: " + serviceName);
}
producer = orProvider.createProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
} else {
// Get all relevant pConfig attributes and initialize the nettyThreadPool objects
RelayEventProducer.DatabusClientNettyThreadPools nettyThreadPools = new RelayEventProducer.DatabusClientNettyThreadPools(0, getNetworkTimeoutTimer(), getBossExecutorService(), getIoExecutorService(), getHttpChannelGroup());
producer = new RelayEventProducer(pConfig, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters, nettyThreadPools);
}
// if a buffer for this partiton exists - we are overwriting it.
_producers.put(pPartition, producer);
plist.add(producer);
// append 'monitoring event producer'
if (producer instanceof OracleEventProducer) {
MonitoringEventProducer monitoringProducer = new MonitoringEventProducer("dbMonitor." + pPartition.toSimpleString(), pConfig.getName(), pConfig.getUri(), ((OracleEventProducer) producer).getMonitoredSourceInfos(), getMbeanServer());
_monitoringProducers.put(pPartition, monitoringProducer);
plist.add(monitoringProducer);
}
if (_csEventRequestProcessor == null)
_csEventRequestProcessor = new ControlSourceEventsRequestProcessor(null, this, plist);
else
_csEventRequestProcessor.addEventProducers(plist);
RequestProcessorRegistry processorRegistry = getProcessorRegistry();
processorRegistry.reregister(ControlSourceEventsRequestProcessor.COMMAND_NAME, _csEventRequestProcessor);
}
use of com.linkedin.databus2.relay.GoldenGateEventProducer in project databus by linkedin.
the class TestGoldenGateEventProducer method testAddEventToBuffer.
@Test
public void testAddEventToBuffer() throws InvalidConfigException, UnsupportedKeyException, DatabusException {
// No rate control
long rate = 0;
PhysicalSourceStaticConfig pssc = buildPssc(rate, 0L);
long scn = 10;
DbusEventBuffer mb = (DbusEventBuffer) createBufMult(pssc);
GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(10);
int sourceId = 505;
HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();
Object key = new String("name");
Schema.Type keyType = Schema.Type.RECORD;
ColumnsState.KeyPair kp = new ColumnsState.KeyPair(key, keyType);
ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(1);
keyPairs.add(kp);
Schema s = Schema.parse(avroSchema);
GenericRecord gr = new GenericData.Record(s);
gr.put("name", "phani");
DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
db.add(dbi);
TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(sourceId, db);
dbUpdates.add(dbUpdate);
long timestamp = System.nanoTime();
gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
Assert.assertEquals(gg.getRateControl().getNumSleeps(), 0);
DbusEventIterator iter = mb.acquireIterator("test");
int count = 0;
long eventTs = 0;
while (iter.hasNext()) {
DbusEvent e = iter.next();
if (count == 1) {
// first event prev control event
eventTs = e.timestampInNanos();
}
count++;
}
Assert.assertEquals("Event timestamp in Ns", timestamp, eventTs);
Assert.assertEquals("Got events ", 3, count);
return;
}
use of com.linkedin.databus2.relay.GoldenGateEventProducer in project databus by linkedin.
the class TestGoldenGateEventProducer method testTransactionsWithDuplicateSCN.
/**
*
* Test to cover merging of transactions with same SCNs
*/
@Test
public void testTransactionsWithDuplicateSCN() throws Exception {
short[] sourceIds = new short[] { 505, 506 };
String[] sourceNames = new String[] { "source1", "source2" };
PhysicalSourceStaticConfig pssc = buildSimplePssc(sourceIds, sourceNames, "gg:///tmp:xxx");
DbusEventBufferAppendable mb = createBufMult(pssc);
// start producer
GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
Object handleXmlCallbackObject = getHandleXmlCallbackInnerInstance(gg);
Method method = getOnTransactionEndMethod();
// generates the updates
List<String> keys = new ArrayList<String>();
keys.add("key1");
// SCN = 10 - Case where both transactions having same SCNs : same set of sources and
// same keys
long scn = 10;
long timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC;
List<TransactionState.PerSourceTransactionalUpdate> dbUpdates1 = generateUpdates(sourceIds, keys, scn);
method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;
List<TransactionState.PerSourceTransactionalUpdate> dbUpdates2 = // Same SCN as before
generateUpdates(sourceIds, keys, scn);
method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp + 1, scn) });
// Expect no events as events are not yet appended to buffer.
testStats(gg, new EventStatsValues(sourceIds[0], 0, 0, 0, 0, 0), new EventStatsValues(sourceIds[1], 0, 0, 0, 0, 0), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 0, 0, 0, 0, 0));
// SCN = 11 - Case where both transactions having same SCNs : same set of sources but
// different keys
{
scn = 11;
timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;
dbUpdates1 = generateUpdates(sourceIds, keys, scn);
method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;
keys.clear();
keys.add("key2");
// Same SCN as before
dbUpdates2 = generateUpdates(sourceIds, keys, scn);
method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
// Testing for SCN = 10 case
testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 1, 0, 10), new EventStatsValues(sourceIds[1], 5, 0, 1, 0, 10), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 2, 0, 10));
}
// SCN = 12 - Case where both transactions having same SCNs but different set of
// sources
{
scn = 12;
keys.clear();
keys.add("key2");
timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 2;
dbUpdates1 = generateUpdates(new short[] { sourceIds[1] }, keys, scn);
method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
// Same SCN
dbUpdates2 = generateUpdates(new short[] { sourceIds[0] }, keys, scn);
// as before
method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
// Testing for SCN = 11 case
testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 3, 0, 11), new EventStatsValues(sourceIds[1], 5, 0, 3, 0, 11), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 6, 0, 11));
}
// SCN = 13 - Case where more than 2 transactions having same SCNs and keys. The keys
// will be merged.
{
scn = 13;
timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 3;
dbUpdates1 = generateUpdates(sourceIds, keys, scn);
method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
// Same SCN as before
dbUpdates2 = generateUpdates(sourceIds, keys, scn);
method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
dbUpdates1 = generateUpdates(sourceIds, keys, scn);
method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
// Same SCN as before
dbUpdates2 = generateUpdates(sourceIds, keys, scn);
method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
// Testing for SCN = 12 case
testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 4, 0, 12), new EventStatsValues(sourceIds[1], 5, 0, 4, 0, 12), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 8, 0, 12));
}
// SCN = 14 - Case where more than 2 transactions having same SCNs but different keys.
{
scn = 14;
timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 3;
dbUpdates1 = generateUpdates(sourceIds, keys, scn);
method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
// Same SCN as before
dbUpdates2 = generateUpdates(sourceIds, keys, scn);
method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
keys.clear();
keys.add("key5");
dbUpdates1 = generateUpdates(sourceIds, keys, scn);
method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
// Same SCN as before
dbUpdates2 = generateUpdates(sourceIds, keys, scn);
method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
// Testing for SCN = 13 case
testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 5, 0, 13), new EventStatsValues(sourceIds[1], 5, 0, 5, 0, 13), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 10, 0, 13));
}
// THis is an extra-call but the corresponding events will not be added to EVB.
// This is needed to flush the events in the above call to EVB
scn = 15;
timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 4;
method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
// Testing for SCN = 12 case
testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 7, 0, 14), new EventStatsValues(sourceIds[1], 5, 0, 7, 0, 14), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 14, 0, 14));
}
use of com.linkedin.databus2.relay.GoldenGateEventProducer in project databus by linkedin.
the class TestGoldenGateEventProducer method testGGParserStats.
/**
* test collection of parser stats, especially lag between parsed and added files
* @throws Exception
*/
@Test
public void testGGParserStats() throws Exception {
short[] sourceIds = new short[] { 505, 506 };
String[] sourceNames = new String[] { "source1", "source2" };
// setup trail Files directory
File ggTrailDir = FileUtils.createTempDir("testGGParserStats");
// configure physical source
String uri = "gg://" + ggTrailDir.getAbsolutePath() + ":x3";
PhysicalSourceStaticConfig pssc = buildSimplePssc(sourceIds, sourceNames, uri);
LOG.info("Uri=" + uri);
// create schema
Schema s = Schema.parse(sourceAvroSchema);
VersionedSchema vs = new VersionedSchema(new VersionedSchemaId("source1", (short) 3), s, null);
// mock for schema registry
SchemaRegistryService srs = EasyMock.createMock(SchemaRegistryService.class);
EasyMock.expect(srs.fetchLatestVersionedSchemaBySourceName("source1")).andReturn(vs).anyTimes();
EasyMock.expect(srs.fetchLatestVersionedSchemaBySourceName("source2")).andReturn(vs).anyTimes();
EasyMock.expect(srs.fetchLatestVersionedSchemaBySourceName(null)).andReturn(vs);
// mock for MaxSCNReadWriter
MaxSCNReaderWriter mscn = EasyMock.createMock(MaxSCNReaderWriter.class);
EasyMock.expect(mscn.getMaxScn()).andReturn((long) -2).atLeastOnce();
mscn.saveMaxScn(EasyMock.anyLong());
EasyMock.expectLastCall().anyTimes();
EasyMock.replay(mscn);
EasyMock.replay(srs);
int totalTransWritten = 0;
int totalFilesWritten = 0;
// buffer
DbusEventBufferAppendable mb = createBufMult(pssc);
// start GG producer
GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, srs, mb, null, mscn);
// create first 2 files
addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x301"), 100, 4);
addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x302"), 200, 4);
totalTransWritten = 8;
totalFilesWritten = 2;
// get hold of parser stats object
final GGParserStatistics ggParserStats = gg.getParserStats();
// all should be 0
Assert.assertEquals(0, ggParserStats.getNumFilesParsed());
Assert.assertEquals(0, ggParserStats.getNumFilesAdded());
Assert.assertEquals(0, ggParserStats.getFilesLag());
Assert.assertEquals(0, ggParserStats.getTimeLag());
Assert.assertEquals(0, ggParserStats.getBytesLag());
try {
LOG.info("starting event producer");
// -2 here does nothing. actual setting happens thru the mock of
gg.start(-2);
// MaxSCNReadWriter
// let it parse first files
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return ggParserStats.getNumFilesParsed() == 2 && (8 * _transactionPatternSize == ggParserStats.getNumBytesTotalParsed());
}
}, "First two files parsed", 2000, LOG);
// stats in the interim
Assert.assertEquals(2, ggParserStats.getNumFilesParsed());
Assert.assertEquals(2, ggParserStats.getNumFilesAdded());
Assert.assertEquals(0, ggParserStats.getFilesLag());
Assert.assertEquals(0, ggParserStats.getTimeLag());
Assert.assertEquals(0, ggParserStats.getBytesLag());
Assert.assertEquals(totalTransWritten * _transactionPatternSize, ggParserStats.getNumBytesTotalParsed());
gg.pause();
// the file will get parsed but not processed
addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x303"), 300, 4);
totalTransWritten += 4;
totalFilesWritten++;
// to get more then a ms lag time
TestUtil.sleep(2000);
addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x304"), 400, 4);
totalTransWritten += 4;
totalFilesWritten++;
// to guarantee we picked up stats update (stats are updated
TestUtil.sleep(6000);
// every 5 seconds)
// now we should be 2 files behind. parser thread gets paused AFTER it start
// processing the file
// so the actuall value will be 1 file behind
// 303(already started being parsed), only 304 is behind
int lagFiles = 1;
// 1 file, 4 transactions each
long lagBytes = 1 * 4 * _transactionPatternSize;
/*
* Assert.assertEquals(totalFilesWritten-1, ggParserStats.getNumFilesParsed());
* Assert.assertEquals(totalFilesWritten, ggParserStats.getNumFilesAdded());
* Assert.assertEquals(lagFiles, ggParserStats.getFilesLag()); // because 303 got
* parsed
*
* // we added 4 files and parsed 3 , so the diff should be 1 file size (4
* trasactions in 1 file) Assert.assertEquals(lagBytes,
* ggParserStats.getBytesLag()); Assert.assertTrue(ggParserStats.getTimeLag()>0);
*/
gg.unpause();
TestUtil.sleep(5000);
// now we should catchup
Assert.assertEquals(4, ggParserStats.getNumFilesParsed());
Assert.assertEquals(4, ggParserStats.getNumFilesAdded());
Assert.assertEquals(0, ggParserStats.getFilesLag());
Assert.assertEquals(0, ggParserStats.getTimeLag());
Assert.assertEquals(0, ggParserStats.getBytesLag());
// append to a file
LOG.info("pausing again");
gg.pause();
addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x304"), 410, 4);
totalTransWritten += 4;
TestUtil.sleep(1000);
addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x304"), 420, 4);
totalTransWritten += 4;
TestUtil.sleep(2000);
gg.unpause();
TestUtil.sleep(5500);
// should be still up
Assert.assertEquals(4, ggParserStats.getNumFilesParsed());
Assert.assertEquals(4, ggParserStats.getNumFilesAdded());
Assert.assertEquals(0, ggParserStats.getFilesLag());
Assert.assertEquals(0, ggParserStats.getTimeLag());
Assert.assertEquals(0, ggParserStats.getBytesLag());
// assert the stats
int totalFilesSize = totalTransWritten * _transactionPatternSize;
Assert.assertEquals((totalFilesSize / totalFilesWritten), ggParserStats.getAvgFileSize());
Assert.assertEquals(true, ggParserStats.getAvgParseTransactionTimeNs() > 0);
Assert.assertEquals("part1", ggParserStats.getPhysicalSourceName());
Assert.assertEquals(totalFilesSize / totalTransWritten, ggParserStats.getAvgTransactionSize());
Assert.assertEquals(423, ggParserStats.getMaxScn());
// 2
Assert.assertEquals(totalTransWritten * 2, ggParserStats.getNumTotalEvents());
// events
// per
// transaction
Assert.assertEquals(totalTransWritten, ggParserStats.getNumTransactionsTotal());
Assert.assertEquals(totalTransWritten, ggParserStats.getNumTransactionsWithEvents());
Assert.assertEquals(0, ggParserStats.getNumTransactionsWithoutEvents());
Assert.assertEquals(true, ggParserStats.getTimeSinceLastAccessMs() > 0);
Assert.assertEquals(totalTransWritten * _transactionPatternSize, ggParserStats.getNumBytesTotalParsed());
Assert.assertEquals("NumSCNRegressions", 0, ggParserStats.getNumSCNRegressions());
Assert.assertEquals("LastSCNRegressed", -1, ggParserStats.getLastRegressedScn());
} finally {
gg.shutdown();
}
return;
}
Aggregations