Examples with TransactionInfo com.linkedin.databus.monitoring.mbean.GGParserStatistics.TransactionInfo used on opensource projects

Example 1 with TransactionInfo

use of com.linkedin.databus.monitoring.mbean.GGParserStatistics.TransactionInfo in project databus by linkedin.

the class TestGoldenGateEventProducer method testAddEventToBuffer.

@Test
public void testAddEventToBuffer() throws InvalidConfigException, UnsupportedKeyException, DatabusException {
    // No rate control
    long rate = 0;
    PhysicalSourceStaticConfig pssc = buildPssc(rate, 0L);
    long scn = 10;
    DbusEventBuffer mb = (DbusEventBuffer) createBufMult(pssc);
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(10);
    int sourceId = 505;
    HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();
    Object key = new String("name");
    Schema.Type keyType = Schema.Type.RECORD;
    ColumnsState.KeyPair kp = new ColumnsState.KeyPair(key, keyType);
    ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(1);
    keyPairs.add(kp);
    Schema s = Schema.parse(avroSchema);
    GenericRecord gr = new GenericData.Record(s);
    gr.put("name", "phani");
    DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    db.add(dbi);
    TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(sourceId, db);
    dbUpdates.add(dbUpdate);
    long timestamp = System.nanoTime();
    gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
    Assert.assertEquals(gg.getRateControl().getNumSleeps(), 0);
    DbusEventIterator iter = mb.acquireIterator("test");
    int count = 0;
    long eventTs = 0;
    while (iter.hasNext()) {
        DbusEvent e = iter.next();
        if (count == 1) {
            // first event prev control event
            eventTs = e.timestampInNanos();
        }
        count++;
    }
    Assert.assertEquals("Event timestamp in Ns", timestamp, eventTs);
    Assert.assertEquals("Got events ", 3, count);
    return;
}

Example 2 with TransactionInfo

use of com.linkedin.databus.monitoring.mbean.GGParserStatistics.TransactionInfo in project databus by linkedin.

the class TestGoldenGateEventProducer method testTransactionsWithDuplicateSCN.

/**
 * Test to cover merging of transactions with same SCNs
 */
@Test
public void testTransactionsWithDuplicateSCN() throws Exception {
    short[] sourceIds = new short[] { 505, 506 };
    String[] sourceNames = new String[] { "source1", "source2" };
    PhysicalSourceStaticConfig pssc = buildSimplePssc(sourceIds, sourceNames, "gg:///tmp:xxx");
    DbusEventBufferAppendable mb = createBufMult(pssc);
    // start producer
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    Object handleXmlCallbackObject = getHandleXmlCallbackInnerInstance(gg);
    Method method = getOnTransactionEndMethod();
    // generates the updates
    List<String> keys = new ArrayList<String>();
    keys.add("key1");
    // SCN = 10 - Case where both transactions having same SCNs : same set of sources and
    // same keys
    long scn = 10;
    long timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC;
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates1 = generateUpdates(sourceIds, keys, scn);
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
    timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates2 = // Same SCN as before
    generateUpdates(sourceIds, keys, scn);
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp + 1, scn) });
    // Expect no events as events are not yet appended to buffer.
    testStats(gg, new EventStatsValues(sourceIds[0], 0, 0, 0, 0, 0), new EventStatsValues(sourceIds[1], 0, 0, 0, 0, 0), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 0, 0, 0, 0, 0));
    // SCN = 11 - Case where both transactions having same SCNs : same set of sources but
    // different keys
    {
        scn = 11;
        timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;
        dbUpdates1 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;
        keys.clear();
        keys.add("key2");
        // Same SCN as before
        dbUpdates2 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        // Testing for SCN = 10 case
        testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 1, 0, 10), new EventStatsValues(sourceIds[1], 5, 0, 1, 0, 10), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 2, 0, 10));
    }
    // SCN = 12 - Case where both transactions having same SCNs but different set of
    // sources
    {
        scn = 12;
        keys.clear();
        keys.add("key2");
        timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 2;
        dbUpdates1 = generateUpdates(new short[] { sourceIds[1] }, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        // Same SCN
        dbUpdates2 = generateUpdates(new short[] { sourceIds[0] }, keys, scn);
        // as before
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        // Testing for SCN = 11 case
        testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 3, 0, 11), new EventStatsValues(sourceIds[1], 5, 0, 3, 0, 11), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 6, 0, 11));
    }
    // SCN = 13 - Case where more than 2 transactions having same SCNs and keys. The keys
    // will be merged.
    {
        scn = 13;
        timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 3;
        dbUpdates1 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        // Same SCN as before
        dbUpdates2 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        dbUpdates1 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        // Same SCN as before
        dbUpdates2 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        // Testing for SCN = 12 case
        testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 4, 0, 12), new EventStatsValues(sourceIds[1], 5, 0, 4, 0, 12), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 8, 0, 12));
    }
    // SCN = 14 - Case where more than 2 transactions having same SCNs but different keys.
    {
        scn = 14;
        timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 3;
        dbUpdates1 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        // Same SCN as before
        dbUpdates2 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        keys.clear();
        keys.add("key5");
        dbUpdates1 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        // Same SCN as before
        dbUpdates2 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        // Testing for SCN = 13 case
        testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 5, 0, 13), new EventStatsValues(sourceIds[1], 5, 0, 5, 0, 13), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 10, 0, 13));
    }
    // THis is an extra-call but the corresponding events will not be added to EVB.
    // This is needed to flush the events in the above call to EVB
    scn = 15;
    timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 4;
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
    // Testing for SCN = 12 case
    testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 7, 0, 14), new EventStatsValues(sourceIds[1], 5, 0, 7, 0, 14), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 14, 0, 14));
}

Example 3 with TransactionInfo

use of com.linkedin.databus.monitoring.mbean.GGParserStatistics.TransactionInfo in project databus by linkedin.

the class TestGoldenGateEventProducer method testAddEventToBufferRateControl.

private void testAddEventToBufferRateControl(long throttleDurationInSecs) throws InvalidConfigException, UnsupportedKeyException, DatabusException, NoSuchFieldException, IllegalAccessException {
    // 1 event per second required. Send 5 events. Must have 4 sleeps.
    long rate = 1;
    int numEvents = 5;
    PhysicalSourceStaticConfig pssc = buildPssc(rate, throttleDurationInSecs);
    long scn = 10;
    DbusEventBufferAppendable mb = createBufMult(pssc);
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    // enable if want to run with mocked timer
    // run_with_mock_timer(gg);
    int sourceId = 505;
    HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();
    // name1 is the only key
    ColumnsState.KeyPair kp1 = new ColumnsState.KeyPair(new String("name1"), Schema.Type.RECORD);
    ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(numEvents);
    keyPairs.add(kp1);
    Schema s = Schema.parse(avroSchema2);
    GenericRecord gr1 = new GenericData.Record(s);
    gr1.put("name1", "phani1");
    gr1.put("name2", "boris1");
    GenericRecord gr2 = new GenericData.Record(s);
    gr2.put("name1", "phani2");
    gr2.put("name2", "boris2");
    GenericRecord gr3 = new GenericData.Record(s);
    gr3.put("name1", "phani3");
    gr3.put("name2", "boris3");
    GenericRecord gr4 = new GenericData.Record(s);
    gr4.put("name1", "phani4");
    gr4.put("name2", "boris4");
    GenericRecord gr5 = new GenericData.Record(s);
    gr5.put("name1", "phani5");
    gr5.put("name2", "boris5");
    DBUpdateImage dbi1 = new DBUpdateImage(keyPairs, scn, gr1, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    DBUpdateImage dbi2 = new DBUpdateImage(keyPairs, scn, gr2, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    DBUpdateImage dbi3 = new DBUpdateImage(keyPairs, scn, gr3, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    DBUpdateImage dbi4 = new DBUpdateImage(keyPairs, scn, gr4, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    DBUpdateImage dbi5 = new DBUpdateImage(keyPairs, scn, gr5, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    db.add(dbi1);
    db.add(dbi2);
    db.add(dbi3);
    db.add(dbi4);
    db.add(dbi5);
    // For a given transaction, and logical source : only 1 update ( the last one succeeds )
    Assert.assertEquals(1, db.size());
    // Generate 5 transactions with the same update
    for (int i = 0; i < numEvents; i++) {
        List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(10);
        TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(sourceId, db);
        dbUpdates.add(dbUpdate);
        long timestamp = 60;
        gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
        scn++;
    }
    // It may not sleep the very first time as 1 second may have elapsed from when the rate control got started to when event in
    // getting inserted. Subsequently, expect rate control to kick in
    long numSleeps = Math.min(numEvents, throttleDurationInSecs);
    Assert.assertEquals(gg.getRateControl().getNumSleeps(), numSleeps);
    gg.getRateControl().resetNumSleeps();
    return;
}

Example 4 with TransactionInfo

use of com.linkedin.databus.monitoring.mbean.GGParserStatistics.TransactionInfo in project databus by linkedin.

the class TestGoldenGateEventProducer method testSCNRegressionStats.

@Test
public void testSCNRegressionStats() throws Exception {
    short[] sourceIds = new short[] { 505, 506 };
    String[] sourceNames = new String[] { "source1", "source2" };
    PhysicalSourceStaticConfig pssc = buildSimplePssc(sourceIds, sourceNames, "gg:///tmp:xxx");
    DbusEventBufferAppendable mb = createBufMult(pssc);
    // start producer
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    Object handleXmlCallbackObject = getHandleXmlCallbackInnerInstance(gg);
    Method method = getOnTransactionEndMethod();
    // generates the updates
    List<String> keys = new ArrayList<String>();
    keys.add("key1");
    GGParserStatistics ggParserStats = gg.getParserStats();
    // SCN = 10
    long timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC;
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates1 = generateUpdates(sourceIds, keys, 10);
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, 10) });
    timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;
    Assert.assertEquals("NumSCNRegressions", 0, ggParserStats.getNumSCNRegressions());
    Assert.assertEquals("NumSCNRegressions", -1, ggParserStats.getLastRegressedScn());
    Assert.assertEquals("MaxScn", 10, ggParserStats.getMaxScn());
    // SCN = 5 Regression
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates2 = // SCN Regression here
    generateUpdates(sourceIds, keys, 5);
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp + 1, 5) });
    Assert.assertEquals("NumSCNRegressions", 1, ggParserStats.getNumSCNRegressions());
    Assert.assertEquals("NumSCNRegressions", 5, ggParserStats.getLastRegressedScn());
    Assert.assertEquals("MaxScn", 10, ggParserStats.getMaxScn());
    // SCN = 6 No Regression here
    dbUpdates2 = generateUpdates(sourceIds, keys, 6);
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp + 1, 6) });
    Assert.assertEquals("NumSCNRegressions", 1, ggParserStats.getNumSCNRegressions());
    Assert.assertEquals("NumSCNRegressions", 5, ggParserStats.getLastRegressedScn());
    Assert.assertEquals("MaxScn", 10, ggParserStats.getMaxScn());
    // SCN = 3 : SCN regression again
    dbUpdates2 = generateUpdates(sourceIds, keys, 3);
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp + 1, 3) });
    Assert.assertEquals("NumSCNRegressions", 2, ggParserStats.getNumSCNRegressions());
    Assert.assertEquals("NumSCNRegressions", 3, ggParserStats.getLastRegressedScn());
    Assert.assertEquals("MaxScn", 10, ggParserStats.getMaxScn());
    // SCN = 11: No regression here
    dbUpdates2 = generateUpdates(sourceIds, keys, 11);
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp + 1, 11) });
    Assert.assertEquals("NumSCNRegressions", 2, ggParserStats.getNumSCNRegressions());
    Assert.assertEquals("NumSCNRegressions", 3, ggParserStats.getLastRegressedScn());
    Assert.assertEquals("MaxScn", 11, ggParserStats.getMaxScn());
}

Example 5 with TransactionInfo

use of com.linkedin.databus.monitoring.mbean.GGParserStatistics.TransactionInfo in project databus by linkedin.

the class TestGoldenGateEventProducer method testGGProducerStats.

@Test
public void testGGProducerStats() throws InvalidConfigException, UnsupportedKeyException, DatabusException, IllegalArgumentException, InstantiationException, IllegalAccessException, InvocationTargetException, SecurityException, NoSuchMethodException {
    short[] sourceIds = new short[] { 505, 506 };
    String[] sourceNames = new String[] { "source1", "source2" };
    PhysicalSourceStaticConfig pssc = buildSimplePssc(sourceIds, sourceNames, "gg:///tmp:xxx");
    long scn = 10;
    DbusEventBufferAppendable mb = createBufMult(pssc);
    // start producer
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    Object handleXmlCallbackObject = getHandleXmlCallbackInnerInstance(gg);
    Method method = getOnTransactionEndMethod();
    // generates the updates
    List<String> keys = new ArrayList<String>();
    keys.add("keys1");
    // SCN = 10
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = generateUpdates(new short[] { sourceIds[0] }, keys, scn);
    // SCN = 11
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates1 = generateUpdates(new short[] { sourceIds[1] }, keys, scn + 1);
    // SCN = 12
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates2 = generateUpdates(new short[] { sourceIds[1] }, keys, scn + 2);
    long timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC;
    // gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
    // gg.addEventToBuffer(dbUpdates1, new TransactionInfo(0, 0, timestamp+1, scn+1));
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates, new TransactionInfo(0, 0, timestamp, scn) });
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp + 1, scn + 1) });
    // THis is an extra-call but the corresponding events will not be added to EVB.
    // This is needed to flush the events in the above call to EVB
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp + 1, scn + 2) });
    for (EventSourceStatisticsIface si : gg.getSources()) {
        EventSourceStatistics ss = si.getStatisticsBean();
        LOG.info(si.getSourceName() + ": scn=" + ss.getMaxScn() + ",averageSize=" + ss.getAvgEventSerializedSize() + ",numErrors=" + ss.getNumErrors() + ",totalEvents=" + ss.getNumTotalEvents() + ",averageFactTime=" + ss.getAvgEventFactoryTimeMillisPerEvent() + ",timeSinceDb=" + ss.getTimeSinceLastDBAccess());
        if (si.getSourceId() == 505) {
            Assert.assertEquals(6, ss.getAvgEventSerializedSize());
            Assert.assertEquals(0, ss.getNumErrors());
            Assert.assertEquals(1, ss.getNumTotalEvents());
            Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent());
            // we are not
            Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent());
            // really
            // reading
            Assert.assertEquals(10, ss.getMaxScn());
        }
        if (si.getSourceId() == 506) {
            Assert.assertEquals(6, ss.getAvgEventSerializedSize());
            Assert.assertEquals(0, ss.getNumErrors());
            Assert.assertEquals(1, ss.getNumTotalEvents());
            Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent());
            // we are not
            Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent());
            // really
            // reading
            Assert.assertEquals(11, ss.getMaxScn());
        }
        if (si.getSourceId() == GoldenGateEventProducer.GLOBAL_SOURCE_ID) {
            Assert.assertEquals(6, ss.getAvgEventSerializedSize());
            Assert.assertEquals(0, ss.getNumErrors());
            Assert.assertEquals(2, ss.getNumTotalEvents());
            Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent());
            // we are not
            Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent());
            // really
            // reading
            Assert.assertEquals(11, ss.getMaxScn());
        }
    }
    long approximateTimeSinceLastTransactionMs = System.currentTimeMillis() - timestamp / DbusConstants.NUM_NSECS_IN_MSEC;
    long diff = gg.getParserStats().getTimeSinceLastTransactionMs() - approximateTimeSinceLastTransactionMs;
    // somewhat
    Assert.assertTrue("time diff is too big:" + diff, diff >= 0 && diff < 30);
    // NS)
    return;
}