Search in sources :

Example 1 with GoldenGateEventProducer

use of com.linkedin.databus2.relay.GoldenGateEventProducer in project databus by linkedin.

the class RelayStatsRequestProcessor method processInboundGGStats.

private void processInboundGGStats(DatabusRequest request, String category) throws IOException, RequestProcessingException {
    if (!(_relay instanceof DatabusRelayMain)) {
        throw new IllegalArgumentException(category + " for relay which is not DatabusRelayMain");
    }
    String psourceName = null;
    if (category.startsWith(INBOUND_GG_PSOURCE_PREFIX)) {
        psourceName = category.substring(INBOUND_GG_PSOURCE_PREFIX.length());
        if (psourceName == null || psourceName.length() <= 0) {
            throw new InvalidRequestParamValueException(request.getName(), INBOUND_GG_PSOURCE_PREFIX, null);
        }
        LOG.info("get parser stats for source " + psourceName);
    }
    List<String> phSourceNames = new ArrayList<String>();
    EventProducer[] prods = ((DatabusRelayMain) _relay).getProducers();
    GGParserStatistics stat = null;
    for (EventProducer prod : prods) {
        if (prod != null && (prod instanceof GoldenGateEventProducer)) {
            GoldenGateEventProducer ggProducer = (GoldenGateEventProducer) prod;
            String pSrcName = ggProducer.getParserStats().getPhysicalSourceName();
            phSourceNames.add(pSrcName);
            if (// remember the stats object
            psourceName != null && psourceName.equals(pSrcName))
                stat = ggProducer.getParserStats();
        }
    }
    if (psourceName != null) {
        if (stat == null)
            throw new InvalidRequestParamValueException(request.getName(), INBOUND_GG_PSOURCE_PREFIX, psourceName);
        writeJsonObjectToResponse(stat, request);
    } else {
        writeJsonObjectToResponse(phSourceNames, request);
    }
}
Also used : GGParserStatistics(com.linkedin.databus.monitoring.mbean.GGParserStatistics) DatabusRelayMain(com.linkedin.databus2.relay.DatabusRelayMain) ArrayList(java.util.ArrayList) EventProducer(com.linkedin.databus2.producers.EventProducer) GoldenGateEventProducer(com.linkedin.databus2.relay.GoldenGateEventProducer) InvalidRequestParamValueException(com.linkedin.databus2.core.container.request.InvalidRequestParamValueException) GoldenGateEventProducer(com.linkedin.databus2.relay.GoldenGateEventProducer)

Example 2 with GoldenGateEventProducer

use of com.linkedin.databus2.relay.GoldenGateEventProducer in project databus by linkedin.

the class DatabusRelayMain method addOneProducer.

/** overrides HTTP relay method */
@Override
public void addOneProducer(PhysicalSourceStaticConfig pConfig) throws DatabusException, EventCreationException, UnsupportedKeyException, SQLException, InvalidConfigException {
    // Register a command to allow start/stop/status of the relay
    List<EventProducer> plist = new ArrayList<EventProducer>();
    PhysicalPartition pPartition = pConfig.getPhysicalPartition();
    MaxSCNReaderWriter maxScnReaderWriters = _maxScnReaderWriters.getOrCreateHandler(pPartition);
    LOG.info("Starting server container with maxScnReaderWriter:" + maxScnReaderWriters);
    // Get the event buffer
    DbusEventBufferAppendable dbusEventBuffer = getEventBuffer().getDbusEventBufferAppendable(pPartition);
    // Get the schema registry service
    SchemaRegistryService schemaRegistryService = getSchemaRegistryService();
    // Get a stats collector per physical source
    addPhysicalPartitionCollectors(pPartition);
    String statsCollectorName = pPartition.toSimpleString();
    /*
		 * _inBoundStatsCollectors.addStatsCollector(statsCollectorName, new
		 * DbusEventsStatisticsCollector(getContainerStaticConfig().getId(),
		 * statsCollectorName+".inbound", true, false, getMbeanServer()));
		 *
		 * _outBoundStatsCollectors.addStatsCollector(statsCollectorName, new
		 * DbusEventsStatisticsCollector(getContainerStaticConfig().getId(),
		 * statsCollectorName+".outbound", true, false, getMbeanServer()));
		 */
    // Create the event producer
    String uri = pConfig.getUri();
    if (uri == null)
        throw new DatabusException("Uri is required to start the relay");
    uri = uri.trim();
    EventProducer producer = null;
    if (uri.startsWith("jdbc:")) {
        SourceType sourceType = pConfig.getReplBitSetter().getSourceType();
        if (SourceType.TOKEN.equals(sourceType))
            throw new DatabusException("Token Source-type for Replication bit setter config cannot be set for trigger-based Databus relay !!");
        // if a buffer for this partiton exists - we are overwri
        producer = new OracleEventProducerFactory().buildEventProducer(pConfig, schemaRegistryService, dbusEventBuffer, getMbeanServer(), _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
    } else if (uri.startsWith("mock")) {
        // Get all relevant pConfig attributes
        //TODO add real instantiation
        EventProducerServiceProvider mockProvider = _producersRegistry.getEventProducerServiceProvider("mock");
        if (null == mockProvider) {
            throw new DatabusRuntimeException("relay event producer not available: " + "mock");
        }
        producer = mockProvider.createProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
    } else if (uri.startsWith("gg:")) {
        producer = new GoldenGateEventProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
    } else if (uri.startsWith("mysql:")) {
        LOG.info("Adding OpenReplicatorEventProducer for uri :" + uri);
        final String serviceName = "or";
        EventProducerServiceProvider orProvider = _producersRegistry.getEventProducerServiceProvider(serviceName);
        if (null == orProvider) {
            throw new DatabusRuntimeException("relay event producer not available: " + serviceName);
        }
        producer = orProvider.createProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
    } else {
        // Get all relevant pConfig attributes and initialize the nettyThreadPool objects
        RelayEventProducer.DatabusClientNettyThreadPools nettyThreadPools = new RelayEventProducer.DatabusClientNettyThreadPools(0, getNetworkTimeoutTimer(), getBossExecutorService(), getIoExecutorService(), getHttpChannelGroup());
        producer = new RelayEventProducer(pConfig, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters, nettyThreadPools);
    }
    // if a buffer for this partiton exists - we are overwriting it.
    _producers.put(pPartition, producer);
    plist.add(producer);
    // append 'monitoring event producer'
    if (producer instanceof OracleEventProducer) {
        MonitoringEventProducer monitoringProducer = new MonitoringEventProducer("dbMonitor." + pPartition.toSimpleString(), pConfig.getName(), pConfig.getUri(), ((OracleEventProducer) producer).getMonitoredSourceInfos(), getMbeanServer());
        _monitoringProducers.put(pPartition, monitoringProducer);
        plist.add(monitoringProducer);
    }
    if (_csEventRequestProcessor == null)
        _csEventRequestProcessor = new ControlSourceEventsRequestProcessor(null, this, plist);
    else
        _csEventRequestProcessor.addEventProducers(plist);
    RequestProcessorRegistry processorRegistry = getProcessorRegistry();
    processorRegistry.reregister(ControlSourceEventsRequestProcessor.COMMAND_NAME, _csEventRequestProcessor);
}
Also used : MaxSCNReaderWriter(com.linkedin.databus2.core.seq.MaxSCNReaderWriter) DbusEventBufferAppendable(com.linkedin.databus.core.DbusEventBufferAppendable) SchemaRegistryService(com.linkedin.databus2.schemas.SchemaRegistryService) SourceType(com.linkedin.databus2.relay.config.ReplicationBitSetterStaticConfig.SourceType) ArrayList(java.util.ArrayList) OracleEventProducer(com.linkedin.databus2.producers.db.OracleEventProducer) RelayEventProducer(com.linkedin.databus2.producers.RelayEventProducer) EventProducer(com.linkedin.databus2.producers.EventProducer) OracleEventProducer(com.linkedin.databus2.producers.db.OracleEventProducer) EventProducerServiceProvider(com.linkedin.databus2.producers.EventProducerServiceProvider) ControlSourceEventsRequestProcessor(com.linkedin.databus.container.request.ControlSourceEventsRequestProcessor) DatabusException(com.linkedin.databus2.core.DatabusException) RequestProcessorRegistry(com.linkedin.databus2.core.container.request.RequestProcessorRegistry) PhysicalPartition(com.linkedin.databus.core.data_model.PhysicalPartition) DatabusRuntimeException(com.linkedin.databus.core.DatabusRuntimeException) RelayEventProducer(com.linkedin.databus2.producers.RelayEventProducer)

Example 3 with GoldenGateEventProducer

use of com.linkedin.databus2.relay.GoldenGateEventProducer in project databus by linkedin.

the class TestGoldenGateEventProducer method testAddEventToBuffer.

@Test
public void testAddEventToBuffer() throws InvalidConfigException, UnsupportedKeyException, DatabusException {
    // No rate control
    long rate = 0;
    PhysicalSourceStaticConfig pssc = buildPssc(rate, 0L);
    long scn = 10;
    DbusEventBuffer mb = (DbusEventBuffer) createBufMult(pssc);
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(10);
    int sourceId = 505;
    HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();
    Object key = new String("name");
    Schema.Type keyType = Schema.Type.RECORD;
    ColumnsState.KeyPair kp = new ColumnsState.KeyPair(key, keyType);
    ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(1);
    keyPairs.add(kp);
    Schema s = Schema.parse(avroSchema);
    GenericRecord gr = new GenericData.Record(s);
    gr.put("name", "phani");
    DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    db.add(dbi);
    TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(sourceId, db);
    dbUpdates.add(dbUpdate);
    long timestamp = System.nanoTime();
    gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
    Assert.assertEquals(gg.getRateControl().getNumSleeps(), 0);
    DbusEventIterator iter = mb.acquireIterator("test");
    int count = 0;
    long eventTs = 0;
    while (iter.hasNext()) {
        DbusEvent e = iter.next();
        if (count == 1) {
            // first event prev control event
            eventTs = e.timestampInNanos();
        }
        count++;
    }
    Assert.assertEquals("Event timestamp in Ns", timestamp, eventTs);
    Assert.assertEquals("Got events ", 3, count);
    return;
}
Also used : PhysicalSourceStaticConfig(com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig) TransactionState(com.linkedin.databus2.ggParser.XmlStateMachine.TransactionState) Schema(org.apache.avro.Schema) VersionedSchema(com.linkedin.databus2.schemas.VersionedSchema) ArrayList(java.util.ArrayList) TransactionInfo(com.linkedin.databus.monitoring.mbean.GGParserStatistics.TransactionInfo) GenericRecord(org.apache.avro.generic.GenericRecord) GenericRecord(org.apache.avro.generic.GenericRecord) HashSet(java.util.HashSet) DbusEvent(com.linkedin.databus.core.DbusEvent) DBUpdateImage(com.linkedin.databus2.ggParser.XmlStateMachine.DbUpdateState.DBUpdateImage) ColumnsState(com.linkedin.databus2.ggParser.XmlStateMachine.ColumnsState) DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer) DbusEventIterator(com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator) Test(org.testng.annotations.Test)

Example 4 with GoldenGateEventProducer

use of com.linkedin.databus2.relay.GoldenGateEventProducer in project databus by linkedin.

the class TestGoldenGateEventProducer method testTransactionsWithDuplicateSCN.

/**
   *
   * Test to cover merging of transactions with same SCNs
   */
@Test
public void testTransactionsWithDuplicateSCN() throws Exception {
    short[] sourceIds = new short[] { 505, 506 };
    String[] sourceNames = new String[] { "source1", "source2" };
    PhysicalSourceStaticConfig pssc = buildSimplePssc(sourceIds, sourceNames, "gg:///tmp:xxx");
    DbusEventBufferAppendable mb = createBufMult(pssc);
    // start producer
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    Object handleXmlCallbackObject = getHandleXmlCallbackInnerInstance(gg);
    Method method = getOnTransactionEndMethod();
    // generates the updates
    List<String> keys = new ArrayList<String>();
    keys.add("key1");
    // SCN = 10 - Case where both transactions having same SCNs : same set of sources and
    // same keys
    long scn = 10;
    long timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC;
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates1 = generateUpdates(sourceIds, keys, scn);
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
    timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates2 = // Same SCN as before
    generateUpdates(sourceIds, keys, scn);
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp + 1, scn) });
    // Expect no events as events are not yet appended to buffer.
    testStats(gg, new EventStatsValues(sourceIds[0], 0, 0, 0, 0, 0), new EventStatsValues(sourceIds[1], 0, 0, 0, 0, 0), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 0, 0, 0, 0, 0));
    // SCN = 11 - Case where both transactions having same SCNs : same set of sources but
    // different keys
    {
        scn = 11;
        timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;
        dbUpdates1 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;
        keys.clear();
        keys.add("key2");
        // Same SCN as before
        dbUpdates2 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        // Testing for SCN = 10 case
        testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 1, 0, 10), new EventStatsValues(sourceIds[1], 5, 0, 1, 0, 10), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 2, 0, 10));
    }
    // SCN = 12 - Case where both transactions having same SCNs but different set of
    // sources
    {
        scn = 12;
        keys.clear();
        keys.add("key2");
        timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 2;
        dbUpdates1 = generateUpdates(new short[] { sourceIds[1] }, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        // Same SCN
        dbUpdates2 = generateUpdates(new short[] { sourceIds[0] }, keys, scn);
        // as before
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        // Testing for SCN = 11 case
        testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 3, 0, 11), new EventStatsValues(sourceIds[1], 5, 0, 3, 0, 11), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 6, 0, 11));
    }
    // SCN = 13 - Case where more than 2 transactions having same SCNs and keys. The keys
    // will be merged.
    {
        scn = 13;
        timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 3;
        dbUpdates1 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        // Same SCN as before
        dbUpdates2 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        dbUpdates1 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        // Same SCN as before
        dbUpdates2 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        // Testing for SCN = 12 case
        testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 4, 0, 12), new EventStatsValues(sourceIds[1], 5, 0, 4, 0, 12), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 8, 0, 12));
    }
    // SCN = 14 - Case where more than 2 transactions having same SCNs but different keys.
    {
        scn = 14;
        timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 3;
        dbUpdates1 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        // Same SCN as before
        dbUpdates2 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        keys.clear();
        keys.add("key5");
        dbUpdates1 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        // Same SCN as before
        dbUpdates2 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        // Testing for SCN = 13 case
        testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 5, 0, 13), new EventStatsValues(sourceIds[1], 5, 0, 5, 0, 13), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 10, 0, 13));
    }
    // THis is an extra-call but the corresponding events will not be added to EVB.
    // This is needed to flush the events in the above call to EVB
    scn = 15;
    timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 4;
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
    // Testing for SCN = 12 case
    testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 7, 0, 14), new EventStatsValues(sourceIds[1], 5, 0, 7, 0, 14), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 14, 0, 14));
}
Also used : PhysicalSourceStaticConfig(com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig) DbusEventBufferAppendable(com.linkedin.databus.core.DbusEventBufferAppendable) ArrayList(java.util.ArrayList) Method(java.lang.reflect.Method) TransactionInfo(com.linkedin.databus.monitoring.mbean.GGParserStatistics.TransactionInfo) Test(org.testng.annotations.Test)

Example 5 with GoldenGateEventProducer

use of com.linkedin.databus2.relay.GoldenGateEventProducer in project databus by linkedin.

the class TestGoldenGateEventProducer method testGGParserStats.

/**
   * test collection of parser stats, especially lag between parsed and added files
   * @throws Exception
   */
@Test
public void testGGParserStats() throws Exception {
    short[] sourceIds = new short[] { 505, 506 };
    String[] sourceNames = new String[] { "source1", "source2" };
    // setup trail Files directory
    File ggTrailDir = FileUtils.createTempDir("testGGParserStats");
    // configure physical source
    String uri = "gg://" + ggTrailDir.getAbsolutePath() + ":x3";
    PhysicalSourceStaticConfig pssc = buildSimplePssc(sourceIds, sourceNames, uri);
    LOG.info("Uri=" + uri);
    // create schema
    Schema s = Schema.parse(sourceAvroSchema);
    VersionedSchema vs = new VersionedSchema(new VersionedSchemaId("source1", (short) 3), s, null);
    // mock for schema registry
    SchemaRegistryService srs = EasyMock.createMock(SchemaRegistryService.class);
    EasyMock.expect(srs.fetchLatestVersionedSchemaBySourceName("source1")).andReturn(vs).anyTimes();
    EasyMock.expect(srs.fetchLatestVersionedSchemaBySourceName("source2")).andReturn(vs).anyTimes();
    EasyMock.expect(srs.fetchLatestVersionedSchemaBySourceName(null)).andReturn(vs);
    // mock for MaxSCNReadWriter
    MaxSCNReaderWriter mscn = EasyMock.createMock(MaxSCNReaderWriter.class);
    EasyMock.expect(mscn.getMaxScn()).andReturn((long) -2).atLeastOnce();
    mscn.saveMaxScn(EasyMock.anyLong());
    EasyMock.expectLastCall().anyTimes();
    EasyMock.replay(mscn);
    EasyMock.replay(srs);
    int totalTransWritten = 0;
    int totalFilesWritten = 0;
    // buffer
    DbusEventBufferAppendable mb = createBufMult(pssc);
    // start GG producer
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, srs, mb, null, mscn);
    // create first 2 files
    addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x301"), 100, 4);
    addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x302"), 200, 4);
    totalTransWritten = 8;
    totalFilesWritten = 2;
    // get hold of parser stats object
    final GGParserStatistics ggParserStats = gg.getParserStats();
    // all should be 0
    Assert.assertEquals(0, ggParserStats.getNumFilesParsed());
    Assert.assertEquals(0, ggParserStats.getNumFilesAdded());
    Assert.assertEquals(0, ggParserStats.getFilesLag());
    Assert.assertEquals(0, ggParserStats.getTimeLag());
    Assert.assertEquals(0, ggParserStats.getBytesLag());
    try {
        LOG.info("starting event producer");
        // -2 here does nothing. actual setting happens thru the mock of
        gg.start(-2);
        // MaxSCNReadWriter
        // let it parse first files
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return ggParserStats.getNumFilesParsed() == 2 && (8 * _transactionPatternSize == ggParserStats.getNumBytesTotalParsed());
            }
        }, "First two files parsed", 2000, LOG);
        // stats in the interim
        Assert.assertEquals(2, ggParserStats.getNumFilesParsed());
        Assert.assertEquals(2, ggParserStats.getNumFilesAdded());
        Assert.assertEquals(0, ggParserStats.getFilesLag());
        Assert.assertEquals(0, ggParserStats.getTimeLag());
        Assert.assertEquals(0, ggParserStats.getBytesLag());
        Assert.assertEquals(totalTransWritten * _transactionPatternSize, ggParserStats.getNumBytesTotalParsed());
        gg.pause();
        // the file will get parsed but not processed
        addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x303"), 300, 4);
        totalTransWritten += 4;
        totalFilesWritten++;
        // to get more then a ms lag time
        TestUtil.sleep(2000);
        addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x304"), 400, 4);
        totalTransWritten += 4;
        totalFilesWritten++;
        // to guarantee we picked up stats update (stats are updated
        TestUtil.sleep(6000);
        // every 5 seconds)
        // now we should be 2 files behind. parser thread gets paused AFTER it start
        // processing the file
        // so the actuall value will be 1 file behind
        // 303(already started being parsed), only 304 is behind
        int lagFiles = 1;
        // 1 file, 4 transactions each
        long lagBytes = 1 * 4 * _transactionPatternSize;
        /*
       * Assert.assertEquals(totalFilesWritten-1, ggParserStats.getNumFilesParsed());
       * Assert.assertEquals(totalFilesWritten, ggParserStats.getNumFilesAdded());
       * Assert.assertEquals(lagFiles, ggParserStats.getFilesLag()); // because 303 got
       * parsed
       *
       * // we added 4 files and parsed 3 , so the diff should be 1 file size (4
       * trasactions in 1 file) Assert.assertEquals(lagBytes,
       * ggParserStats.getBytesLag()); Assert.assertTrue(ggParserStats.getTimeLag()>0);
       */
        gg.unpause();
        TestUtil.sleep(5000);
        // now we should catchup
        Assert.assertEquals(4, ggParserStats.getNumFilesParsed());
        Assert.assertEquals(4, ggParserStats.getNumFilesAdded());
        Assert.assertEquals(0, ggParserStats.getFilesLag());
        Assert.assertEquals(0, ggParserStats.getTimeLag());
        Assert.assertEquals(0, ggParserStats.getBytesLag());
        // append to a file
        LOG.info("pausing again");
        gg.pause();
        addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x304"), 410, 4);
        totalTransWritten += 4;
        TestUtil.sleep(1000);
        addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x304"), 420, 4);
        totalTransWritten += 4;
        TestUtil.sleep(2000);
        gg.unpause();
        TestUtil.sleep(5500);
        // should be still up
        Assert.assertEquals(4, ggParserStats.getNumFilesParsed());
        Assert.assertEquals(4, ggParserStats.getNumFilesAdded());
        Assert.assertEquals(0, ggParserStats.getFilesLag());
        Assert.assertEquals(0, ggParserStats.getTimeLag());
        Assert.assertEquals(0, ggParserStats.getBytesLag());
        // assert the stats
        int totalFilesSize = totalTransWritten * _transactionPatternSize;
        Assert.assertEquals((totalFilesSize / totalFilesWritten), ggParserStats.getAvgFileSize());
        Assert.assertEquals(true, ggParserStats.getAvgParseTransactionTimeNs() > 0);
        Assert.assertEquals("part1", ggParserStats.getPhysicalSourceName());
        Assert.assertEquals(totalFilesSize / totalTransWritten, ggParserStats.getAvgTransactionSize());
        Assert.assertEquals(423, ggParserStats.getMaxScn());
        // 2
        Assert.assertEquals(totalTransWritten * 2, ggParserStats.getNumTotalEvents());
        // events
        // per
        // transaction
        Assert.assertEquals(totalTransWritten, ggParserStats.getNumTransactionsTotal());
        Assert.assertEquals(totalTransWritten, ggParserStats.getNumTransactionsWithEvents());
        Assert.assertEquals(0, ggParserStats.getNumTransactionsWithoutEvents());
        Assert.assertEquals(true, ggParserStats.getTimeSinceLastAccessMs() > 0);
        Assert.assertEquals(totalTransWritten * _transactionPatternSize, ggParserStats.getNumBytesTotalParsed());
        Assert.assertEquals("NumSCNRegressions", 0, ggParserStats.getNumSCNRegressions());
        Assert.assertEquals("LastSCNRegressed", -1, ggParserStats.getLastRegressedScn());
    } finally {
        gg.shutdown();
    }
    return;
}
Also used : ConditionCheck(com.linkedin.databus2.test.ConditionCheck) PhysicalSourceStaticConfig(com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig) MaxSCNReaderWriter(com.linkedin.databus2.core.seq.MaxSCNReaderWriter) GGParserStatistics(com.linkedin.databus.monitoring.mbean.GGParserStatistics) VersionedSchemaId(com.linkedin.databus2.schemas.VersionedSchemaId) Schema(org.apache.avro.Schema) VersionedSchema(com.linkedin.databus2.schemas.VersionedSchema) SchemaRegistryService(com.linkedin.databus2.schemas.SchemaRegistryService) DbusEventBufferAppendable(com.linkedin.databus.core.DbusEventBufferAppendable) VersionedSchema(com.linkedin.databus2.schemas.VersionedSchema) File(java.io.File) Test(org.testng.annotations.Test)

Aggregations

ArrayList (java.util.ArrayList)7 DbusEventBufferAppendable (com.linkedin.databus.core.DbusEventBufferAppendable)6 PhysicalSourceStaticConfig (com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig)6 TransactionInfo (com.linkedin.databus.monitoring.mbean.GGParserStatistics.TransactionInfo)5 Test (org.testng.annotations.Test)5 GGParserStatistics (com.linkedin.databus.monitoring.mbean.GGParserStatistics)3 VersionedSchema (com.linkedin.databus2.schemas.VersionedSchema)3 Method (java.lang.reflect.Method)3 Schema (org.apache.avro.Schema)3 EventSourceStatistics (com.linkedin.databus.monitoring.mbean.EventSourceStatistics)2 MaxSCNReaderWriter (com.linkedin.databus2.core.seq.MaxSCNReaderWriter)2 ColumnsState (com.linkedin.databus2.ggParser.XmlStateMachine.ColumnsState)2 DBUpdateImage (com.linkedin.databus2.ggParser.XmlStateMachine.DbUpdateState.DBUpdateImage)2 TransactionState (com.linkedin.databus2.ggParser.XmlStateMachine.TransactionState)2 EventProducer (com.linkedin.databus2.producers.EventProducer)2 EventSourceStatisticsIface (com.linkedin.databus2.producers.db.EventSourceStatisticsIface)2 SchemaRegistryService (com.linkedin.databus2.schemas.SchemaRegistryService)2 HashSet (java.util.HashSet)2 GenericRecord (org.apache.avro.generic.GenericRecord)2 ControlSourceEventsRequestProcessor (com.linkedin.databus.container.request.ControlSourceEventsRequestProcessor)1