Examples with PhysicalSourceStaticConfig com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig used on opensource projects

Example 21 with PhysicalSourceStaticConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig in project databus by linkedin.

the class TestGoldenGateEventProducer method testGGProducerStats.

@Test
public void testGGProducerStats() throws InvalidConfigException, UnsupportedKeyException, DatabusException, IllegalArgumentException, InstantiationException, IllegalAccessException, InvocationTargetException, SecurityException, NoSuchMethodException {
    short[] sourceIds = new short[] { 505, 506 };
    String[] sourceNames = new String[] { "source1", "source2" };
    PhysicalSourceStaticConfig pssc = buildSimplePssc(sourceIds, sourceNames, "gg:///tmp:xxx");
    long scn = 10;
    DbusEventBufferAppendable mb = createBufMult(pssc);
    // start producer
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    Object handleXmlCallbackObject = getHandleXmlCallbackInnerInstance(gg);
    Method method = getOnTransactionEndMethod();
    // generates the updates
    List<String> keys = new ArrayList<String>();
    keys.add("keys1");
    // SCN = 10
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = generateUpdates(new short[] { sourceIds[0] }, keys, scn);
    // SCN = 11
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates1 = generateUpdates(new short[] { sourceIds[1] }, keys, scn + 1);
    // SCN = 12
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates2 = generateUpdates(new short[] { sourceIds[1] }, keys, scn + 2);
    long timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC;
    // gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
    // gg.addEventToBuffer(dbUpdates1, new TransactionInfo(0, 0, timestamp+1, scn+1));
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates, new TransactionInfo(0, 0, timestamp, scn) });
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp + 1, scn + 1) });
    // THis is an extra-call but the corresponding events will not be added to EVB.
    // This is needed to flush the events in the above call to EVB
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp + 1, scn + 2) });
    for (EventSourceStatisticsIface si : gg.getSources()) {
        EventSourceStatistics ss = si.getStatisticsBean();
        LOG.info(si.getSourceName() + ": scn=" + ss.getMaxScn() + ",averageSize=" + ss.getAvgEventSerializedSize() + ",numErrors=" + ss.getNumErrors() + ",totalEvents=" + ss.getNumTotalEvents() + ",averageFactTime=" + ss.getAvgEventFactoryTimeMillisPerEvent() + ",timeSinceDb=" + ss.getTimeSinceLastDBAccess());
        if (si.getSourceId() == 505) {
            Assert.assertEquals(6, ss.getAvgEventSerializedSize());
            Assert.assertEquals(0, ss.getNumErrors());
            Assert.assertEquals(1, ss.getNumTotalEvents());
            Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent());
            // we are not
            Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent());
            // really
            // reading
            Assert.assertEquals(10, ss.getMaxScn());
        }
        if (si.getSourceId() == 506) {
            Assert.assertEquals(6, ss.getAvgEventSerializedSize());
            Assert.assertEquals(0, ss.getNumErrors());
            Assert.assertEquals(1, ss.getNumTotalEvents());
            Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent());
            // we are not
            Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent());
            // really
            // reading
            Assert.assertEquals(11, ss.getMaxScn());
        }
        if (si.getSourceId() == GoldenGateEventProducer.GLOBAL_SOURCE_ID) {
            Assert.assertEquals(6, ss.getAvgEventSerializedSize());
            Assert.assertEquals(0, ss.getNumErrors());
            Assert.assertEquals(2, ss.getNumTotalEvents());
            Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent());
            // we are not
            Assert.assertEquals(0, ss.getAvgEventFactoryTimeMillisPerEvent());
            // really
            // reading
            Assert.assertEquals(11, ss.getMaxScn());
        }
    }
    long approximateTimeSinceLastTransactionMs = System.currentTimeMillis() - timestamp / DbusConstants.NUM_NSECS_IN_MSEC;
    long diff = gg.getParserStats().getTimeSinceLastTransactionMs() - approximateTimeSinceLastTransactionMs;
    // somewhat
    Assert.assertTrue("time diff is too big:" + diff, diff >= 0 && diff < 30);
    // NS)
    return;
}

Example 22 with PhysicalSourceStaticConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig in project databus by linkedin.

the class TestGoldenGateEventProducer method buildPssc.

private PhysicalSourceStaticConfig buildPssc(long rate, long throttleDuration) throws InvalidConfigException {
    String name = "anet";
    short id = 505, partition = 0;
    String uri = "gg:///mnt/dbext/x1";
    String resourceKey = "test";
    String partitionFunction = "constant:1";
    boolean skipInfinityScn = false;
    String queryHints = null;
    LogicalSourceStaticConfig[] sources = new LogicalSourceStaticConfig[1];
    LogicalSourceStaticConfig lssc = new LogicalSourceStaticConfig(id, name, uri, partitionFunction, partition, skipInfinityScn, queryHints, queryHints, queryHints);
    sources[0] = lssc;
    String role = "MASTER";
    long slowSourceQueryThreshold = 0L;
    long restartScnOffset = 0L;
    BackoffTimerStaticConfigBuilder bsc = new BackoffTimerStaticConfigBuilder();
    ChunkingType ct = ChunkingType.NO_CHUNKING;
    long txnsPerChunk = 0L;
    long scnChunkSize = 0L;
    long chunkedScnThreshold = 0L;
    long maxScnDelayMs = 0L;
    long eventRatePerSec = rate;
    long eventRateThrottleDuration = throttleDuration;
    int largestEventSizeInBytes = 10240;
    long largestWindowSizeInBytes = 10240;
    DbusEventBuffer.Config cfgBuilder = new DbusEventBuffer.Config();
    cfgBuilder.setMaxSize(10 * 1024 * 1024);
    cfgBuilder.setScnIndexSize(2 * 1024 * 1024);
    cfgBuilder.setAllocationPolicy("MMAPPED_MEMORY");
    DbusEventBuffer.StaticConfig dbusEventBuffer = cfgBuilder.build();
    boolean errorOnMissingFields = true;
    String xmlVersion = "1.0";
    String xmlEncoding = "";
    String fieldName = "";
    String regex = "";
    ReplicationBitSetterStaticConfig replicationBitSetter = new ReplicationBitSetterStaticConfig(ReplicationBitSetterStaticConfig.SourceType.NONE, fieldName, regex, MissingValueBehavior.STOP_WITH_ERROR);
    PhysicalSourceStaticConfig pssc = new PhysicalSourceStaticConfig(name, id, uri, resourceKey, sources, role, slowSourceQueryThreshold, restartScnOffset, bsc.build(), ct, txnsPerChunk, scnChunkSize, chunkedScnThreshold, maxScnDelayMs, eventRatePerSec, eventRateThrottleDuration, dbusEventBuffer, largestEventSizeInBytes, largestWindowSizeInBytes, errorOnMissingFields, xmlVersion, xmlEncoding, replicationBitSetter);
    return pssc;
}

Example 23 with PhysicalSourceStaticConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig in project databus by linkedin.

the class TestGoldenGateEventProducer method buildSimplePssc.

private PhysicalSourceStaticConfig buildSimplePssc(short[] sourceIds, String[] sourceNames, String uri) throws InvalidConfigException {
    String partitionFunction = "constant:1";
    String pPartName = "part1";
    short pPartId = 0;
    PhysicalSourceConfig pc = new PhysicalSourceConfig(pPartName, uri, pPartId);
    for (int i = 0; i < sourceIds.length; i++) {
        LogicalSourceConfig lc = new LogicalSourceConfig();
        lc.setId(sourceIds[i]);
        lc.setName(sourceNames[i]);
        lc.setPartitionFunction(partitionFunction);
        // this format is expected by GG
        lc.setUri(pPartName + "." + sourceNames[i]);
        pc.addSource(lc);
    }
    return pc.build();
}

Example 24 with PhysicalSourceStaticConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig in project databus by linkedin.

the class TestGoldenGateEventProducer method testSCNRegressionStats.

@Test
public void testSCNRegressionStats() throws Exception {
    short[] sourceIds = new short[] { 505, 506 };
    String[] sourceNames = new String[] { "source1", "source2" };
    PhysicalSourceStaticConfig pssc = buildSimplePssc(sourceIds, sourceNames, "gg:///tmp:xxx");
    DbusEventBufferAppendable mb = createBufMult(pssc);
    // start producer
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    Object handleXmlCallbackObject = getHandleXmlCallbackInnerInstance(gg);
    Method method = getOnTransactionEndMethod();
    // generates the updates
    List<String> keys = new ArrayList<String>();
    keys.add("key1");
    GGParserStatistics ggParserStats = gg.getParserStats();
    // SCN = 10
    long timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC;
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates1 = generateUpdates(sourceIds, keys, 10);
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, 10) });
    timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;
    Assert.assertEquals("NumSCNRegressions", 0, ggParserStats.getNumSCNRegressions());
    Assert.assertEquals("NumSCNRegressions", -1, ggParserStats.getLastRegressedScn());
    Assert.assertEquals("MaxScn", 10, ggParserStats.getMaxScn());
    // SCN = 5 Regression
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates2 = // SCN Regression here
    generateUpdates(sourceIds, keys, 5);
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp + 1, 5) });
    Assert.assertEquals("NumSCNRegressions", 1, ggParserStats.getNumSCNRegressions());
    Assert.assertEquals("NumSCNRegressions", 5, ggParserStats.getLastRegressedScn());
    Assert.assertEquals("MaxScn", 10, ggParserStats.getMaxScn());
    // SCN = 6 No Regression here
    dbUpdates2 = generateUpdates(sourceIds, keys, 6);
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp + 1, 6) });
    Assert.assertEquals("NumSCNRegressions", 1, ggParserStats.getNumSCNRegressions());
    Assert.assertEquals("NumSCNRegressions", 5, ggParserStats.getLastRegressedScn());
    Assert.assertEquals("MaxScn", 10, ggParserStats.getMaxScn());
    // SCN = 3 : SCN regression again
    dbUpdates2 = generateUpdates(sourceIds, keys, 3);
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp + 1, 3) });
    Assert.assertEquals("NumSCNRegressions", 2, ggParserStats.getNumSCNRegressions());
    Assert.assertEquals("NumSCNRegressions", 3, ggParserStats.getLastRegressedScn());
    Assert.assertEquals("MaxScn", 10, ggParserStats.getMaxScn());
    // SCN = 11: No regression here
    dbUpdates2 = generateUpdates(sourceIds, keys, 11);
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp + 1, 11) });
    Assert.assertEquals("NumSCNRegressions", 2, ggParserStats.getNumSCNRegressions());
    Assert.assertEquals("NumSCNRegressions", 3, ggParserStats.getLastRegressedScn());
    Assert.assertEquals("MaxScn", 11, ggParserStats.getMaxScn());
}

Example 25 with PhysicalSourceStaticConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig in project databus by linkedin.

the class PhysicalSourceConfigBuilder method build.

@Override
public PhysicalSourceStaticConfig[] build() throws InvalidConfigException {
    ObjectMapper mapper = new ObjectMapper();
    PhysicalSourceStaticConfig[] list = new PhysicalSourceStaticConfig[null == _fileNames ? 0 : _fileNames.length];
    if (_fileNames == null)
        return list;
    for (int i = 0; i < _fileNames.length; ++i) {
        File sourceJson = _fileNames[i];
        PhysicalSourceConfig pConfig = null;
        Exception e = null;
        try {
            pConfig = mapper.readValue(sourceJson, PhysicalSourceConfig.class);
        } catch (JsonParseException jpe) {
            e = jpe;
        } catch (JsonMappingException jme) {
            e = jme;
        } catch (IOException ioe) {
            e = ioe;
        }
        if (e != null || pConfig == null) {
            throw new InvalidConfigException(e);
        }
        pConfig.checkForNulls();
        LOG.info("Generated Physical source config: name= " + pConfig.getId());
        list[i] = pConfig.build();
    }
    /*
    for(PhysicalSourceStaticConfig pCfg : pConfigs) {
      for(LogicalSourceStaticConfig lSC : pCfg.getSources()) {
        config.setSourceName("" + lSC.getId(), lSC.getName());
      }
    }
    */
    return list;
}