Examples with IdNamePair com.linkedin.databus.core.util.IdNamePair used on opensource projects

Example 16 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class BootstrapPullThread method doRequestBootstrapStream.

protected void doRequestBootstrapStream(ConnectionState curState) {
    boolean debugEnabled = _log.isDebugEnabled();
    if (debugEnabled)
        _log.debug("Checking for free space");
    //curState.getDataEventsBuffer().waitForFreeSpace(FREE_BUFFER_THRESHOLD);
    int freeBufferThreshold = (int) (_sourcesConn.getConnectionConfig().getFreeBufferThreshold() * 100.0 / _pullerBufferUtilizationPct);
    int freeSpace = curState.getDataEventsBuffer().getBufferFreeReadSpace();
    if (freeSpace >= freeBufferThreshold) {
        Checkpoint cp = curState.getCheckpoint();
        if (debugEnabled)
            _log.debug("Checkpoint at RequestBootstrapData: " + cp.toString());
        _log.debug("Sending /bootstrap request");
        Map<String, IdNamePair> srcNameMap = curState.getSourcesNameMap();
        String curSrcName = null;
        if (cp.getConsumptionMode() == DbusClientMode.BOOTSTRAP_SNAPSHOT) {
            curSrcName = cp.getSnapshotSource();
        } else {
            curSrcName = cp.getCatchupSource();
        }
        if (null == _bootstrapFilter) {
            _bootstrapFilter = new DbusKeyCompositeFilter();
            Map<String, IdNamePair> srcNameIdMap = curState.getSourcesNameMap();
            for (DbusKeyCompositeFilterConfig conf : _bootstrapFilterConfigs) {
                Map<String, KeyFilterConfigHolder> cMap = conf.getConfigMap();
                Map<Long, KeyFilterConfigHolder> fConfMap = new HashMap<Long, KeyFilterConfigHolder>();
                for (Entry<String, KeyFilterConfigHolder> e : cMap.entrySet()) {
                    IdNamePair idName = srcNameIdMap.get(e.getKey());
                    if (null != idName) {
                        fConfMap.put(idName.getId(), e.getValue());
                    }
                }
                _bootstrapFilter.merge(new DbusKeyCompositeFilter(fConfMap));
            }
            _bootstrapFilter.dedupe();
        }
        DbusKeyFilter filter = null;
        IdNamePair srcEntry = srcNameMap.get(curSrcName);
        if (null != srcEntry) {
            Map<Long, DbusKeyFilter> fMap = _bootstrapFilter.getFilterMap();
            if (null != fMap)
                filter = fMap.get(srcEntry.getId());
        }
        int fetchSize = (int) ((curState.getDataEventsBuffer().getBufferFreeReadSpace() / 100.0) * _pullerBufferUtilizationPct);
        fetchSize = Math.max(freeBufferThreshold, fetchSize);
        curState.switchToStreamRequestSent();
        sendHeartbeat(_sourcesConn.getUnifiedClientStats());
        curState.getBootstrapConnection().requestStream(curState.getSourcesIdListString(), filter, fetchSize, cp, curState);
    } else {
        try {
            Thread.sleep(50);
        } catch (InterruptedException ie) {
        }
        enqueueMessage(curState);
    }
}

Example 17 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class TestMultiConsumerCallback method test2ConsumerTimeout.

@Test
public void test2ConsumerTimeout() {
    Logger log = Logger.getLogger("TestMultiConsumerCallback.test2ConsumerTimeout");
    //Logger.getRootLogger().setLevel(Level.INFO);
    log.info("\n\nstarting test2ConsumerTimeout()");
    log.info("create dummy events");
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 2, (short) 1, keyCounts);
    initBufferWithEvents(eventsBuf, 3, 1, (short) 2, keyCounts);
    eventsBuf.endEvents(100L);
    DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    //skip over the first system event
    iter.next();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event1 = iter.next().createCopy();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event2 = iter.next().createCopy();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    log.info("make up some sources");
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    log.info("create the consumer mock up");
    DatabusStreamConsumer mockConsumer1 = EasyMock.createStrictMock("consumer1", DatabusStreamConsumer.class);
    SelectingDatabusCombinedConsumer sdccMockConsumer1 = new SelectingDatabusCombinedConsumer(mockConsumer1);
    EasyMock.makeThreadSafe(mockConsumer1, true);
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer1, sources, null);
    EasyMock.expect(mockConsumer1.onStartConsumption()).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "startConsumption() called"), 1));
    EasyMock.expect(mockConsumer1.onStartDataEventSequence(null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartDataEventSequence() called"), 1));
    EasyMock.expect(mockConsumer1.onStartSource("source1", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 1));
    EasyMock.expect(mockConsumer1.onDataEvent(event1, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(1) called"), 1));
    EasyMock.expect(mockConsumer1.onDataEvent(event2, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(2) called"), 1));
    EasyMock.expect(mockConsumer1.onDataEvent(event1, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(1) called"), 1));
    EasyMock.expect(mockConsumer1.onEndSource("source1", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 1));
    EasyMock.replay(mockConsumer1);
    ConsumerCallbackStats consumerStatsCollector = new ConsumerCallbackStats(1, "test", "test", true, false, null);
    UnifiedClientStats unifiedStatsCollector = new UnifiedClientStats(1, "test", "test.unified");
    log.info("Create and fire up callbacks");
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    TimingOutMultiConsumerCallback callback = new TimingOutMultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 300, new StreamConsumerCallbackFactory(consumerStatsCollector, unifiedStatsCollector), consumerStatsCollector, unifiedStatsCollector, 3);
    callback.setSourceMap(sourcesMap);
    ConsumerCallbackResult startConsumptionRes = callback.onStartConsumption();
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(startConsumptionRes), "startConsumption() succeeded: " + startConsumptionRes);
    ConsumerCallbackResult startWindowRes = callback.onStartDataEventSequence(null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(startWindowRes), "startDataEventSequence() succeeded");
    ConsumerCallbackResult startSourceRes = callback.onStartSource("source1", null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(startSourceRes), "startSources(source1) succeeded");
    ConsumerCallbackResult event1Res = callback.onDataEvent(event1, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event1Res), "onDataEvent(1) succeeded");
    ConsumerCallbackResult event2Res = callback.onDataEvent(event2, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event2Res), "onDataEvent(2) queued up");
    ConsumerCallbackResult event3Res = callback.onDataEvent(event1, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event3Res), "onDataEvent(1) queued up");
    ConsumerCallbackResult endSourceRes = callback.onEndSource("source1", null);
    Assert.assertTrue(ConsumerCallbackResult.isFailure(endSourceRes), "onEndSource fails because of timeout in onDataEvent(2)");
    EasyMock.reset(mockConsumer1);
    log.info("test2ConsumerTimeout: end");
}

Example 18 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class MultiConsumerCallback method onDataEvent.

@Override
public ConsumerCallbackResult onDataEvent(DbusEvent e, DbusEventDecoder eventDecoder) {
    boolean debugEnabled = _log.isDebugEnabled();
    long curNanos = System.nanoTime();
    if (null == _sourceMap) {
        _log.error("No sources map specified");
        if (_consumerStats != null)
            _consumerStats.registerSrcErrors();
        return ConsumerCallbackResult.ERROR;
    }
    long srcid = e.srcId();
    short lPartitionId = e.logicalPartitionId();
    IdNamePair eventSource = _sourceMap.get(srcid);
    if (null == eventSource) {
        _log.error("Unknown source");
        if (_consumerStats != null)
            _consumerStats.registerSrcErrors();
        return ConsumerCallbackResult.ERROR;
    }
    for (DatabusV2ConsumerRegistration reg : _registrations) {
        DatabusSubscription eventSourceName = DatabusSubscription.createSubscription(eventSource, lPartitionId);
        if (debugEnabled)
            _log.debug("event source=" + eventSource + " lpart=" + lPartitionId);
        if (reg.checkSourceSubscription(eventSourceName)) {
            if (debugEnabled)
                _log.debug("consumer matches:" + reg.getConsumer());
            ConsumerCallable<ConsumerCallbackResult> dataEventCallable = _callbackFactory.createDataEventCallable(curNanos, e, eventDecoder, reg.getConsumer(), true);
            _currentBatch.add(dataEventCallable);
            if (_consumerStats != null)
                _consumerStats.registerDataEventReceived(e);
            if (_unifiedClientStats != null)
                _unifiedClientStats.registerDataEventReceived(e);
        }
    }
    if (_loggingConsumer != null) {
        ConsumerCallable<ConsumerCallbackResult> dataEventCallable = _callbackFactory.createDataEventCallable(curNanos, e, eventDecoder, _loggingConsumer, false);
        _currentBatch.add(dataEventCallable);
    }
    if (debugEnabled) {
        long endNanos = System.nanoTime();
        _log.debug("Time spent in databus clientlib by onDataEvent = " + (endNanos - curNanos) / DbusConstants.NUM_NSECS_IN_MSEC + "ms");
    }
    return submitBatch(curNanos, false, false);
}

Example 19 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class SourcesMessage method calcSourcesIdListString.

private static String calcSourcesIdListString(Collection<IdNamePair> sources) {
    StringBuilder sb = new StringBuilder();
    boolean first = true;
    for (IdNamePair pair : sources) {
        if (!first) {
            sb.append(",");
        }
        sb.append(pair.getId());
        first = false;
    }
    return sb.toString();
}

Example 20 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class MockBootstrapConnection method testTransition_StreamResponsePhaseCompleted.

@Test
public /** Test bootstrap transition: Stream_Response_Success - Happy Path: Phase Completed */
void testTransition_StreamResponsePhaseCompleted() throws Exception {
    BootstrapPullThread bsPuller = createBootstrapPullThread(false, false, false, false, false, null, 12000, 1, true);
    Checkpoint cp = _ckptHandlerSource1.createInitialBootstrapCheckpoint(null, 0L);
    //TODO remove
    //cp.setSnapshotSource("source1");
    //cp.setCatchupSource("source1");
    //cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    cp.setBootstrapServerInfo(_serverInfoName);
    bsPuller.getComponentStatus().start();
    ConnectionState connState = bsPuller.getConnectionState();
    connState.switchToBootstrap(cp);
    testTransitionCase(bsPuller, StateId.BOOTSTRAP, StateId.REQUEST_START_SCN, cp);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.REQUEST_START_SCN, StateId.START_SCN_RESPONSE_SUCCESS, null);
    bsPuller.getMessageQueue().clear();
    Map<Long, List<RegisterResponseEntry>> entries = new HashMap<Long, List<RegisterResponseEntry>>();
    entries.put(1L, new ArrayList<RegisterResponseEntry>());
    connState.setSourcesSchemas(entries);
    testTransitionCase(bsPuller, StateId.START_SCN_RESPONSE_SUCCESS, StateId.REQUEST_STREAM, null);
    bsPuller.getMessageQueue().clear();
    connState.getSourcesNameMap().put("source1", new IdNamePair(1L, "source1"));
    connState.getSourceIdMap().put(1L, new IdNamePair(1L, "source1"));
    testTransitionCase(bsPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS, null);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.STREAM_RESPONSE_DONE, null);
}