Examples with DatabusSubscription com.linkedin.databus.core.data_model.DatabusSubscription used on opensource projects

Example 6 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class DatabusHttpClientImpl method initializeRelayConnections.

private synchronized void initializeRelayConnections() {
    for (List<DatabusSubscription> subsList : _relayGroups.keySet()) {
        List<String> sourcesStrList = DatabusSubscription.getStrList(subsList);
        List<DatabusV2ConsumerRegistration> relayConsumers = getRelayGroupStreamConsumers().get(subsList);
        //nothing to do
        if (null == relayConsumers || 0 == relayConsumers.size())
            continue;
        try {
            DatabusSourcesConnection.StaticConfig connConfig = getClientStaticConfig().getConnection(sourcesStrList);
            if (null == connConfig) {
                connConfig = getClientStaticConfig().getConnectionDefaults();
            }
            // make sure we have the right policy.
            if (!connConfig.getEventBuffer().isEnableScnIndex() && connConfig.getEventBuffer().getQueuePolicy() != DbusEventBuffer.QueuePolicy.BLOCK_ON_WRITE) {
                throw new InvalidConfigException("If SCN index is disabled, queue policy must be BLOCK_ON_WRITE");
            }
            CheckpointPersistenceProvider cpPersistenceProvder = getCheckpointPersistenceProvider();
            if (null != cpPersistenceProvder && getClientStaticConfig().getCheckpointPersistence().isClearBeforeUse()) {
                cpPersistenceProvder.removeCheckpoint(sourcesStrList);
            }
            ServerInfo server0 = _relayGroups.get(subsList).iterator().next();
            ArrayList<DatabusV2ConsumerRegistration> bstConsumersRegs = new ArrayList<DatabusV2ConsumerRegistration>();
            for (List<DatabusSubscription> bstSubSourcesList : getRelayGroupBootstrapConsumers().keySet()) {
                List<DatabusV2ConsumerRegistration> bstRegsistrations = getRelayGroupBootstrapConsumers().get(bstSubSourcesList);
                for (DatabusV2ConsumerRegistration bstConsumerReg : bstRegsistrations) {
                    if (server0.supportsSources(bstConsumerReg.getSources())) {
                        bstConsumersRegs.add(bstConsumerReg);
                    }
                }
            }
            DbusEventBuffer eventBuffer = connConfig.getEventBuffer().getOrCreateEventBuffer(_eventFactory);
            eventBuffer.setDropOldEvents(true);
            eventBuffer.start(0);
            DbusEventBuffer bootstrapBuffer = null;
            // create bootstrap only if it is enabled
            if (_clientStaticConfig.getRuntime().getBootstrap().isEnabled()) {
                bootstrapBuffer = new DbusEventBuffer(connConfig.getEventBuffer());
                bootstrapBuffer.setDropOldEvents(false);
                bootstrapBuffer.start(0);
            }
            LOG.info("The sourcesList is " + sourcesStrList);
            LOG.info("The relayGroupStreamConsumers is " + getRelayGroupStreamConsumers().get(subsList));
            Set<ServerInfo> relays = _relayGroups.get(subsList);
            Set<ServerInfo> bootstrapServices = _bootstrapGroups.get(subsList);
            String statsCollectorName = generateSubsStatsName(sourcesStrList);
            int ownerId = getContainerStaticConfig().getId();
            _bootstrapEventsStats.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".inbound.bs", true, false, getMbeanServer()));
            _inBoundStatsCollectors.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".inbound", true, false, getMbeanServer()));
            _outBoundStatsCollectors.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".outbound", true, false, getMbeanServer()));
            _consumerStatsCollectors.addStatsCollector(statsCollectorName, new ConsumerCallbackStats(ownerId, statsCollectorName + ".inbound.cons", statsCollectorName + ".inbound.cons", true, false, null, getMbeanServer()));
            _bsConsumerStatsCollectors.addStatsCollector(statsCollectorName, new ConsumerCallbackStats(ownerId, statsCollectorName + ".inbound.bs.cons", statsCollectorName + ".inbound.bs.cons", true, false, null, getMbeanServer()));
            _unifiedClientStatsCollectors.addStatsCollector(statsCollectorName, new UnifiedClientStats(ownerId, statsCollectorName + ".inbound.unified.cons", statsCollectorName + ".inbound.unified.cons", true, false, _clientStaticConfig.getPullerThreadDeadnessThresholdMs(), null, getMbeanServer()));
            ConnectionStateFactory connStateFactory = new ConnectionStateFactory(DatabusSubscription.getStrList(subsList));
            DatabusSourcesConnection newConn = new DatabusSourcesConnection(connConfig, subsList, relays, bootstrapServices, relayConsumers, //_relayGroupBootstrapConsumers.get(sourcesList),
            bstConsumersRegs, eventBuffer, bootstrapBuffer, getDefaultExecutorService(), getContainerStatsCollector(), _inBoundStatsCollectors.getStatsCollector(statsCollectorName), _bootstrapEventsStats.getStatsCollector(statsCollectorName), _consumerStatsCollectors.getStatsCollector(statsCollectorName), _bsConsumerStatsCollectors.getStatsCollector(statsCollectorName), _unifiedClientStatsCollectors.getStatsCollector(statsCollectorName), getCheckpointPersistenceProvider(), getRelayConnFactory(), getBootstrapConnFactory(), getHttpStatsCollector(), null, this, _eventFactory, connStateFactory);
            newConn.start();
            _relayConnections.add(newConn);
        } catch (Exception e) {
            LOG.error("connection initialization issue for source(s):" + subsList + "; please check your configuration", e);
        }
    }
    if (0 == _relayConnections.size()) {
        LOG.warn("No connections specified");
    }
}

Example 7 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class RelayPullThread method doRequestStream.

protected void doRequestStream(ConnectionState curState) {
    boolean debugEnabled = _log.isDebugEnabled();
    if (debugEnabled)
        _log.debug("Checking for free space in buffer");
    int freeBufferThreshold = (int) (_sourcesConn.getConnectionConfig().getFreeBufferThreshold() * 100.0 / _pullerBufferUtilizationPct);
    try {
        curState.getDataEventsBuffer().waitForFreeSpace(freeBufferThreshold);
    } catch (InterruptedException ie) {
        //loop
        enqueueMessage(curState);
        return;
    }
    Checkpoint cp = curState.getCheckpoint();
    if (debugEnabled)
        _log.debug("Checkpoint at RequestDataEvents: " + cp.toString());
    if (null == _relayFilter) {
        if (debugEnabled)
            _log.debug("Initializing relay filter config");
        _relayFilter = new DbusKeyCompositeFilter();
        Map<String, IdNamePair> srcNameIdMap = curState.getSourcesNameMap();
        for (DbusKeyCompositeFilterConfig conf : _relayFilterConfigs) {
            Map<String, KeyFilterConfigHolder> cMap = conf.getConfigMap();
            Map<Long, KeyFilterConfigHolder> fConfMap = new HashMap<Long, KeyFilterConfigHolder>();
            for (Entry<String, KeyFilterConfigHolder> e : cMap.entrySet()) {
                IdNamePair idName = srcNameIdMap.get(e.getKey());
                if (null != idName) {
                    fConfMap.put(idName.getId(), e.getValue());
                }
            }
            if (debugEnabled)
                _log.debug("FilterConfMap is :" + fConfMap);
            _relayFilter.merge(new DbusKeyCompositeFilter(fConfMap));
        }
        if (debugEnabled)
            _log.debug("Merged Filter (before deduping) is :" + _relayFilter);
        _relayFilter.dedupe();
        if (debugEnabled)
            _log.debug("Merged Filter (after deduping) is :" + _relayFilter);
    }
    _streamCallStartMs = System.currentTimeMillis();
    if (null != _relayCallsStats)
        _relayCallsStats.registerStreamRequest(cp, EMPTY_STREAM_LIST);
    int fetchSize = (int) ((curState.getDataEventsBuffer().getBufferFreeReadSpace() / 100.0) * _pullerBufferUtilizationPct);
    fetchSize = Math.max(freeBufferThreshold, fetchSize);
    CheckpointMult cpMult = new CheckpointMult();
    String args;
    if (curState.getRelayConnection().getProtocolVersion() >= 3) {
        // for version 3 and higher we pass subscriptions
        args = curState.getSubsListString();
        for (DatabusSubscription sub : curState.getSubscriptions()) {
            PhysicalPartition p = sub.getPhysicalPartition();
            cpMult.addCheckpoint(p, cp);
        }
    } else {
        args = curState.getSourcesIdListString();
        cpMult.addCheckpoint(PhysicalPartition.ANY_PHYSICAL_PARTITION, cp);
    }
    curState.switchToStreamRequestSent();
    sendHeartbeat(_sourcesConn.getUnifiedClientStats());
    curState.getRelayConnection().requestStream(args, _relayFilter, fetchSize, cpMult, _sourcesConn.getConnectionConfig().getKeyRange(), curState);
}

Example 8 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class DatabusHttpClientImpl method registerDatabusListener.

protected List<DatabusV2ConsumerRegistration> registerDatabusListener(DatabusV2ConsumerRegistration listener, Map<List<DatabusSubscription>, Set<ServerInfo>> groupsServers, Map<List<DatabusSubscription>, List<DatabusV2ConsumerRegistration>> groupsListeners, List<DatabusSubscription> sources) throws DatabusClientException {
    List<DatabusSubscription> subsSources = null;
    ServerInfo randomRelay = getRandomRelay(groupsServers, sources);
    if (null == randomRelay) {
        // even if there is no relay available to serve it immediately.
        assert getClientStaticConfig().usesDynamicRelayConfiguration() : "Client relay(s) configured statically but no relays available at listener registration";
        subsSources = sources;
    } else {
        try {
            subsSources = DatabusSubscription.createFromUriList(randomRelay.getSources());
        } catch (DatabusException e) {
            throw new DatabusClientException("source list decode error:" + e.getMessage(), e);
        } catch (URISyntaxException e) {
            throw new DatabusClientException("source list decode error:" + e.getMessage(), e);
        }
    }
    List<DatabusV2ConsumerRegistration> consumers = getListOfConsumerRegsFromSubList(groupsListeners, subsSources);
    if (null == consumers) {
        consumers = new CopyOnWriteArrayList<DatabusV2ConsumerRegistration>();
        groupsListeners.put(subsSources, consumers);
    }
    consumers.add(listener);
    return consumers;
}

Example 9 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class MultiConsumerCallback method onDataEvent.

@Override
public ConsumerCallbackResult onDataEvent(DbusEvent e, DbusEventDecoder eventDecoder) {
    boolean debugEnabled = _log.isDebugEnabled();
    long curNanos = System.nanoTime();
    if (null == _sourceMap) {
        _log.error("No sources map specified");
        if (_consumerStats != null)
            _consumerStats.registerSrcErrors();
        return ConsumerCallbackResult.ERROR;
    }
    long srcid = e.srcId();
    short lPartitionId = e.logicalPartitionId();
    IdNamePair eventSource = _sourceMap.get(srcid);
    if (null == eventSource) {
        _log.error("Unknown source");
        if (_consumerStats != null)
            _consumerStats.registerSrcErrors();
        return ConsumerCallbackResult.ERROR;
    }
    for (DatabusV2ConsumerRegistration reg : _registrations) {
        DatabusSubscription eventSourceName = DatabusSubscription.createSubscription(eventSource, lPartitionId);
        if (debugEnabled)
            _log.debug("event source=" + eventSource + " lpart=" + lPartitionId);
        if (reg.checkSourceSubscription(eventSourceName)) {
            if (debugEnabled)
                _log.debug("consumer matches:" + reg.getConsumer());
            ConsumerCallable<ConsumerCallbackResult> dataEventCallable = _callbackFactory.createDataEventCallable(curNanos, e, eventDecoder, reg.getConsumer(), true);
            _currentBatch.add(dataEventCallable);
            if (_consumerStats != null)
                _consumerStats.registerDataEventReceived(e);
            if (_unifiedClientStats != null)
                _unifiedClientStats.registerDataEventReceived(e);
        }
    }
    if (_loggingConsumer != null) {
        ConsumerCallable<ConsumerCallbackResult> dataEventCallable = _callbackFactory.createDataEventCallable(curNanos, e, eventDecoder, _loggingConsumer, false);
        _currentBatch.add(dataEventCallable);
    }
    if (debugEnabled) {
        long endNanos = System.nanoTime();
        _log.debug("Time spent in databus clientlib by onDataEvent = " + (endNanos - curNanos) / DbusConstants.NUM_NSECS_IN_MSEC + "ms");
    }
    return submitBatch(curNanos, false, false);
}

Example 10 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class DatabusV2RegistrationImpl method start.

@Override
public synchronized boolean start() throws IllegalStateException, DatabusClientException {
    _log.info("Starting registration (" + toString() + ") !!");
    if (_state.isRunning()) {
        _log.info("Registration (" + _id + ") already started !!");
        return false;
    }
    if (_state != RegistrationState.REGISTERED)
        throw new IllegalStateException("Registration (" + _id + ") not in startable state !! Current State is :" + _state);
    if ((null == _sources) || (_sources.isEmpty()))
        throw new DatabusClientException("Registration (" + _id + ") does not have any sources to start !!");
    if ((null == _consumers) || (_consumers.isEmpty()))
        throw new DatabusClientException("Registration (" + _id + ") does not have any consumers to start !!");
    List<ServerInfo> relays = _client.getRelays();
    List<ServerInfo> bootstrapServers = _client.getBootstrapServices();
    List<DatabusCombinedConsumer> streamConsumers = new ArrayList<DatabusCombinedConsumer>();
    List<DatabusCombinedConsumer> bootstrapConsumers = new ArrayList<DatabusCombinedConsumer>();
    if ((null == relays) || (relays.isEmpty()))
        throw new DatabusClientException("No configured relays in the client to start");
    Set<ServerInfo> candidateRelays = new HashSet<ServerInfo>();
    for (ServerInfo s : relays) {
        if (canServe(s, _sources))
            candidateRelays.add(s);
    }
    if (candidateRelays.isEmpty())
        throw new DatabusClientException("No candidate relays for source : " + _sources);
    streamConsumers.addAll(_consumers);
    boolean canConsumerBootstrap = false;
    _streamConsumerRawRegistrations = new ArrayList<DatabusV2ConsumerRegistration>();
    _streamConsumerRawRegistrations.add(new DatabusV2ConsumerRegistration(streamConsumers, _sources, _filterConfig));
    for (DatabusCombinedConsumer c : _consumers) {
        if (c.canBootstrap()) {
            canConsumerBootstrap = true;
            bootstrapConsumers.add(c);
        }
    }
    boolean enableBootstrap = _client.getClientStaticConfig().getRuntime().getBootstrap().isEnabled();
    Set<ServerInfo> candidateBootstrapServers = new HashSet<ServerInfo>();
    if (enableBootstrap && canConsumerBootstrap) {
        if ((null == bootstrapServers) || (bootstrapServers.isEmpty()))
            throw new DatabusClientException("No configured bootstrap servers in the client to start");
        for (ServerInfo s : bootstrapServers) {
            if (canServe(s, _sources))
                candidateBootstrapServers.add(s);
        }
        if (candidateBootstrapServers.isEmpty())
            throw new DatabusClientException("No candidate bootstrap servers for source : " + _sources);
        _bootstrapConsumerRawRegistrations = new ArrayList<DatabusV2ConsumerRegistration>();
        ;
        _bootstrapConsumerRawRegistrations.add(new DatabusV2ConsumerRegistration(bootstrapConsumers, _sources, _filterConfig));
    }
    // All validations done. Setup and start
    initializeStatsCollectors();
    DatabusSourcesConnection.StaticConfig connConfig = _client.getClientStaticConfig().getConnection(_sources);
    if (null == connConfig)
        connConfig = _client.getClientStaticConfig().getConnectionDefaults();
    DbusEventBuffer eventBuffer = null;
    {
        DbusEventBuffer.StaticConfig cfg = connConfig.getEventBuffer();
        eventBuffer = new DbusEventBuffer(cfg.getMaxSize(), cfg.getMaxIndividualBufferSize(), cfg.getScnIndexSize(), cfg.getReadBufferSize(), cfg.getMaxEventSize(), cfg.getAllocationPolicy(), new File(cfg.getMmapDirectory().getAbsolutePath() + "_stream_" + _id), cfg.getQueuePolicy(), cfg.getTrace(), null, cfg.getAssertLevel(), cfg.getBufferRemoveWaitPeriod(), cfg.getRestoreMMappedBuffers(), cfg.getRestoreMMappedBuffersValidateEvents(), cfg.isEnableScnIndex(), _client.getEventFactory());
        eventBuffer.setDropOldEvents(true);
        eventBuffer.start(0);
    }
    DbusEventBuffer bootstrapBuffer = null;
    if (enableBootstrap && canConsumerBootstrap) {
        DbusEventBuffer.StaticConfig bstCfg = connConfig.getBstEventBuffer();
        bootstrapBuffer = new DbusEventBuffer(bstCfg.getMaxSize(), bstCfg.getMaxIndividualBufferSize(), bstCfg.getScnIndexSize(), bstCfg.getReadBufferSize(), bstCfg.getMaxEventSize(), bstCfg.getAllocationPolicy(), new File(bstCfg.getMmapDirectory().getAbsolutePath() + "_bootstrap_" + _id), bstCfg.getQueuePolicy(), bstCfg.getTrace(), null, bstCfg.getAssertLevel(), bstCfg.getBufferRemoveWaitPeriod(), bstCfg.getRestoreMMappedBuffers(), bstCfg.getRestoreMMappedBuffersValidateEvents(), bstCfg.isEnableScnIndex(), _client.getEventFactory());
        bootstrapBuffer.setDropOldEvents(false);
        bootstrapBuffer.start(0);
    }
    List<DatabusSubscription> subs = createSubscriptions(_sources);
    if (null != _checkpointPersistenceProvider && _client.getClientStaticConfig().getCheckpointPersistence().isClearBeforeUse()) {
        _log.info("Clearing checkpoint for sources :" + _sources + " with regId :" + _id);
        _checkpointPersistenceProvider.removeCheckpoint(_sources);
    }
    _sourcesConnection = createConnection(connConfig, subs, candidateRelays, candidateBootstrapServers, eventBuffer, bootstrapBuffer);
    _sourcesConnection.start();
    _state = RegistrationState.STARTED;
    _status.start();
    _state = RegistrationState.STARTED;
    return true;
}