Examples with DatabusCombinedConsumer com.linkedin.databus.client.pub.DatabusCombinedConsumer used on opensource projects

Example 6 with DatabusCombinedConsumer

use of com.linkedin.databus.client.pub.DatabusCombinedConsumer in project databus by linkedin.

the class MultiConsumerCallback method onStartConsumption.

@Override
public ConsumerCallbackResult onStartConsumption() {
    long curNanos = System.nanoTime();
    for (DatabusV2ConsumerRegistration consumerReg : _registrations) {
        for (DatabusCombinedConsumer consumer : consumerReg.getConsumers()) {
            ConsumerCallable<ConsumerCallbackResult> startConsumptionCallable = _callbackFactory.createStartConsumptionCallable(curNanos, consumer, true);
            _currentBatch.add(startConsumptionCallable);
            if (_consumerStats != null)
                _consumerStats.registerEventsReceived(1);
        }
    }
    if (_loggingConsumer != null) {
        ConsumerCallable<ConsumerCallbackResult> startConsumptionCallable = _callbackFactory.createStartConsumptionCallable(curNanos, _loggingConsumer, false);
        _currentBatch.add(startConsumptionCallable);
    }
    if (_log.isDebugEnabled()) {
        long endNanos = System.nanoTime();
        _log.debug("Time spent in databus clientlib by onStartConsumption = " + (endNanos - curNanos) / DbusConstants.NUM_NSECS_IN_MSEC + "ms");
    }
    return submitBatch(curNanos, false, true);
}

Example 7 with DatabusCombinedConsumer

use of com.linkedin.databus.client.pub.DatabusCombinedConsumer in project databus by linkedin.

the class MultiConsumerCallback method onEndSource.

@Override
public ConsumerCallbackResult onEndSource(String source, Schema sourceSchema) {
    long curNanos = System.nanoTime();
    for (DatabusV2ConsumerRegistration consumerReg : _registrations) {
        for (DatabusCombinedConsumer consumer : consumerReg.getConsumers()) {
            ConsumerCallable<ConsumerCallbackResult> endSourceCallable = _callbackFactory.createEndSourceCallable(curNanos, source, sourceSchema, consumer, true);
            _currentBatch.add(endSourceCallable);
            if (_consumerStats != null)
                _consumerStats.registerEventsReceived(1);
        }
    }
    if (_loggingConsumer != null) {
        ConsumerCallable<ConsumerCallbackResult> endSourceCallable = _callbackFactory.createEndSourceCallable(curNanos, source, sourceSchema, _loggingConsumer, false);
        _currentBatch.add(endSourceCallable);
    }
    if (_log.isDebugEnabled()) {
        long endNanos = System.nanoTime();
        _log.debug("Time spent in databus clientlib by onEndSource = " + (endNanos - curNanos) / DbusConstants.NUM_NSECS_IN_MSEC + "ms");
    }
    return submitBatch(curNanos, true, true);
}

Example 8 with DatabusCombinedConsumer

use of com.linkedin.databus.client.pub.DatabusCombinedConsumer in project databus by linkedin.

the class RegistrationStatsInfo method initSourcesConn.

private void initSourcesConn(DatabusSourcesConnection sourcesConn) {
    if (null != sourcesConn) {
        RelayPullThread rp = sourcesConn.getRelayPullThread();
        BootstrapPullThread bp = sourcesConn.getBootstrapPullThread();
        GenericDispatcher<DatabusCombinedConsumer> rd = sourcesConn.getRelayDispatcher();
        GenericDispatcher<DatabusCombinedConsumer> bd = sourcesConn.getBootstrapDispatcher();
        if (null != rp) {
            setRelayPullerConnectionState(rp.getConnectionState().getStateId());
            if (null != rp.getComponentStatus())
                setRelayPullerComponentStatus(rp.getComponentStatus().getStatus());
            setCurrentRelay(rp.getCurentServer());
            setCandidateRelays(rp.getServers());
        }
        if (null != bp) {
            setBootstrapPullerConnectionState(bp.getConnectionState().getStateId());
            if (null != bp.getComponentStatus())
                setBootstrapPullerComponentStatus(bp.getComponentStatus().getStatus());
            setCurrentBootstrapServer(bp.getCurentServer());
            setCandidateBootstrapServers(bp.getServers());
        }
        if (null != rd) {
            if (null != rd.getComponentStatus())
                setRelayDispatcherComponentStatus(rd.getComponentStatus().getStatus());
            if (null != rd.getDispatcherState())
                setRelayDispatcherConnectionState(rd.getDispatcherState().getStateId());
        }
        if (null != bd) {
            if (null != bd.getComponentStatus())
                setBootstrapDispatcherComponentStatus(bd.getComponentStatus().getStatus());
            if (null != bd.getDispatcherState())
                setBootstrapDispatcherConnectionState(bd.getDispatcherState().getStateId());
        }
    }
}

Example 9 with DatabusCombinedConsumer

use of com.linkedin.databus.client.pub.DatabusCombinedConsumer in project databus by linkedin.

the class DatabusV2RegistrationImpl method start.

@Override
public synchronized boolean start() throws IllegalStateException, DatabusClientException {
    _log.info("Starting registration (" + toString() + ") !!");
    if (_state.isRunning()) {
        _log.info("Registration (" + _id + ") already started !!");
        return false;
    }
    if (_state != RegistrationState.REGISTERED)
        throw new IllegalStateException("Registration (" + _id + ") not in startable state !! Current State is :" + _state);
    if ((null == _sources) || (_sources.isEmpty()))
        throw new DatabusClientException("Registration (" + _id + ") does not have any sources to start !!");
    if ((null == _consumers) || (_consumers.isEmpty()))
        throw new DatabusClientException("Registration (" + _id + ") does not have any consumers to start !!");
    List<ServerInfo> relays = _client.getRelays();
    List<ServerInfo> bootstrapServers = _client.getBootstrapServices();
    List<DatabusCombinedConsumer> streamConsumers = new ArrayList<DatabusCombinedConsumer>();
    List<DatabusCombinedConsumer> bootstrapConsumers = new ArrayList<DatabusCombinedConsumer>();
    if ((null == relays) || (relays.isEmpty()))
        throw new DatabusClientException("No configured relays in the client to start");
    Set<ServerInfo> candidateRelays = new HashSet<ServerInfo>();
    for (ServerInfo s : relays) {
        if (canServe(s, _sources))
            candidateRelays.add(s);
    }
    if (candidateRelays.isEmpty())
        throw new DatabusClientException("No candidate relays for source : " + _sources);
    streamConsumers.addAll(_consumers);
    boolean canConsumerBootstrap = false;
    _streamConsumerRawRegistrations = new ArrayList<DatabusV2ConsumerRegistration>();
    _streamConsumerRawRegistrations.add(new DatabusV2ConsumerRegistration(streamConsumers, _sources, _filterConfig));
    for (DatabusCombinedConsumer c : _consumers) {
        if (c.canBootstrap()) {
            canConsumerBootstrap = true;
            bootstrapConsumers.add(c);
        }
    }
    boolean enableBootstrap = _client.getClientStaticConfig().getRuntime().getBootstrap().isEnabled();
    Set<ServerInfo> candidateBootstrapServers = new HashSet<ServerInfo>();
    if (enableBootstrap && canConsumerBootstrap) {
        if ((null == bootstrapServers) || (bootstrapServers.isEmpty()))
            throw new DatabusClientException("No configured bootstrap servers in the client to start");
        for (ServerInfo s : bootstrapServers) {
            if (canServe(s, _sources))
                candidateBootstrapServers.add(s);
        }
        if (candidateBootstrapServers.isEmpty())
            throw new DatabusClientException("No candidate bootstrap servers for source : " + _sources);
        _bootstrapConsumerRawRegistrations = new ArrayList<DatabusV2ConsumerRegistration>();
        ;
        _bootstrapConsumerRawRegistrations.add(new DatabusV2ConsumerRegistration(bootstrapConsumers, _sources, _filterConfig));
    }
    // All validations done. Setup and start
    initializeStatsCollectors();
    DatabusSourcesConnection.StaticConfig connConfig = _client.getClientStaticConfig().getConnection(_sources);
    if (null == connConfig)
        connConfig = _client.getClientStaticConfig().getConnectionDefaults();
    DbusEventBuffer eventBuffer = null;
    {
        DbusEventBuffer.StaticConfig cfg = connConfig.getEventBuffer();
        eventBuffer = new DbusEventBuffer(cfg.getMaxSize(), cfg.getMaxIndividualBufferSize(), cfg.getScnIndexSize(), cfg.getReadBufferSize(), cfg.getMaxEventSize(), cfg.getAllocationPolicy(), new File(cfg.getMmapDirectory().getAbsolutePath() + "_stream_" + _id), cfg.getQueuePolicy(), cfg.getTrace(), null, cfg.getAssertLevel(), cfg.getBufferRemoveWaitPeriod(), cfg.getRestoreMMappedBuffers(), cfg.getRestoreMMappedBuffersValidateEvents(), cfg.isEnableScnIndex(), _client.getEventFactory());
        eventBuffer.setDropOldEvents(true);
        eventBuffer.start(0);
    }
    DbusEventBuffer bootstrapBuffer = null;
    if (enableBootstrap && canConsumerBootstrap) {
        DbusEventBuffer.StaticConfig bstCfg = connConfig.getBstEventBuffer();
        bootstrapBuffer = new DbusEventBuffer(bstCfg.getMaxSize(), bstCfg.getMaxIndividualBufferSize(), bstCfg.getScnIndexSize(), bstCfg.getReadBufferSize(), bstCfg.getMaxEventSize(), bstCfg.getAllocationPolicy(), new File(bstCfg.getMmapDirectory().getAbsolutePath() + "_bootstrap_" + _id), bstCfg.getQueuePolicy(), bstCfg.getTrace(), null, bstCfg.getAssertLevel(), bstCfg.getBufferRemoveWaitPeriod(), bstCfg.getRestoreMMappedBuffers(), bstCfg.getRestoreMMappedBuffersValidateEvents(), bstCfg.isEnableScnIndex(), _client.getEventFactory());
        bootstrapBuffer.setDropOldEvents(false);
        bootstrapBuffer.start(0);
    }
    List<DatabusSubscription> subs = createSubscriptions(_sources);
    if (null != _checkpointPersistenceProvider && _client.getClientStaticConfig().getCheckpointPersistence().isClearBeforeUse()) {
        _log.info("Clearing checkpoint for sources :" + _sources + " with regId :" + _id);
        _checkpointPersistenceProvider.removeCheckpoint(_sources);
    }
    _sourcesConnection = createConnection(connConfig, subs, candidateRelays, candidateBootstrapServers, eventBuffer, bootstrapBuffer);
    _sourcesConnection.start();
    _state = RegistrationState.STARTED;
    _status.start();
    _state = RegistrationState.STARTED;
    return true;
}

Example 10 with DatabusCombinedConsumer

use of com.linkedin.databus.client.pub.DatabusCombinedConsumer in project databus by linkedin.

the class DatabusV2ClusterRegistrationImpl method activateOnePartition.

/**
   * Callback to activate one partition that was added
   * @param partition
   * @throws DatabusException
   */
private synchronized void activateOnePartition(DbusPartitionInfo partition) throws DatabusClientException {
    _log.info("Trying to activate partition :" + partition);
    try {
        if (regMap.containsKey(partition)) {
            _log.info("Partition (" + partition + ") is already added and is currently in state : " + regMap.get(partition).getState() + " skipping !!");
            return;
        }
        // Call factories to get consumer callbacks and server-side filter config.
        Collection<DatabusCombinedConsumer> consumers = _consumerFactory.createPartitionedConsumers(_clusterInfo, partition);
        DbusKeyCompositeFilterConfig filterConfig = null;
        if (_serverSideFilterFactory != null)
            filterConfig = _serverSideFilterFactory.createServerSideFilter(_clusterInfo, partition);
        if ((null == consumers) || (consumers.isEmpty())) {
            _log.error("ConsumerFactory for cluster (" + _clusterInfo + ") returned null or empty consumers ");
            throw new DatabusClientException("ConsumerFactory for cluster (" + _clusterInfo + ") returned null or empty consumers");
        }
        // Create Registration
        RegistrationId id = new RegistrationId(_id + "-" + partition.getPartitionId());
        CheckpointPersistenceProvider ckptProvider = createCheckpointPersistenceProvider(partition);
        DatabusV2RegistrationImpl reg = createChildRegistration(id, _client, ckptProvider);
        reg.addDatabusConsumers(consumers);
        String[] srcs = new String[_sources.size()];
        srcs = _sources.toArray(srcs);
        reg.addSubscriptions(srcs);
        regMap.put(partition, reg);
        reg.onRegister();
        // Add Server-Side Filter
        if (null != filterConfig)
            reg.withServerSideFilter(filterConfig);
        // Notify Listener
        if (null != _partitionListener)
            _partitionListener.onAddPartition(partition, reg);
        // Start the registration
        try {
            reg.start();
        } catch (DatabusClientException e) {
            _log.error("Got exception while starting the registration for partition (" + partition + ")", e);
            throw e;
        }
        //Add partition Specific metrics to cluster-merge
        _relayEventStatsMerger.addStatsCollector(id.getId(), (DbusEventsStatisticsCollector) reg.getRelayEventStats());
        _bootstrapEventStatsMerger.addStatsCollector(id.getId(), (DbusEventsStatisticsCollector) reg.getBootstrapEventStats());
        _relayCallbackStatsMerger.addStatsCollector(id.getId(), (ConsumerCallbackStats) reg.getRelayCallbackStats());
        _bootstrapCallbackStatsMerger.addStatsCollector(id.getId(), (ConsumerCallbackStats) reg.getBootstrapCallbackStats());
        _log.info("Partition (" + partition + ") started !!");
    } catch (DatabusException e) {
        _log.error("Got exception while activating partition(" + partition + ")", e);
        throw new DatabusClientException(e);
    } catch (ClusterCheckpointException e) {
        _log.error("Got exception while activating partition(" + partition + ")", e);
        throw new DatabusClientException(e);
    }
}