Examples with ServerInfo com.linkedin.databus.client.pub.ServerInfo used on opensource projects

Example 6 with ServerInfo

use of com.linkedin.databus.client.pub.ServerInfo in project databus by linkedin.

the class DatabusHttpClientImpl method initializeRelayConnections.

private synchronized void initializeRelayConnections() {
    for (List<DatabusSubscription> subsList : _relayGroups.keySet()) {
        List<String> sourcesStrList = DatabusSubscription.getStrList(subsList);
        List<DatabusV2ConsumerRegistration> relayConsumers = getRelayGroupStreamConsumers().get(subsList);
        //nothing to do
        if (null == relayConsumers || 0 == relayConsumers.size())
            continue;
        try {
            DatabusSourcesConnection.StaticConfig connConfig = getClientStaticConfig().getConnection(sourcesStrList);
            if (null == connConfig) {
                connConfig = getClientStaticConfig().getConnectionDefaults();
            }
            // make sure we have the right policy.
            if (!connConfig.getEventBuffer().isEnableScnIndex() && connConfig.getEventBuffer().getQueuePolicy() != DbusEventBuffer.QueuePolicy.BLOCK_ON_WRITE) {
                throw new InvalidConfigException("If SCN index is disabled, queue policy must be BLOCK_ON_WRITE");
            }
            CheckpointPersistenceProvider cpPersistenceProvder = getCheckpointPersistenceProvider();
            if (null != cpPersistenceProvder && getClientStaticConfig().getCheckpointPersistence().isClearBeforeUse()) {
                cpPersistenceProvder.removeCheckpoint(sourcesStrList);
            }
            ServerInfo server0 = _relayGroups.get(subsList).iterator().next();
            ArrayList<DatabusV2ConsumerRegistration> bstConsumersRegs = new ArrayList<DatabusV2ConsumerRegistration>();
            for (List<DatabusSubscription> bstSubSourcesList : getRelayGroupBootstrapConsumers().keySet()) {
                List<DatabusV2ConsumerRegistration> bstRegsistrations = getRelayGroupBootstrapConsumers().get(bstSubSourcesList);
                for (DatabusV2ConsumerRegistration bstConsumerReg : bstRegsistrations) {
                    if (server0.supportsSources(bstConsumerReg.getSources())) {
                        bstConsumersRegs.add(bstConsumerReg);
                    }
                }
            }
            DbusEventBuffer eventBuffer = connConfig.getEventBuffer().getOrCreateEventBuffer(_eventFactory);
            eventBuffer.setDropOldEvents(true);
            eventBuffer.start(0);
            DbusEventBuffer bootstrapBuffer = null;
            // create bootstrap only if it is enabled
            if (_clientStaticConfig.getRuntime().getBootstrap().isEnabled()) {
                bootstrapBuffer = new DbusEventBuffer(connConfig.getEventBuffer());
                bootstrapBuffer.setDropOldEvents(false);
                bootstrapBuffer.start(0);
            }
            LOG.info("The sourcesList is " + sourcesStrList);
            LOG.info("The relayGroupStreamConsumers is " + getRelayGroupStreamConsumers().get(subsList));
            Set<ServerInfo> relays = _relayGroups.get(subsList);
            Set<ServerInfo> bootstrapServices = _bootstrapGroups.get(subsList);
            String statsCollectorName = generateSubsStatsName(sourcesStrList);
            int ownerId = getContainerStaticConfig().getId();
            _bootstrapEventsStats.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".inbound.bs", true, false, getMbeanServer()));
            _inBoundStatsCollectors.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".inbound", true, false, getMbeanServer()));
            _outBoundStatsCollectors.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".outbound", true, false, getMbeanServer()));
            _consumerStatsCollectors.addStatsCollector(statsCollectorName, new ConsumerCallbackStats(ownerId, statsCollectorName + ".inbound.cons", statsCollectorName + ".inbound.cons", true, false, null, getMbeanServer()));
            _bsConsumerStatsCollectors.addStatsCollector(statsCollectorName, new ConsumerCallbackStats(ownerId, statsCollectorName + ".inbound.bs.cons", statsCollectorName + ".inbound.bs.cons", true, false, null, getMbeanServer()));
            _unifiedClientStatsCollectors.addStatsCollector(statsCollectorName, new UnifiedClientStats(ownerId, statsCollectorName + ".inbound.unified.cons", statsCollectorName + ".inbound.unified.cons", true, false, _clientStaticConfig.getPullerThreadDeadnessThresholdMs(), null, getMbeanServer()));
            ConnectionStateFactory connStateFactory = new ConnectionStateFactory(DatabusSubscription.getStrList(subsList));
            DatabusSourcesConnection newConn = new DatabusSourcesConnection(connConfig, subsList, relays, bootstrapServices, relayConsumers, //_relayGroupBootstrapConsumers.get(sourcesList),
            bstConsumersRegs, eventBuffer, bootstrapBuffer, getDefaultExecutorService(), getContainerStatsCollector(), _inBoundStatsCollectors.getStatsCollector(statsCollectorName), _bootstrapEventsStats.getStatsCollector(statsCollectorName), _consumerStatsCollectors.getStatsCollector(statsCollectorName), _bsConsumerStatsCollectors.getStatsCollector(statsCollectorName), _unifiedClientStatsCollectors.getStatsCollector(statsCollectorName), getCheckpointPersistenceProvider(), getRelayConnFactory(), getBootstrapConnFactory(), getHttpStatsCollector(), null, this, _eventFactory, connStateFactory);
            newConn.start();
            _relayConnections.add(newConn);
        } catch (Exception e) {
            LOG.error("connection initialization issue for source(s):" + subsList + "; please check your configuration", e);
        }
    }
    if (0 == _relayConnections.size()) {
        LOG.warn("No connections specified");
    }
}

Example 7 with ServerInfo

use of com.linkedin.databus.client.pub.ServerInfo in project databus by linkedin.

the class RelayPullThread method doPickRelay.

protected void doPickRelay(ConnectionState curState) {
    int serversNum = _servers.size();
    if (0 == serversNum) {
        enqueueMessage(LifecycleMessage.createSuspendOnErroMessage(new DatabusException("No relays specified")));
        return;
    }
    Random rng = new Random();
    DatabusRelayConnection relayConn = null;
    ServerInfo serverInfo = null;
    int retriesLeft;
    BackoffTimer originalCounter = _status.getRetriesCounter();
    if (curState.isRelayFellOff())
        _status.setRetriesCounter(_retriesOnFallOff);
    while (null == relayConn && (retriesLeft = _status.getRetriesLeft()) >= 0 && !checkForShutdownRequest()) {
        _log.info("picking a relay; retries left:" + retriesLeft + ", Backoff Timer :" + _status.getRetriesCounter() + ", Are we retrying because of SCNNotFoundException : " + curState.isRelayFellOff());
        backoffOnPullError();
        _curServerIdx = (_curServerIdx < 0) ? rng.nextInt(serversNum) : (_curServerIdx + 1) % serversNum;
        Iterator<ServerInfo> setIter = _servers.iterator();
        for (int i = 0; i <= _curServerIdx; ++i) serverInfo = setIter.next();
        try {
            relayConn = _sourcesConn.getRelayConnFactory().createRelayConnection(serverInfo, this, _remoteExceptionHandler);
            _log.info("picked a relay:" + serverInfo.toSimpleString());
        } catch (Exception e) {
            _log.error("Unable to get connection to relay:" + serverInfo.toSimpleString(), e);
        }
    }
    _status.setRetriesCounter(originalCounter);
    if (!checkForShutdownRequest()) {
        _curServer = serverInfo;
        if (null == relayConn) {
            if (_currentState.isRelayFellOff()) {
                boolean enqueueMessage = false;
                try {
                    enqueueMessage = onRelayFellOff(curState, curState.getCheckpoint(), new ScnNotFoundException("Retries on SCNNotFoundException exhausted !!"));
                } catch (InterruptedException ie) {
                    _log.error("interrupted while processing onRelayFellOff", ie);
                    curState.switchToPickServer();
                    enqueueMessage(curState);
                } catch (InvalidEventException e) {
                    _log.error("error trying to notify dispatcher of bootstrapping :" + e.getMessage(), e);
                    curState.switchToPickServer();
                    enqueueMessage(curState);
                }
                if (enqueueMessage)
                    enqueueMessage(curState);
            } else {
                // There are no retries left. Invoke an onError callback
                try {
                    _log.info("Puller retries exhausted. Injecting an error event on dispatcher queue to invoke onError callback");
                    _remoteExceptionHandler.handleException(new PullerRetriesExhaustedException());
                } catch (InterruptedException ie) {
                    _log.error("Interrupted while processing retries exhausted", ie);
                } catch (InvalidEventException e) {
                    _log.error("Error trying to notify dispatcher of puller retries getting exhausted", e);
                }
                _log.error("Cannot find a relay");
            }
        } else {
            DatabusRelayConnection oldRelayConn = curState.getRelayConnection();
            if (null != oldRelayConn) {
                resetConnectionAndSetFlag();
            }
            sendHeartbeat(_sourcesConn.getUnifiedClientStats());
            _log.info("Relay Puller switching to request sources");
            curState.switchToRequestSources(serverInfo, serverInfo.getAddress(), relayConn);
            _lastOpenConnection = relayConn;
            enqueueMessage(curState);
        }
    }
}

Example 8 with ServerInfo

use of com.linkedin.databus.client.pub.ServerInfo in project databus by linkedin.

the class BasePullThread method doAddServer.

private void doAddServer(ServerSetChangeMessage serverSetChangeMsg) {
    ServerInfo newServer = serverSetChangeMsg.getServer();
    _log.info("About to add new Server (" + newServer + ") to Server set. Current Server set is :" + _servers);
    if (null == newServer) {
        _log.error("No new Server to add");
        return;
    }
    if (_servers.contains(newServer)) {
        _log.warn("Server already exists:" + newServer.toString() + " Skipping this addition !!");
    } else {
        _log.info("Adding new Server: " + newServer.toString());
        _servers.add(newServer);
    }
    resetServerRetries();
}

Example 9 with ServerInfo

use of com.linkedin.databus.client.pub.ServerInfo in project databus by linkedin.

the class DatabusHttpClientImpl method registerDatabusListener.

protected List<DatabusV2ConsumerRegistration> registerDatabusListener(DatabusV2ConsumerRegistration listener, Map<List<DatabusSubscription>, Set<ServerInfo>> groupsServers, Map<List<DatabusSubscription>, List<DatabusV2ConsumerRegistration>> groupsListeners, List<DatabusSubscription> sources) throws DatabusClientException {
    List<DatabusSubscription> subsSources = null;
    ServerInfo randomRelay = getRandomRelay(groupsServers, sources);
    if (null == randomRelay) {
        // even if there is no relay available to serve it immediately.
        assert getClientStaticConfig().usesDynamicRelayConfiguration() : "Client relay(s) configured statically but no relays available at listener registration";
        subsSources = sources;
    } else {
        try {
            subsSources = DatabusSubscription.createFromUriList(randomRelay.getSources());
        } catch (DatabusException e) {
            throw new DatabusClientException("source list decode error:" + e.getMessage(), e);
        } catch (URISyntaxException e) {
            throw new DatabusClientException("source list decode error:" + e.getMessage(), e);
        }
    }
    List<DatabusV2ConsumerRegistration> consumers = getListOfConsumerRegsFromSubList(groupsListeners, subsSources);
    if (null == consumers) {
        consumers = new CopyOnWriteArrayList<DatabusV2ConsumerRegistration>();
        groupsListeners.put(subsSources, consumers);
    }
    consumers.add(listener);
    return consumers;
}

Example 10 with ServerInfo

use of com.linkedin.databus.client.pub.ServerInfo in project databus by linkedin.

the class DatabusV2RegistrationImpl method start.

@Override
public synchronized boolean start() throws IllegalStateException, DatabusClientException {
    _log.info("Starting registration (" + toString() + ") !!");
    if (_state.isRunning()) {
        _log.info("Registration (" + _id + ") already started !!");
        return false;
    }
    if (_state != RegistrationState.REGISTERED)
        throw new IllegalStateException("Registration (" + _id + ") not in startable state !! Current State is :" + _state);
    if ((null == _sources) || (_sources.isEmpty()))
        throw new DatabusClientException("Registration (" + _id + ") does not have any sources to start !!");
    if ((null == _consumers) || (_consumers.isEmpty()))
        throw new DatabusClientException("Registration (" + _id + ") does not have any consumers to start !!");
    List<ServerInfo> relays = _client.getRelays();
    List<ServerInfo> bootstrapServers = _client.getBootstrapServices();
    List<DatabusCombinedConsumer> streamConsumers = new ArrayList<DatabusCombinedConsumer>();
    List<DatabusCombinedConsumer> bootstrapConsumers = new ArrayList<DatabusCombinedConsumer>();
    if ((null == relays) || (relays.isEmpty()))
        throw new DatabusClientException("No configured relays in the client to start");
    Set<ServerInfo> candidateRelays = new HashSet<ServerInfo>();
    for (ServerInfo s : relays) {
        if (canServe(s, _sources))
            candidateRelays.add(s);
    }
    if (candidateRelays.isEmpty())
        throw new DatabusClientException("No candidate relays for source : " + _sources);
    streamConsumers.addAll(_consumers);
    boolean canConsumerBootstrap = false;
    _streamConsumerRawRegistrations = new ArrayList<DatabusV2ConsumerRegistration>();
    _streamConsumerRawRegistrations.add(new DatabusV2ConsumerRegistration(streamConsumers, _sources, _filterConfig));
    for (DatabusCombinedConsumer c : _consumers) {
        if (c.canBootstrap()) {
            canConsumerBootstrap = true;
            bootstrapConsumers.add(c);
        }
    }
    boolean enableBootstrap = _client.getClientStaticConfig().getRuntime().getBootstrap().isEnabled();
    Set<ServerInfo> candidateBootstrapServers = new HashSet<ServerInfo>();
    if (enableBootstrap && canConsumerBootstrap) {
        if ((null == bootstrapServers) || (bootstrapServers.isEmpty()))
            throw new DatabusClientException("No configured bootstrap servers in the client to start");
        for (ServerInfo s : bootstrapServers) {
            if (canServe(s, _sources))
                candidateBootstrapServers.add(s);
        }
        if (candidateBootstrapServers.isEmpty())
            throw new DatabusClientException("No candidate bootstrap servers for source : " + _sources);
        _bootstrapConsumerRawRegistrations = new ArrayList<DatabusV2ConsumerRegistration>();
        ;
        _bootstrapConsumerRawRegistrations.add(new DatabusV2ConsumerRegistration(bootstrapConsumers, _sources, _filterConfig));
    }
    // All validations done. Setup and start
    initializeStatsCollectors();
    DatabusSourcesConnection.StaticConfig connConfig = _client.getClientStaticConfig().getConnection(_sources);
    if (null == connConfig)
        connConfig = _client.getClientStaticConfig().getConnectionDefaults();
    DbusEventBuffer eventBuffer = null;
    {
        DbusEventBuffer.StaticConfig cfg = connConfig.getEventBuffer();
        eventBuffer = new DbusEventBuffer(cfg.getMaxSize(), cfg.getMaxIndividualBufferSize(), cfg.getScnIndexSize(), cfg.getReadBufferSize(), cfg.getMaxEventSize(), cfg.getAllocationPolicy(), new File(cfg.getMmapDirectory().getAbsolutePath() + "_stream_" + _id), cfg.getQueuePolicy(), cfg.getTrace(), null, cfg.getAssertLevel(), cfg.getBufferRemoveWaitPeriod(), cfg.getRestoreMMappedBuffers(), cfg.getRestoreMMappedBuffersValidateEvents(), cfg.isEnableScnIndex(), _client.getEventFactory());
        eventBuffer.setDropOldEvents(true);
        eventBuffer.start(0);
    }
    DbusEventBuffer bootstrapBuffer = null;
    if (enableBootstrap && canConsumerBootstrap) {
        DbusEventBuffer.StaticConfig bstCfg = connConfig.getBstEventBuffer();
        bootstrapBuffer = new DbusEventBuffer(bstCfg.getMaxSize(), bstCfg.getMaxIndividualBufferSize(), bstCfg.getScnIndexSize(), bstCfg.getReadBufferSize(), bstCfg.getMaxEventSize(), bstCfg.getAllocationPolicy(), new File(bstCfg.getMmapDirectory().getAbsolutePath() + "_bootstrap_" + _id), bstCfg.getQueuePolicy(), bstCfg.getTrace(), null, bstCfg.getAssertLevel(), bstCfg.getBufferRemoveWaitPeriod(), bstCfg.getRestoreMMappedBuffers(), bstCfg.getRestoreMMappedBuffersValidateEvents(), bstCfg.isEnableScnIndex(), _client.getEventFactory());
        bootstrapBuffer.setDropOldEvents(false);
        bootstrapBuffer.start(0);
    }
    List<DatabusSubscription> subs = createSubscriptions(_sources);
    if (null != _checkpointPersistenceProvider && _client.getClientStaticConfig().getCheckpointPersistence().isClearBeforeUse()) {
        _log.info("Clearing checkpoint for sources :" + _sources + " with regId :" + _id);
        _checkpointPersistenceProvider.removeCheckpoint(_sources);
    }
    _sourcesConnection = createConnection(connConfig, subs, candidateRelays, candidateBootstrapServers, eventBuffer, bootstrapBuffer);
    _sourcesConnection.start();
    _state = RegistrationState.STARTED;
    _status.start();
    _state = RegistrationState.STARTED;
    return true;
}