Examples with ServerInfo com.linkedin.databus.client.pub.ServerInfo used on opensource projects

Example 1 with ServerInfo

use of com.linkedin.databus.client.pub.ServerInfo in project databus by linkedin.

the class RelayEventProducer method createDatabusSourcesConnection.

public static DatabusSourcesConnection createDatabusSourcesConnection(String producerName, int id, String serverName, String subscriptionString, DatabusCombinedConsumer consumer, long internalBufferMaxSize, int largestEventSize, long consumerTimeoutMs, long pollIntervalMs, long connTimeoutMs, int consumerParallelism, boolean blockingBuffer, DatabusClientNettyThreadPools nettyThreadPools, int noEventsTimeoutSec, int maxEventVersion, int initReadBufferSize) throws InvalidConfigException {
    // the assumption here is that the list of subscriptions will become the
    // list of sources hosted by the relay
    Set<ServerInfo> relayServices = createServerInfo(serverName, subscriptionString);
    // null bootstrapService
    Set<ServerInfo> bootstrapServices = null;
    // create subscription objects based on what is required by subscription
    String[] subscriptionList = subscriptionString.split(",");
    List<DatabusSubscription> subsList = DatabusSubscription.createSubscriptionList(Arrays.asList(subscriptionList));
    List<String> sourcesStrList = DatabusSubscription.getStrList(subsList);
    LOG.info("The sourcesList is " + sourcesStrList);
    // create registration objects with consumers
    List<DatabusV2ConsumerRegistration> relayConsumers = createDatabusV2ConsumerRegistration(consumer, sourcesStrList);
    List<DatabusV2ConsumerRegistration> bstConsumers = null;
    // setup sources connection config
    DatabusSourcesConnection.Config confBuilder = new DatabusSourcesConnection.Config();
    confBuilder.setId(id);
    // consume whatever is in relay
    confBuilder.setConsumeCurrent(true);
    //this is set to false as the behaviour is to read the latest SCN when SCN is not found, the buffer isn't cleared
    //as such , so a possibility of gaps in events arises. What we want ideally is to clear existing buffer and then consume from latest SCN
    confBuilder.setReadLatestScnOnError(false);
    // 10s max consumer timeout
    confBuilder.setConsumerTimeBudgetMs(consumerTimeoutMs);
    // poll interval in ms and infinite retry;
    confBuilder.getPullerRetries().setMaxRetryNum(-1);
    confBuilder.getPullerRetries().setInitSleep(pollIntervalMs);
    confBuilder.setConsumerParallelism(consumerParallelism);
    confBuilder.getDispatcherRetries().setMaxRetryNum(1);
    // internal buffer conf
    DbusEventBuffer.Config bufferConf = new DbusEventBuffer.Config();
    bufferConf.setMaxSize(internalBufferMaxSize);
    bufferConf.setAllocationPolicy("DIRECT_MEMORY");
    if (initReadBufferSize > 0) {
        bufferConf.setAverageEventSize(initReadBufferSize);
    }
    bufferConf.setMaxEventSize(largestEventSize);
    //client buffer's scn index- not used
    bufferConf.setScnIndexSize(64 * 1024);
    String queuePolicy = blockingBuffer ? "BLOCK_ON_WRITE" : "OVERWRITE_ON_WRITE";
    bufferConf.setQueuePolicy(queuePolicy);
    //get appropriate checkpointThresholdPct
    double newCkptPct = confBuilder.computeSafeCheckpointThresholdPct(bufferConf);
    if (newCkptPct < 5.0 || newCkptPct > 95.0) {
        LOG.warn("Not setting required checkpointThresholdPct : " + newCkptPct + "to  accommodate largestEventSize= " + largestEventSize + " in buffer of size " + bufferConf.getMaxSize());
        if (newCkptPct <= 0.0) {
            //unlikely to happen: if it does retain default
            newCkptPct = confBuilder.getCheckpointThresholdPct();
        }
        if (newCkptPct < 5.0) {
            newCkptPct = 5.0;
        } else if (newCkptPct > 95.0) {
            newCkptPct = 95.0;
        }
    }
    LOG.info("Setting checkpointThresholdPct:" + newCkptPct);
    confBuilder.setCheckpointThresholdPct(newCkptPct);
    confBuilder.setEventBuffer(bufferConf);
    confBuilder.setNoEventsConnectionResetTimeSec(noEventsTimeoutSec);
    DatabusSourcesConnection.StaticConfig connConfig = confBuilder.build();
    // internal buffers of databus client library
    DbusEventBuffer buffer = new DbusEventBuffer(connConfig.getEventBuffer());
    buffer.start(0);
    DbusEventBuffer bootstrapBuffer = null;
    // Create threadpools and netty managers
    // read - write timeout in ms
    long readTimeoutMs = connTimeoutMs;
    long writeTimeoutMs = connTimeoutMs;
    long bstReadTimeoutMs = connTimeoutMs;
    int protocolVersion = 2;
    // connection factory
    NettyHttpConnectionFactory defaultConnFactory = new NettyHttpConnectionFactory(nettyThreadPools.getBossExecutorService(), nettyThreadPools.getIoExecutorService(), null, nettyThreadPools.getTimer(), writeTimeoutMs, readTimeoutMs, bstReadTimeoutMs, protocolVersion, maxEventVersion, nettyThreadPools.getChannelGroup());
    // Create Thread pool for consumer threads
    int maxThreadsNum = 1;
    int keepAliveMs = 1000;
    ThreadPoolExecutor defaultExecutorService = new OrderedMemoryAwareThreadPoolExecutor(maxThreadsNum, 0, 0, keepAliveMs, TimeUnit.MILLISECONDS);
    ConsumerCallbackStats relayConsumerStats = new ConsumerCallbackStats(id, producerName + ".inbound.cons", producerName + ".inbound.cons", true, false, null, ManagementFactory.getPlatformMBeanServer());
    ConsumerCallbackStats bootstrapConsumerStats = new ConsumerCallbackStats(id, producerName + ".inbound.bs.cons", producerName + ".inbound.bs.cons", true, false, null, ManagementFactory.getPlatformMBeanServer());
    UnifiedClientStats unifiedClientStats = new UnifiedClientStats(id, producerName + ".inbound.unified.cons", producerName + ".inbound.unified.cons", true, false, UnifiedClientStats.DEFAULT_DEADNESS_THRESHOLD_MS, null, ManagementFactory.getPlatformMBeanServer());
    DatabusRelayConnectionFactory relayConnFactory = defaultConnFactory;
    DatabusBootstrapConnectionFactory bootstrapConnFactory = defaultConnFactory;
    ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sourcesStrList);
    DatabusSourcesConnection conn = new DatabusSourcesConnection(connConfig, subsList, relayServices, bootstrapServices, relayConsumers, bstConsumers, buffer, bootstrapBuffer, defaultExecutorService, // getContainerStatsCollector(),
    null, // getInboundEventStatisticsCollector(),
    null, // getBootstrapEventsStatsCollector(),
    null, // relay callback stats
    relayConsumerStats, // bootstrap callback stats
    bootstrapConsumerStats, // combined relay/bootstrap callback stats
    unifiedClientStats, // getCheckpointPersistenceProvider(),
    null, relayConnFactory, bootstrapConnFactory, // getHttpStatsCollector(),
    null, // RegistrationId
    null, null, new DbusEventV2Factory(), // TODO Get the ref to factory from HttpRelay.
    connStateFactory);
    return conn;
}

Example 2 with ServerInfo

use of com.linkedin.databus.client.pub.ServerInfo in project databus by linkedin.

the class BootstrapRequestProcessorBase method initBootstrapServerInfo.

private void initBootstrapServerInfo() {
    if (!_isServerInfoInitialized) {
        String host = null;
        try {
            host = InetAddress.getLocalHost().getHostName();
        } catch (UnknownHostException e) {
            LOG.error("Unable to fetch the local hostname !! Trying to fetch IP Address !!", e);
            try {
                host = InetAddress.getLocalHost().getHostAddress();
            } catch (UnknownHostException e1) {
                LOG.error("Unable to fetch the local IP Address too !! Giving up", e1);
                host = null;
            }
        }
        if (null != host) {
            int port = _bootstrapServer.getHttpPort();
            _serverHostPort = host + DbusConstants.HOSTPORT_DELIMITER + port;
            ServerInfo serverInfo = null;
            try {
                serverInfo = ServerInfo.buildServerInfoFromHostPort(_serverHostPort, DbusConstants.HOSTPORT_DELIMITER);
            } catch (Exception e) {
                LOG.error("Unable to build serverInfo from string (" + _serverHostPort + ")", e);
            }
            _serverInfo = serverInfo;
        } else {
            _serverHostPort = null;
            _serverInfo = null;
            LOG.error("Unable to fetch local address !! Clients connecting to this bootstrap server will restart bootstrap on failures !!");
        }
        _isServerInfoInitialized = true;
    }
}

Example 3 with ServerInfo

use of com.linkedin.databus.client.pub.ServerInfo in project databus by linkedin.

the class BootstrapRequestProcessorBase method process.

@Override
public final DatabusRequest process(DatabusRequest request) throws IOException, RequestProcessingException {
    initBootstrapServerInfo();
    String ckptStr = request.getParams().getProperty(CHECKPOINT_PARAM);
    Checkpoint ckpt = null;
    if (null != ckptStr) {
        ckpt = new Checkpoint(ckptStr);
        String bsServerInfo = ckpt.getBootstrapServerInfo();
        if ((null != bsServerInfo) && (null != _serverInfo)) {
            ServerInfo expServerInfo = null;
            try {
                expServerInfo = ServerInfo.buildServerInfoFromHostPort(bsServerInfo.trim(), DbusConstants.HOSTPORT_DELIMITER);
            } catch (Exception ex) {
                LOG.error("Unable to fetch ServerInfo from ckpt. Passed ServerInfo was :" + bsServerInfo.trim());
            }
            if ((null != expServerInfo) && (!_serverInfo.equals(expServerInfo))) {
                String msg = "Bootstrap Server Request should be served by different host : " + bsServerInfo + ", This instance is :" + _serverHostPort;
                LOG.error(msg);
                throw new RequestProcessingException(msg);
            }
        }
    }
    DatabusRequest req = doProcess(request);
    if (null != _serverHostPort) {
        req.getResponseContent().setMetadata(DbusConstants.SERVER_INFO_HOSTPORT_HEADER_PARAM, _serverHostPort);
    }
    return req;
}

Example 4 with ServerInfo

use of com.linkedin.databus.client.pub.ServerInfo in project databus by linkedin.

the class DatabusBootstrapProducer method decouplePhysicalSources.

/**
   * The method helps identify the Physical sources that are listed in the
   * bootstrap producer config (Which is really the client config) The Physical
   * sources are identified by comparing the list of subscriptions/logical
   * sources. There are two acceptable cases: 1. Two relays provide different
   * logical sources -> Two different physical sources 2. Two relays provide the
   * same logical sources (for load balancing/fault tolerance) -> Only one
   * physical source (The multitenant-client library underneath is aware that
   * two relays provide the same logical sources)
   *
   * @return A set of Physical sources which each contains a list of logical
   *         sources.
   *
   */
private void decouplePhysicalSources() {
    DatabusHttpClientImpl.RuntimeConfig clientRtConfig = getClientConfigManager().getReadOnlyConfig();
    for (ServerInfo relayInfo : clientRtConfig.getRelays()) {
        if (relayInfo == null || relayInfo.getSources() == null)
            LOG.error("No sources specified in the client config for the bootstrap producer");
        if (relayInfo.getPhysicalSourceName() == null) {
            LOG.error("PhysicalSource name not specified");
        }
        SourceInfo sourceInfo = new SourceInfo(relayInfo.getPhysicalSourceName(), relayInfo.getSources());
        _registeredPhysicalSources.add(sourceInfo);
    }
}

Example 5 with ServerInfo

use of com.linkedin.databus.client.pub.ServerInfo in project databus by linkedin.

the class DatabusBootstrapProducer method initBootstrapDBMetadata.

private void initBootstrapDBMetadata() throws SQLException, BootstrapDatabaseTooOldException {
    DatabusHttpClientImpl.RuntimeConfig clientRtConfig = getClientConfigManager().getReadOnlyConfig();
    // create source list
    for (ServerInfo relayInfo : clientRtConfig.getRelays()) {
        _registeredSources.addAll(relayInfo.getSources());
        for (String source : _registeredSources) {
            BootstrapDBMetaDataDAO.SourceStatusInfo srcIdStatus = _dbDao.getSrcIdStatusFromDB(source, false);
            if (0 > srcIdStatus.getSrcId()) {
                int newState = BootstrapProducerStatus.NEW;
                if (!_bootstrapProducerStaticConfig.isBootstrapDBStateCheck()) {
                    // TO allow test framework to listen to relay directly,DBStateCheck
                    // flag is used
                    newState = BootstrapProducerStatus.ACTIVE;
                }
                _dbDao.addNewSourceInDB(source, newState);
            }
            srcIdStatus = _dbDao.getSrcIdStatusFromDB(source, false);
            _srcNameIdMap.put(source, srcIdStatus.getSrcId());
            if (_bootstrapProducerStaticConfig.isBootstrapDBStateCheck()) {
                if (!BootstrapProducerStatus.isReadyForConsumption(srcIdStatus.getStatus()))
                    throw new BootstrapDatabaseTooOldException("Bootstrap DB is not ready to read from relay !! Status :" + srcIdStatus);
            }
        }
    }
}