Search in sources :

Example 1 with UnifiedClientStats

use of com.linkedin.databus.client.pub.mbean.UnifiedClientStats in project databus by linkedin.

the class RelayEventProducer method createDatabusSourcesConnection.

public static DatabusSourcesConnection createDatabusSourcesConnection(String producerName, int id, String serverName, String subscriptionString, DatabusCombinedConsumer consumer, long internalBufferMaxSize, int largestEventSize, long consumerTimeoutMs, long pollIntervalMs, long connTimeoutMs, int consumerParallelism, boolean blockingBuffer, DatabusClientNettyThreadPools nettyThreadPools, int noEventsTimeoutSec, int maxEventVersion, int initReadBufferSize) throws InvalidConfigException {
    // the assumption here is that the list of subscriptions will become the
    // list of sources hosted by the relay
    Set<ServerInfo> relayServices = createServerInfo(serverName, subscriptionString);
    // null bootstrapService
    Set<ServerInfo> bootstrapServices = null;
    // create subscription objects based on what is required by subscription
    String[] subscriptionList = subscriptionString.split(",");
    List<DatabusSubscription> subsList = DatabusSubscription.createSubscriptionList(Arrays.asList(subscriptionList));
    List<String> sourcesStrList = DatabusSubscription.getStrList(subsList);
    LOG.info("The sourcesList is " + sourcesStrList);
    // create registration objects with consumers
    List<DatabusV2ConsumerRegistration> relayConsumers = createDatabusV2ConsumerRegistration(consumer, sourcesStrList);
    List<DatabusV2ConsumerRegistration> bstConsumers = null;
    // setup sources connection config
    DatabusSourcesConnection.Config confBuilder = new DatabusSourcesConnection.Config();
    confBuilder.setId(id);
    // consume whatever is in relay
    confBuilder.setConsumeCurrent(true);
    //this is set to false as the behaviour is to read the latest SCN when SCN is not found, the buffer isn't cleared
    //as such , so a possibility of gaps in events arises. What we want ideally is to clear existing buffer and then consume from latest SCN
    confBuilder.setReadLatestScnOnError(false);
    // 10s max consumer timeout
    confBuilder.setConsumerTimeBudgetMs(consumerTimeoutMs);
    // poll interval in ms and infinite retry;
    confBuilder.getPullerRetries().setMaxRetryNum(-1);
    confBuilder.getPullerRetries().setInitSleep(pollIntervalMs);
    confBuilder.setConsumerParallelism(consumerParallelism);
    confBuilder.getDispatcherRetries().setMaxRetryNum(1);
    // internal buffer conf
    DbusEventBuffer.Config bufferConf = new DbusEventBuffer.Config();
    bufferConf.setMaxSize(internalBufferMaxSize);
    bufferConf.setAllocationPolicy("DIRECT_MEMORY");
    if (initReadBufferSize > 0) {
        bufferConf.setAverageEventSize(initReadBufferSize);
    }
    bufferConf.setMaxEventSize(largestEventSize);
    //client buffer's scn index- not used
    bufferConf.setScnIndexSize(64 * 1024);
    String queuePolicy = blockingBuffer ? "BLOCK_ON_WRITE" : "OVERWRITE_ON_WRITE";
    bufferConf.setQueuePolicy(queuePolicy);
    //get appropriate checkpointThresholdPct
    double newCkptPct = confBuilder.computeSafeCheckpointThresholdPct(bufferConf);
    if (newCkptPct < 5.0 || newCkptPct > 95.0) {
        LOG.warn("Not setting required checkpointThresholdPct : " + newCkptPct + "to  accommodate largestEventSize= " + largestEventSize + " in buffer of size " + bufferConf.getMaxSize());
        if (newCkptPct <= 0.0) {
            //unlikely to happen: if it does retain default
            newCkptPct = confBuilder.getCheckpointThresholdPct();
        }
        if (newCkptPct < 5.0) {
            newCkptPct = 5.0;
        } else if (newCkptPct > 95.0) {
            newCkptPct = 95.0;
        }
    }
    LOG.info("Setting checkpointThresholdPct:" + newCkptPct);
    confBuilder.setCheckpointThresholdPct(newCkptPct);
    confBuilder.setEventBuffer(bufferConf);
    confBuilder.setNoEventsConnectionResetTimeSec(noEventsTimeoutSec);
    DatabusSourcesConnection.StaticConfig connConfig = confBuilder.build();
    // internal buffers of databus client library
    DbusEventBuffer buffer = new DbusEventBuffer(connConfig.getEventBuffer());
    buffer.start(0);
    DbusEventBuffer bootstrapBuffer = null;
    // Create threadpools and netty managers
    // read - write timeout in ms
    long readTimeoutMs = connTimeoutMs;
    long writeTimeoutMs = connTimeoutMs;
    long bstReadTimeoutMs = connTimeoutMs;
    int protocolVersion = 2;
    // connection factory
    NettyHttpConnectionFactory defaultConnFactory = new NettyHttpConnectionFactory(nettyThreadPools.getBossExecutorService(), nettyThreadPools.getIoExecutorService(), null, nettyThreadPools.getTimer(), writeTimeoutMs, readTimeoutMs, bstReadTimeoutMs, protocolVersion, maxEventVersion, nettyThreadPools.getChannelGroup());
    // Create Thread pool for consumer threads
    int maxThreadsNum = 1;
    int keepAliveMs = 1000;
    ThreadPoolExecutor defaultExecutorService = new OrderedMemoryAwareThreadPoolExecutor(maxThreadsNum, 0, 0, keepAliveMs, TimeUnit.MILLISECONDS);
    ConsumerCallbackStats relayConsumerStats = new ConsumerCallbackStats(id, producerName + ".inbound.cons", producerName + ".inbound.cons", true, false, null, ManagementFactory.getPlatformMBeanServer());
    ConsumerCallbackStats bootstrapConsumerStats = new ConsumerCallbackStats(id, producerName + ".inbound.bs.cons", producerName + ".inbound.bs.cons", true, false, null, ManagementFactory.getPlatformMBeanServer());
    UnifiedClientStats unifiedClientStats = new UnifiedClientStats(id, producerName + ".inbound.unified.cons", producerName + ".inbound.unified.cons", true, false, UnifiedClientStats.DEFAULT_DEADNESS_THRESHOLD_MS, null, ManagementFactory.getPlatformMBeanServer());
    DatabusRelayConnectionFactory relayConnFactory = defaultConnFactory;
    DatabusBootstrapConnectionFactory bootstrapConnFactory = defaultConnFactory;
    ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sourcesStrList);
    DatabusSourcesConnection conn = new DatabusSourcesConnection(connConfig, subsList, relayServices, bootstrapServices, relayConsumers, bstConsumers, buffer, bootstrapBuffer, defaultExecutorService, // getContainerStatsCollector(),
    null, // getInboundEventStatisticsCollector(),
    null, // getBootstrapEventsStatsCollector(),
    null, // relay callback stats
    relayConsumerStats, // bootstrap callback stats
    bootstrapConsumerStats, // combined relay/bootstrap callback stats
    unifiedClientStats, // getCheckpointPersistenceProvider(),
    null, relayConnFactory, bootstrapConnFactory, // getHttpStatsCollector(),
    null, // RegistrationId
    null, null, new DbusEventV2Factory(), // TODO Get the ref to factory from HttpRelay.
    connStateFactory);
    return conn;
}
Also used : DatabusV2ConsumerRegistration(com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration) ServerInfo(com.linkedin.databus.client.pub.ServerInfo) PhysicalSourceStaticConfig(com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig) LogicalSourceStaticConfig(com.linkedin.databus2.relay.config.LogicalSourceStaticConfig) NettyHttpConnectionFactory(com.linkedin.databus.client.netty.NettyHttpConnectionFactory) OrderedMemoryAwareThreadPoolExecutor(org.jboss.netty.handler.execution.OrderedMemoryAwareThreadPoolExecutor) UnifiedClientStats(com.linkedin.databus.client.pub.mbean.UnifiedClientStats) DatabusRelayConnectionFactory(com.linkedin.databus.client.DatabusRelayConnectionFactory) ConsumerCallbackStats(com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats) DatabusSubscription(com.linkedin.databus.core.data_model.DatabusSubscription) DatabusBootstrapConnectionFactory(com.linkedin.databus.client.DatabusBootstrapConnectionFactory) Checkpoint(com.linkedin.databus.core.Checkpoint) DatabusSourcesConnection(com.linkedin.databus.client.DatabusSourcesConnection) DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) OrderedMemoryAwareThreadPoolExecutor(org.jboss.netty.handler.execution.OrderedMemoryAwareThreadPoolExecutor) DbusEventV2Factory(com.linkedin.databus.core.DbusEventV2Factory) ConnectionStateFactory(com.linkedin.databus.client.ConnectionStateFactory)

Example 2 with UnifiedClientStats

use of com.linkedin.databus.client.pub.mbean.UnifiedClientStats in project databus by linkedin.

the class TestUnifiedClientStats method testHistogramMetricsAggregationDeadSourcesConnection.

/**
   * Tests aggregation (merging) of the timeLagSourceToReceiptMs histogram/percentile metric in the case
   * that one of the connections is dead (i.e., no data events received).
   *
   * Blast out 1000 data values for stats #1 but none for stats #2 (in particular, no registerDataEventReceived()
   * calls):  timestampLastDataEventWasReceivedMs will be zero for stats #2 (and its reservoir empty), so
   * merging it won't affect the aggregate value for timeLagSourceToReceiptMs; all such aggregate stats should
   * be identical to those for stats #1.  Also, all values for stats #2 should be -1.0, per our design spec.
   * (This is similar to testHistogramMetricsAggregationBootstrapMode().)
   */
@Test
public void testHistogramMetricsAggregationDeadSourcesConnection() {
    // create stats objects:  two low-level (per-connection) ones and one aggregator
    UnifiedClientStats unifiedClientStats1 = new UnifiedClientStats(1, /* ownerId */
    "test1", "dim1");
    UnifiedClientStats unifiedClientStats2 = new UnifiedClientStats(2, /* ownerId */
    "test2", "dim2");
    UnifiedClientStats unifiedClientStatsAgg = new UnifiedClientStats(99, /* ownerId */
    "testAgg", "dimAgg");
    // could break this into two (or more) parts and do multiple merges, but not clear there's any point...
    for (// 2*1028
    int i = 0; // 2*1028
    i < 2 * MergeableExponentiallyDecayingReservoir.DEFAULT_SIZE; // 2*1028
    ++i) {
        long sourceTimestampNs1 = (System.currentTimeMillis() - 1000L - i) * DbusConstants.NUM_NSECS_IN_MSEC;
        // no data events for connection #2 => no need for sourceTimestampNs2
        unifiedClientStats1.registerDataEventReceived(createEvent(sourceTimestampNs1));
    }
    unifiedClientStatsAgg.merge(unifiedClientStats1);
    unifiedClientStatsAgg.merge(unifiedClientStats2);
    assertEquals("unexpected timeLagSourceToReceiptMs 50th percentile for aggregated stats", unifiedClientStats1.getTimeLagSourceToReceiptMs_HistPct_50(), unifiedClientStatsAgg.getTimeLagSourceToReceiptMs_HistPct_50());
    assertEquals("unexpected timeLagSourceToReceiptMs 90th percentile for aggregated stats", unifiedClientStats1.getTimeLagSourceToReceiptMs_HistPct_90(), unifiedClientStatsAgg.getTimeLagSourceToReceiptMs_HistPct_90());
    assertEquals("unexpected timeLagSourceToReceiptMs 95th percentile for aggregated stats", unifiedClientStats1.getTimeLagSourceToReceiptMs_HistPct_95(), unifiedClientStatsAgg.getTimeLagSourceToReceiptMs_HistPct_95());
    assertEquals("unexpected timeLagSourceToReceiptMs 99th percentile for aggregated stats", unifiedClientStats1.getTimeLagSourceToReceiptMs_HistPct_99(), unifiedClientStatsAgg.getTimeLagSourceToReceiptMs_HistPct_99());
    // no data values => should return -1.0 for all percentiles
    assertEquals("unexpected timeLagSourceToReceiptMs 50th percentile for connection #2", -1.0, unifiedClientStats2.getTimeLagSourceToReceiptMs_HistPct_50());
    assertEquals("unexpected timeLagSourceToReceiptMs 90th percentile for connection #2", -1.0, unifiedClientStats2.getTimeLagSourceToReceiptMs_HistPct_90());
    assertEquals("unexpected timeLagSourceToReceiptMs 95th percentile for connection #2", -1.0, unifiedClientStats2.getTimeLagSourceToReceiptMs_HistPct_95());
    assertEquals("unexpected timeLagSourceToReceiptMs 99th percentile for connection #2", -1.0, unifiedClientStats2.getTimeLagSourceToReceiptMs_HistPct_99());
}
Also used : UnifiedClientStats(com.linkedin.databus.client.pub.mbean.UnifiedClientStats) Test(org.testng.annotations.Test)

Example 3 with UnifiedClientStats

use of com.linkedin.databus.client.pub.mbean.UnifiedClientStats in project databus by linkedin.

the class TestUnifiedClientStats method testHistogramMetricsAggregationNoData.

/**
   * Tests aggregation (merging) of the histogram/percentile metrics (timeLagSourceToReceiptMs and
   * timeLagConsumerCallbacksMs) in the case that there have been no data events or callbacks, i.e.,
   * there are no data points in the histogram reservoirs.  All timeLagSourceToReceiptMs metrics
   * and timeLagConsumerCallbacksMs metrics should be -1.0, per the design spec.
   */
@Test
public void testHistogramMetricsAggregationNoData() {
    UnifiedClientStats unifiedClientStats1 = new UnifiedClientStats(1, /* ownerId */
    "test1", "dim1");
    UnifiedClientStats unifiedClientStats2 = new UnifiedClientStats(2, /* ownerId */
    "test2", "dim2");
    UnifiedClientStats unifiedClientStatsAgg = new UnifiedClientStats(99, /* ownerId */
    "testAgg", "dimAgg");
    assertEquals("unexpected timeLagSourceToReceiptMs 50th percentile for connection #1", -1.0, unifiedClientStats1.getTimeLagSourceToReceiptMs_HistPct_50());
    assertEquals("unexpected timeLagSourceToReceiptMs 90th percentile for connection #1", -1.0, unifiedClientStats1.getTimeLagSourceToReceiptMs_HistPct_90());
    assertEquals("unexpected timeLagSourceToReceiptMs 95th percentile for connection #1", -1.0, unifiedClientStats1.getTimeLagSourceToReceiptMs_HistPct_95());
    assertEquals("unexpected timeLagSourceToReceiptMs 99th percentile for connection #1", -1.0, unifiedClientStats1.getTimeLagSourceToReceiptMs_HistPct_99());
    assertEquals("unexpected timeLagSourceToReceiptMs 50th percentile for connection #2", -1.0, unifiedClientStats2.getTimeLagSourceToReceiptMs_HistPct_50());
    assertEquals("unexpected timeLagSourceToReceiptMs 90th percentile for connection #2", -1.0, unifiedClientStats2.getTimeLagSourceToReceiptMs_HistPct_90());
    assertEquals("unexpected timeLagSourceToReceiptMs 95th percentile for connection #2", -1.0, unifiedClientStats2.getTimeLagSourceToReceiptMs_HistPct_95());
    assertEquals("unexpected timeLagSourceToReceiptMs 99th percentile for connection #2", -1.0, unifiedClientStats2.getTimeLagSourceToReceiptMs_HistPct_99());
    assertEquals("unexpected timeLagSourceToReceiptMs 50th percentile for aggregated stats", -1.0, unifiedClientStatsAgg.getTimeLagSourceToReceiptMs_HistPct_50());
    assertEquals("unexpected timeLagSourceToReceiptMs 90th percentile for aggregated stats", -1.0, unifiedClientStatsAgg.getTimeLagSourceToReceiptMs_HistPct_90());
    assertEquals("unexpected timeLagSourceToReceiptMs 95th percentile for aggregated stats", -1.0, unifiedClientStatsAgg.getTimeLagSourceToReceiptMs_HistPct_95());
    assertEquals("unexpected timeLagSourceToReceiptMs 99th percentile for aggregated stats", -1.0, unifiedClientStatsAgg.getTimeLagSourceToReceiptMs_HistPct_99());
    assertEquals("unexpected timeLagConsumerCallbacksMs 50th percentile for connection #1", -1.0, unifiedClientStats1.getTimeLagConsumerCallbacksMs_HistPct_50());
    assertEquals("unexpected timeLagConsumerCallbacksMs 90th percentile for connection #1", -1.0, unifiedClientStats1.getTimeLagConsumerCallbacksMs_HistPct_90());
    assertEquals("unexpected timeLagConsumerCallbacksMs 95th percentile for connection #1", -1.0, unifiedClientStats1.getTimeLagConsumerCallbacksMs_HistPct_95());
    assertEquals("unexpected timeLagConsumerCallbacksMs 99th percentile for connection #1", -1.0, unifiedClientStats1.getTimeLagConsumerCallbacksMs_HistPct_99());
    assertEquals("unexpected timeLagConsumerCallbacksMs max for connection #1", -1.0, unifiedClientStats1.getTimeLagConsumerCallbacksMs_Max());
    assertEquals("unexpected timeLagConsumerCallbacksMs 50th percentile for connection #2", -1.0, unifiedClientStats2.getTimeLagConsumerCallbacksMs_HistPct_50());
    assertEquals("unexpected timeLagConsumerCallbacksMs 90th percentile for connection #2", -1.0, unifiedClientStats2.getTimeLagConsumerCallbacksMs_HistPct_90());
    assertEquals("unexpected timeLagConsumerCallbacksMs 95th percentile for connection #2", -1.0, unifiedClientStats2.getTimeLagConsumerCallbacksMs_HistPct_95());
    assertEquals("unexpected timeLagConsumerCallbacksMs 99th percentile for connection #2", -1.0, unifiedClientStats2.getTimeLagConsumerCallbacksMs_HistPct_99());
    assertEquals("unexpected timeLagConsumerCallbacksMs max for connection #2", -1.0, unifiedClientStats2.getTimeLagConsumerCallbacksMs_Max());
    assertEquals("unexpected timeLagConsumerCallbacksMs 50th percentile for aggregated stats", -1.0, unifiedClientStatsAgg.getTimeLagConsumerCallbacksMs_HistPct_50());
    assertEquals("unexpected timeLagConsumerCallbacksMs 90th percentile for aggregated stats", -1.0, unifiedClientStatsAgg.getTimeLagConsumerCallbacksMs_HistPct_90());
    assertEquals("unexpected timeLagConsumerCallbacksMs 95th percentile for aggregated stats", -1.0, unifiedClientStatsAgg.getTimeLagConsumerCallbacksMs_HistPct_95());
    assertEquals("unexpected timeLagConsumerCallbacksMs 99th percentile for aggregated stats", -1.0, unifiedClientStatsAgg.getTimeLagConsumerCallbacksMs_HistPct_99());
    assertEquals("unexpected timeLagConsumerCallbacksMs max for aggregated stats", -1.0, unifiedClientStatsAgg.getTimeLagConsumerCallbacksMs_Max());
}
Also used : UnifiedClientStats(com.linkedin.databus.client.pub.mbean.UnifiedClientStats) Test(org.testng.annotations.Test)

Example 4 with UnifiedClientStats

use of com.linkedin.databus.client.pub.mbean.UnifiedClientStats in project databus by linkedin.

the class DatabusHttpClientImpl method initializeRelayConnections.

private synchronized void initializeRelayConnections() {
    for (List<DatabusSubscription> subsList : _relayGroups.keySet()) {
        List<String> sourcesStrList = DatabusSubscription.getStrList(subsList);
        List<DatabusV2ConsumerRegistration> relayConsumers = getRelayGroupStreamConsumers().get(subsList);
        //nothing to do
        if (null == relayConsumers || 0 == relayConsumers.size())
            continue;
        try {
            DatabusSourcesConnection.StaticConfig connConfig = getClientStaticConfig().getConnection(sourcesStrList);
            if (null == connConfig) {
                connConfig = getClientStaticConfig().getConnectionDefaults();
            }
            // make sure we have the right policy.
            if (!connConfig.getEventBuffer().isEnableScnIndex() && connConfig.getEventBuffer().getQueuePolicy() != DbusEventBuffer.QueuePolicy.BLOCK_ON_WRITE) {
                throw new InvalidConfigException("If SCN index is disabled, queue policy must be BLOCK_ON_WRITE");
            }
            CheckpointPersistenceProvider cpPersistenceProvder = getCheckpointPersistenceProvider();
            if (null != cpPersistenceProvder && getClientStaticConfig().getCheckpointPersistence().isClearBeforeUse()) {
                cpPersistenceProvder.removeCheckpoint(sourcesStrList);
            }
            ServerInfo server0 = _relayGroups.get(subsList).iterator().next();
            ArrayList<DatabusV2ConsumerRegistration> bstConsumersRegs = new ArrayList<DatabusV2ConsumerRegistration>();
            for (List<DatabusSubscription> bstSubSourcesList : getRelayGroupBootstrapConsumers().keySet()) {
                List<DatabusV2ConsumerRegistration> bstRegsistrations = getRelayGroupBootstrapConsumers().get(bstSubSourcesList);
                for (DatabusV2ConsumerRegistration bstConsumerReg : bstRegsistrations) {
                    if (server0.supportsSources(bstConsumerReg.getSources())) {
                        bstConsumersRegs.add(bstConsumerReg);
                    }
                }
            }
            DbusEventBuffer eventBuffer = connConfig.getEventBuffer().getOrCreateEventBuffer(_eventFactory);
            eventBuffer.setDropOldEvents(true);
            eventBuffer.start(0);
            DbusEventBuffer bootstrapBuffer = null;
            // create bootstrap only if it is enabled
            if (_clientStaticConfig.getRuntime().getBootstrap().isEnabled()) {
                bootstrapBuffer = new DbusEventBuffer(connConfig.getEventBuffer());
                bootstrapBuffer.setDropOldEvents(false);
                bootstrapBuffer.start(0);
            }
            LOG.info("The sourcesList is " + sourcesStrList);
            LOG.info("The relayGroupStreamConsumers is " + getRelayGroupStreamConsumers().get(subsList));
            Set<ServerInfo> relays = _relayGroups.get(subsList);
            Set<ServerInfo> bootstrapServices = _bootstrapGroups.get(subsList);
            String statsCollectorName = generateSubsStatsName(sourcesStrList);
            int ownerId = getContainerStaticConfig().getId();
            _bootstrapEventsStats.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".inbound.bs", true, false, getMbeanServer()));
            _inBoundStatsCollectors.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".inbound", true, false, getMbeanServer()));
            _outBoundStatsCollectors.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".outbound", true, false, getMbeanServer()));
            _consumerStatsCollectors.addStatsCollector(statsCollectorName, new ConsumerCallbackStats(ownerId, statsCollectorName + ".inbound.cons", statsCollectorName + ".inbound.cons", true, false, null, getMbeanServer()));
            _bsConsumerStatsCollectors.addStatsCollector(statsCollectorName, new ConsumerCallbackStats(ownerId, statsCollectorName + ".inbound.bs.cons", statsCollectorName + ".inbound.bs.cons", true, false, null, getMbeanServer()));
            _unifiedClientStatsCollectors.addStatsCollector(statsCollectorName, new UnifiedClientStats(ownerId, statsCollectorName + ".inbound.unified.cons", statsCollectorName + ".inbound.unified.cons", true, false, _clientStaticConfig.getPullerThreadDeadnessThresholdMs(), null, getMbeanServer()));
            ConnectionStateFactory connStateFactory = new ConnectionStateFactory(DatabusSubscription.getStrList(subsList));
            DatabusSourcesConnection newConn = new DatabusSourcesConnection(connConfig, subsList, relays, bootstrapServices, relayConsumers, //_relayGroupBootstrapConsumers.get(sourcesList),
            bstConsumersRegs, eventBuffer, bootstrapBuffer, getDefaultExecutorService(), getContainerStatsCollector(), _inBoundStatsCollectors.getStatsCollector(statsCollectorName), _bootstrapEventsStats.getStatsCollector(statsCollectorName), _consumerStatsCollectors.getStatsCollector(statsCollectorName), _bsConsumerStatsCollectors.getStatsCollector(statsCollectorName), _unifiedClientStatsCollectors.getStatsCollector(statsCollectorName), getCheckpointPersistenceProvider(), getRelayConnFactory(), getBootstrapConnFactory(), getHttpStatsCollector(), null, this, _eventFactory, connStateFactory);
            newConn.start();
            _relayConnections.add(newConn);
        } catch (Exception e) {
            LOG.error("connection initialization issue for source(s):" + subsList + "; please check your configuration", e);
        }
    }
    if (0 == _relayConnections.size()) {
        LOG.warn("No connections specified");
    }
}
Also used : DatabusV2ConsumerRegistration(com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration) UnifiedClientStats(com.linkedin.databus.client.pub.mbean.UnifiedClientStats) ServerInfo(com.linkedin.databus.client.pub.ServerInfo) ConsumerCallbackStats(com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) ArrayList(java.util.ArrayList) DbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector) AggregatedDbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.AggregatedDbusEventsStatisticsCollector) InvalidConfigException(com.linkedin.databus.core.util.InvalidConfigException) DatabusSubscription(com.linkedin.databus.core.data_model.DatabusSubscription) URISyntaxException(java.net.URISyntaxException) DatabusException(com.linkedin.databus2.core.DatabusException) InvalidConfigException(com.linkedin.databus.core.util.InvalidConfigException) IOException(java.io.IOException) DatabusClientException(com.linkedin.databus.client.pub.DatabusClientException) DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer) SharedCheckpointPersistenceProvider(com.linkedin.databus.client.pub.SharedCheckpointPersistenceProvider) CheckpointPersistenceProvider(com.linkedin.databus.client.pub.CheckpointPersistenceProvider) ClusterCheckpointPersistenceProvider(com.linkedin.databus.client.pub.ClusterCheckpointPersistenceProvider) FileSystemCheckpointPersistenceProvider(com.linkedin.databus.client.pub.FileSystemCheckpointPersistenceProvider)

Example 5 with UnifiedClientStats

use of com.linkedin.databus.client.pub.mbean.UnifiedClientStats in project databus by linkedin.

the class RelayPullThread method writeEventToRelayDispatcher.

private void writeEventToRelayDispatcher(ConnectionState curState, DbusEvent event, String message) throws InterruptedException, InvalidEventException {
    boolean success = false;
    // Create a infinite backoff timer that waits for maximum of 1 sec
    // for writing the control message to evb
    BackoffTimerStaticConfig timerConfig = new BackoffTimerStaticConfig(1, 1000, 1, 1, -1);
    BackoffTimer timer = new BackoffTimer("EVB More Space Timer", timerConfig);
    timer.reset();
    byte[] eventBytes = new byte[event.size()];
    _log.info("Event size: " + eventBytes.length);
    _log.info("Event:" + event.toString());
    event.getRawBytes().get(eventBytes);
    UnifiedClientStats unifiedClientStats = _sourcesConn.getUnifiedClientStats();
    while ((!success) && (timer.getRemainingRetriesNum() > 0)) {
        ByteArrayInputStream cpIs = new ByteArrayInputStream(eventBytes);
        ReadableByteChannel cpRbc = Channels.newChannel(cpIs);
        sendHeartbeat(unifiedClientStats);
        int ecnt = curState.getDataEventsBuffer().readEvents(cpRbc);
        if (ecnt <= 0) {
            _log.error("Not enough spece in the event buffer to add a control message :" + message);
            boolean interrupted = !timer.backoffAndSleep();
            if (interrupted)
                throw new InterruptedException("Got interrupted while waiting to write control Message to EVB : " + message);
        } else {
            _log.info("Sent a control message :" + message);
            success = true;
        }
    }
}
Also used : ReadableByteChannel(java.nio.channels.ReadableByteChannel) UnifiedClientStats(com.linkedin.databus.client.pub.mbean.UnifiedClientStats) ByteArrayInputStream(java.io.ByteArrayInputStream) BackoffTimerStaticConfig(com.linkedin.databus2.core.BackoffTimerStaticConfig) Checkpoint(com.linkedin.databus.core.Checkpoint) BackoffTimer(com.linkedin.databus2.core.BackoffTimer)

Aggregations

UnifiedClientStats (com.linkedin.databus.client.pub.mbean.UnifiedClientStats)21 Test (org.testng.annotations.Test)10 ConsumerCallbackStats (com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats)9 Checkpoint (com.linkedin.databus.core.Checkpoint)9 DbusEvent (com.linkedin.databus.core.DbusEvent)9 ArrayList (java.util.ArrayList)7 DbusEventBuffer (com.linkedin.databus.core.DbusEventBuffer)6 DbusEventsStatisticsCollector (com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector)6 IdNamePair (com.linkedin.databus.core.util.IdNamePair)6 HashMap (java.util.HashMap)6 DatabusV2ConsumerRegistration (com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration)5 DatabusSubscription (com.linkedin.databus.core.data_model.DatabusSubscription)5 MultiConsumerCallback (com.linkedin.databus.client.consumer.MultiConsumerCallback)3 SelectingDatabusCombinedConsumer (com.linkedin.databus.client.consumer.SelectingDatabusCombinedConsumer)3 StreamConsumerCallbackFactory (com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory)3 DatabusStreamConsumer (com.linkedin.databus.client.pub.DatabusStreamConsumer)3 ConsumerCallbackResult (com.linkedin.databus.client.pub.ConsumerCallbackResult)2 ServerInfo (com.linkedin.databus.client.pub.ServerInfo)2 ConsumerCallbackStatsEvent (com.linkedin.databus.client.pub.monitoring.events.ConsumerCallbackStatsEvent)2 UnifiedClientStatsEvent (com.linkedin.databus.client.pub.monitoring.events.UnifiedClientStatsEvent)2