Examples with DbusEventsStatisticsCollector com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector used on opensource projects

Example 16 with DbusEventsStatisticsCollector

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.

the class ReadEventsTestParams method testScnIndexInvalidation.

@Test
public void testScnIndexInvalidation() throws Exception {
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    EventBufferTestInput input = new EventBufferTestInput();
    int numEvents = 1000;
    int numScns = 10;
    int windowSize = numEvents / numScns;
    DbusEventGenerator evGen = new DbusEventGenerator();
    for (int i = 1; i < 2; ++i) {
        LOG.info("I=" + i);
        int bufferSize = (windowSize * 2);
        input.setNumEvents(numEvents).setWindowSize(windowSize).setSharedBufferSize(bufferSize).setStagingBufferSize(bufferSize).setIndexSize(bufferSize).setIndividualBufferSize(bufferSize).setBatchSize(bufferSize).setProducerBufferSize(bufferSize).setPayloadSize(10).setDeleteInterval(1).setProdQueuePolicy(QueuePolicy.OVERWRITE_ON_WRITE);
        DbusEventsStatisticsCollector emitterStats = new DbusEventsStatisticsCollector(1, "appenderStats", true, true, null);
        if (srcTestEvents.isEmpty()) {
            assertTrue(evGen.generateEvents(numEvents, input.getWindowSize(), 512, input.getPayloadSize(), false, srcTestEvents) > 0);
        }
        int eventSize = 0;
        for (DbusEvent e : srcTestEvents) {
            if (!e.isControlMessage()) {
                eventSize = e.size();
                break;
            }
        }
        LOG.info("event size=" + eventSize);
        int indexSize = (input.getIndexSize() * eventSize / 100);
        LOG.info("indexSize=" + indexSize);
        DbusEventBuffer prodEventBuffer = new DbusEventBuffer(getConfig(input.getProducerBufferSize() * eventSize + i, input.getIndividualBufferSize() * eventSize, indexSize, input.getStagingBufferSize() * eventSize, AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(), input.getProdBufferAssertLevel()));
        LOG.info("Allocated buffer size=" + prodEventBuffer.getAllocatedSize());
        DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, prodEventBuffer, emitterStats, 0.600);
        Thread t = new Thread(eventProducer);
        t.start();
        t.join();
        DbusEventsTotalStats stats = emitterStats.getTotalStats();
        LOG.info("minScn=" + prodEventBuffer.getMinScn() + "totalEvents=" + stats.getNumDataEvents());
    //weak assertion ! just check it's not -1 for now
    //TODO! get this assertion to succeed
    //assertTrue(prodEventBuffer.getMinScn()!=-1);
    }
}

Example 17 with DbusEventsStatisticsCollector

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.

the class ReadEventsTestParams method testStreamEventsError.

/**
   * This tests makes sure that the event buffer generates only one error if an output channel
   * is closed (see DDSDBUS-835).
   */
@Test
public void testStreamEventsError() throws Exception {
    //allocate a small buffer
    DbusEventBuffer.Config confBuilder = new DbusEventBuffer.Config();
    confBuilder.setAllocationPolicy(AllocationPolicy.HEAP_MEMORY.toString());
    confBuilder.setMaxSize(1000);
    confBuilder.setScnIndexSize(80);
    confBuilder.setAverageEventSize(80);
    DbusEventBuffer buf = new DbusEventBuffer(confBuilder);
    DbusEventsStatisticsCollector statsCollector = new DbusEventsStatisticsCollector(1, "in", true, false, null);
    //generate some data in the buffer
    buf.start(0);
    final int W = 2;
    final int E = 2;
    for (int w = 1; w <= W; ++w) {
        buf.startEvents();
        int seq = w * 100;
        for (int i = 0; i < E; ++i) {
            buf.appendEvent(new DbusEventKey(i), (short) 1, (short) 1, System.nanoTime(), (short) 1, new byte[16], new byte[1], false, statsCollector);
        }
        buf.endEvents(seq, statsCollector);
    }
    //sanity check for the data
    Assert.assertEquals(W, statsCollector.getTotalStats().getNumSysEvents());
    Assert.assertEquals(W * E, statsCollector.getTotalStats().getNumDataEvents());
    File tmpFile = File.createTempFile(TestDbusEventBuffer.class.getSimpleName(), ".tmp");
    tmpFile.deleteOnExit();
    OutputStream outStream = new FileOutputStream(tmpFile);
    WritableByteChannel outChannel = Channels.newChannel(outStream);
    //stream events
    DbusEventsStatisticsCollector outStatsCollector = new DbusEventsStatisticsCollector(1, "out", true, false, null);
    Checkpoint cp = new Checkpoint();
    cp.setFlexible();
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    ErrorCountingAppender errorCountAppender = new ErrorCountingAppender();
    errorCountAppender.setName("errorCountAppender");
    DbusEventSerializable.LOG.addAppender(errorCountAppender);
    Level oldLevel = DbusEventSerializable.LOG.getLevel();
    //ensure ERROR log level because gradle likes to change it
    DbusEventSerializable.LOG.setLevel(Level.ERROR);
    int errNum = errorCountAppender.getErrorNum();
    StreamEventsArgs args = new StreamEventsArgs(10000).setStatsCollector(outStatsCollector);
    int written = buf.streamEvents(cp, outChannel, args).getNumEventsStreamed();
    Assert.assertEquals(errorCountAppender, DbusEventSerializable.LOG.getAppender("errorCountAppender"));
    Assert.assertEquals(W * (E + 1), written);
    Assert.assertEquals(errNum, errorCountAppender.getErrorNum());
    //close the channel to force an error for streamEvents
    outStream.close();
    outChannel.close();
    cp = new Checkpoint();
    cp.setFlexible();
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    errNum = errorCountAppender.getErrorNum();
    written = buf.streamEvents(cp, outChannel, args).getNumEventsStreamed();
    DbusEventSerializable.LOG.setLevel(oldLevel);
    Assert.assertEquals(errorCountAppender, DbusEventSerializable.LOG.getAppender("errorCountAppender"));
    Assert.assertEquals(0, written);
    Assert.assertEquals(errNum + 1, errorCountAppender.getErrorNum());
}

Example 18 with DbusEventsStatisticsCollector

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.

the class DatabusV2RegistrationImpl method initializeStatsCollectors.

/**
   * Initialize Statistics Collectors
   */
protected void initializeStatsCollectors(String regId, int ownerId, MBeanServer mbeanServer) {
    _inboundEventsStatsCollector = new DbusEventsStatisticsCollector(ownerId, regId + STREAM_EVENT_STATS_SUFFIX_NAME, true, false, mbeanServer);
    _bootstrapEventsStatsCollector = new DbusEventsStatisticsCollector(ownerId, regId + BOOTSTRAP_EVENT_STATS_SUFFIX_NAME, true, false, mbeanServer);
    _relayConsumerStats = new ConsumerCallbackStats(ownerId, regId + RELAY_CONSUMER_STATS_SUFFIX_NAME, regId, true, false, new ConsumerCallbackStatsEvent());
    _bootstrapConsumerStats = new ConsumerCallbackStats(ownerId, regId + BOOTSTRAP_CONSUMER_STATS_SUFFIX_NAME, regId, true, false, new ConsumerCallbackStatsEvent());
    _unifiedClientStats = new UnifiedClientStats(ownerId, regId + UNIFIED_CLIENT_STATS_SUFFIX_NAME, regId, true, false, _client.getClientStaticConfig().getPullerThreadDeadnessThresholdMs(), new UnifiedClientStatsEvent());
}

Example 19 with DbusEventsStatisticsCollector

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.

the class LoadDataEventsRequestProcessor method process.

@Override
public DatabusRequest process(DatabusRequest request) throws IOException, RequestProcessingException {
    String fileStr = request.getRequiredStringParam(FILE_PATH_PARAM);
    String startWinStr = request.getParams().getProperty(START_WINDOW_PARAM, "false");
    String physicalParitionParameter = request.getParams().getProperty(PHYSICAL_PARTITION_ID_PARAM);
    LOG.info(PHYSICAL_PARTITION_ID_PARAM + "=" + physicalParitionParameter);
    PhysicalPartition pPartition = PhysicalSourceStaticConfig.getDefaultPhysicalPartition();
    if (physicalParitionParameter != null) {
        // physical partition parameter format is PPName_PPId
        pPartition = PhysicalPartition.parsePhysicalPartitionString(physicalParitionParameter, "_");
    }
    boolean startWin = Boolean.valueOf(startWinStr);
    BufferedReader in = new BufferedReader(new FileReader(fileStr));
    try {
        //PhysicalPartition pPartition = new PhysicalPartition(physicalPartitionId);
        // TODO this should actually use DbusEventBufferAppendable (DDSDBUS-78)
        DbusEventBuffer buf = (DbusEventBuffer) _eventBuffer.getDbusEventBufferAppendable(pPartition);
        if (buf == null)
            throw new RequestProcessingException("cannot find buffer for ph. partion " + pPartition);
        if ((buf.getMinScn() < 0) && (buf.getPrevScn() < 0))
            buf.start(0);
        try {
            DbusEventsStatisticsCollector statsCollector = _relay.getInBoundStatsCollectors().getStatsCollector(pPartition.toSimpleString());
            int eventsAppended = 0;
            if (!((eventsAppended = DbusEventSerializable.appendToEventBuffer(in, buf, statsCollector, startWin)) > 0)) {
                throw new RequestProcessingException("event loading failed");
            }
            StringBuilder res = new StringBuilder(20);
            res.append("{\"eventsAppended\":");
            res.append(eventsAppended);
            res.append("}");
            request.getResponseContent().write(ByteBuffer.wrap(res.toString().getBytes(Charset.defaultCharset())));
        } catch (InvalidEventException iee) {
            throw new RequestProcessingException(iee);
        } catch (RuntimeException re) {
            LOG.error("runttime excception: " + re.getMessage(), re);
            throw re;
        }
    } finally {
        in.close();
    }
    return request;
}

Example 20 with DbusEventsStatisticsCollector

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.

the class ReadEventsRequestProcessor method process.

@Override
public DatabusRequest process(DatabusRequest request) throws IOException, RequestProcessingException, DatabusException {
    boolean isDebug = LOG.isDebugEnabled();
    try {
        ObjectMapper objMapper = new ObjectMapper();
        String checkpointString = request.getParams().getProperty(CHECKPOINT_PARAM, null);
        String checkpointStringMult = request.getParams().getProperty(CHECKPOINT_PARAM_MULT, null);
        int fetchSize = request.getRequiredIntParam(FETCH_SIZE_PARAM);
        String formatStr = request.getRequiredStringParam(OUTPUT_FORMAT_PARAM);
        Encoding enc = Encoding.valueOf(formatStr.toUpperCase());
        String sourcesListStr = request.getParams().getProperty(SOURCES_PARAM, null);
        String subsStr = request.getParams().getProperty(SUBS_PARAM, null);
        String partitionInfoStr = request.getParams().getProperty(PARTITION_INFO_STRING);
        String streamFromLatestSCNStr = request.getParams().getProperty(STREAM_FROM_LATEST_SCN);
        String clientMaxEventVersionStr = request.getParams().getProperty(DatabusHttpHeaders.MAX_EVENT_VERSION);
        int clientEventVersion = (clientMaxEventVersionStr != null) ? Integer.parseInt(clientMaxEventVersionStr) : DbusEventFactory.DBUS_EVENT_V1;
        if (clientEventVersion < 0 || clientEventVersion == 1 || clientEventVersion > DbusEventFactory.DBUS_EVENT_V2) {
            throw new InvalidRequestParamValueException(COMMAND_NAME, DatabusHttpHeaders.MAX_EVENT_VERSION, clientMaxEventVersionStr);
        }
        if (null == sourcesListStr && null == subsStr) {
            throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM + "|" + SUBS_PARAM, "null");
        }
        //TODO for now we separte the code paths to limit the impact on existing Databus 2 deployments (DDSDBUS-79)
        //We have to get rid of this eventually and have a single data path.
        boolean v2Mode = null == subsStr;
        DbusKeyCompositeFilter keyCompositeFilter = null;
        if (null != partitionInfoStr) {
            try {
                Map<Long, DbusKeyFilter> fMap = KeyFilterConfigJSONFactory.parseSrcIdFilterConfigMap(partitionInfoStr);
                keyCompositeFilter = new DbusKeyCompositeFilter();
                keyCompositeFilter.setFilterMap(fMap);
                if (isDebug)
                    LOG.debug("keyCompositeFilter is :" + keyCompositeFilter);
            } catch (Exception ex) {
                String msg = "Got exception while parsing partition Configs. PartitionInfo is:" + partitionInfoStr;
                LOG.error(msg, ex);
                throw new InvalidRequestParamValueException(COMMAND_NAME, PARTITION_INFO_STRING, partitionInfoStr);
            }
        }
        boolean streamFromLatestSCN = false;
        if (null != streamFromLatestSCNStr) {
            streamFromLatestSCN = Boolean.valueOf(streamFromLatestSCNStr);
        }
        long start = System.currentTimeMillis();
        List<DatabusSubscription> subs = null;
        //parse source ids
        SourceIdNameRegistry srcRegistry = _relay.getSourcesIdNameRegistry();
        HashSet<Integer> sourceIds = new HashSet<Integer>();
        if (null != sourcesListStr) {
            String[] sourcesList = sourcesListStr.split(",");
            for (String sourceId : sourcesList) {
                try {
                    Integer srcId = Integer.valueOf(sourceId);
                    sourceIds.add(srcId);
                } catch (NumberFormatException nfe) {
                    HttpStatisticsCollector globalHttpStatsCollector = _relay.getHttpStatisticsCollector();
                    if (null != globalHttpStatsCollector) {
                        globalHttpStatsCollector.registerInvalidStreamRequest();
                    }
                    throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM, sourceId);
                }
            }
        }
        //process explicit subscriptions and generate respective logical partition filters
        NavigableSet<PhysicalPartitionKey> ppartKeys = null;
        if (null != subsStr) {
            List<DatabusSubscription.Builder> subsBuilder = null;
            subsBuilder = objMapper.readValue(subsStr, new TypeReference<List<DatabusSubscription.Builder>>() {
            });
            subs = new ArrayList<DatabusSubscription>(subsBuilder.size());
            for (DatabusSubscription.Builder subBuilder : subsBuilder) {
                subs.add(subBuilder.build());
            }
            ppartKeys = new TreeSet<PhysicalPartitionKey>();
            for (DatabusSubscription sub : subs) {
                PhysicalPartition ppart = sub.getPhysicalPartition();
                if (ppart.isAnyPartitionWildcard()) {
                    ppartKeys = _eventBuffer.getAllPhysicalPartitionKeys();
                    break;
                } else {
                    ppartKeys.add(new PhysicalPartitionKey(ppart));
                }
            }
        }
        // Need to make sure that we don't have tests that send requests in this form.
        if (subs != null && checkpointStringMult == null && checkpointString != null) {
            throw new RequestProcessingException("Both Subscriptions and CheckpointMult should be present");
        }
        //convert source ids into subscriptions
        if (null == subs)
            subs = new ArrayList<DatabusSubscription>();
        for (Integer srcId : sourceIds) {
            LogicalSource lsource = srcRegistry.getSource(srcId);
            if (lsource == null)
                throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM, srcId.toString());
            if (isDebug)
                LOG.debug("registry returns " + lsource + " for srcid=" + srcId);
            DatabusSubscription newSub = DatabusSubscription.createSimpleSourceSubscription(lsource);
            subs.add(newSub);
        }
        DbusFilter ppartFilters = null;
        if (subs.size() > 0) {
            try {
                ppartFilters = _eventBuffer.constructFilters(subs);
            } catch (DatabusException de) {
                throw new RequestProcessingException("unable to generate physical partitions filters:" + de.getMessage(), de);
            }
        }
        ConjunctionDbusFilter filters = new ConjunctionDbusFilter();
        // Source filter comes first
        if (v2Mode)
            filters.addFilter(new SourceDbusFilter(sourceIds));
        else if (null != ppartFilters)
            filters.addFilter(ppartFilters);
        /*
      // Key range filter comes next
      if ((keyMin >0) && (keyMax > 0))
      {
        filters.addFilter(new KeyRangeFilter(keyMin, keyMax));
      }
      */
        if (null != keyCompositeFilter) {
            filters.addFilter(keyCompositeFilter);
        }
        // need to update registerStreamRequest to support Mult checkpoint TODO (DDSDBUS-80)
        // temp solution
        // 3 options:
        // 1. checkpointStringMult not null - generate checkpoint from it
        // 2. checkpointStringMult null, checkpointString not null - create empty CheckpointMult
        // and add create Checkpoint(checkpointString) and add it to cpMult;
        // 3 both are null - create empty CheckpointMult and add empty Checkpoint to it for each ppartition
        PhysicalPartition pPartition;
        Checkpoint cp = null;
        CheckpointMult cpMult = null;
        if (checkpointStringMult != null) {
            try {
                cpMult = new CheckpointMult(checkpointStringMult);
            } catch (InvalidParameterSpecException e) {
                LOG.error("Invalid CheckpointMult:" + checkpointStringMult, e);
                throw new InvalidRequestParamValueException("stream", "CheckpointMult", checkpointStringMult);
            }
        } else {
            // there is no checkpoint - create an empty one
            cpMult = new CheckpointMult();
            Iterator<Integer> it = sourceIds.iterator();
            while (it.hasNext()) {
                Integer srcId = it.next();
                pPartition = _eventBuffer.getPhysicalPartition(srcId);
                if (pPartition == null)
                    throw new RequestProcessingException("unable to find physical partitions for source:" + srcId);
                if (checkpointString != null) {
                    cp = new Checkpoint(checkpointString);
                } else {
                    cp = new Checkpoint();
                    cp.setFlexible();
                }
                cpMult.addCheckpoint(pPartition, cp);
            }
        }
        if (isDebug)
            LOG.debug("checkpointStringMult = " + checkpointStringMult + ";singlecheckpointString=" + checkpointString + ";CPM=" + cpMult);
        // of the server context.
        if (cpMult.getCursorPartition() == null) {
            cpMult.setCursorPartition(request.getCursorPartition());
        }
        if (isDebug) {
            if (cpMult.getCursorPartition() != null) {
                LOG.debug("Using physical paritition cursor " + cpMult.getCursorPartition());
            }
        }
        // for registerStreamRequest we need a single Checkpoint (TODO - fix it) (DDSDBUS-81)
        if (cp == null) {
            Iterator<Integer> it = sourceIds.iterator();
            if (it.hasNext()) {
                Integer srcId = it.next();
                pPartition = _eventBuffer.getPhysicalPartition(srcId);
                cp = cpMult.getCheckpoint(pPartition);
            } else {
                cp = new Checkpoint();
                cp.setFlexible();
            }
        }
        if (null != checkpointString && isDebug)
            LOG.debug("About to stream from cp: " + checkpointString.toString());
        HttpStatisticsCollector globalHttpStatsCollector = _relay.getHttpStatisticsCollector();
        HttpStatisticsCollector connHttpStatsCollector = null;
        if (null != globalHttpStatsCollector) {
            connHttpStatsCollector = (HttpStatisticsCollector) request.getParams().get(globalHttpStatsCollector.getName());
        }
        if (null != globalHttpStatsCollector)
            globalHttpStatsCollector.registerStreamRequest(cp, sourceIds);
        StatsCollectors<DbusEventsStatisticsCollector> statsCollectors = _relay.getOutBoundStatsCollectors();
        try {
            DbusEventBufferBatchReadable bufRead = v2Mode ? _eventBuffer.getDbusEventBufferBatchReadable(sourceIds, cpMult, statsCollectors) : _eventBuffer.getDbusEventBufferBatchReadable(cpMult, ppartKeys, statsCollectors);
            int eventsRead = 0;
            int minPendingEventSize = 0;
            StreamEventsResult result = null;
            bufRead.setClientMaxEventVersion(clientEventVersion);
            if (v2Mode) {
                result = bufRead.streamEvents(streamFromLatestSCN, fetchSize, request.getResponseContent(), enc, filters);
                eventsRead = result.getNumEventsStreamed();
                minPendingEventSize = result.getSizeOfPendingEvent();
                if (isDebug) {
                    LOG.debug("Process: streamed " + eventsRead + " from sources " + Arrays.toString(sourceIds.toArray()));
                    //can be used for debugging to stream from a cp
                    LOG.debug("CP=" + cpMult);
                }
            //if (null != statsCollectors) statsCollectors.mergeStatsCollectors();
            } else {
                result = bufRead.streamEvents(streamFromLatestSCN, fetchSize, request.getResponseContent(), enc, filters);
                eventsRead = result.getNumEventsStreamed();
                minPendingEventSize = result.getSizeOfPendingEvent();
                if (isDebug)
                    LOG.debug("Process: streamed " + eventsRead + " with subscriptions " + subs);
                cpMult = bufRead.getCheckpointMult();
                if (cpMult != null) {
                    request.setCursorPartition(cpMult.getCursorPartition());
                }
            }
            if (eventsRead == 0 && minPendingEventSize > 0) {
                // Append a header to indicate to the client that we do have at least one event to
                // send, but it is too large to fit into client's offered buffer.
                request.getResponseContent().addMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE, minPendingEventSize);
                LOG.debug("Returning 0 events but have pending event of size " + minPendingEventSize);
            }
        } catch (ScnNotFoundException snfe) {
            if (null != globalHttpStatsCollector) {
                globalHttpStatsCollector.registerScnNotFoundStreamResponse();
            }
            throw new RequestProcessingException(snfe);
        } catch (OffsetNotFoundException snfe) {
            LOG.error("OffsetNotFound", snfe);
            if (null != globalHttpStatsCollector) {
                globalHttpStatsCollector.registerScnNotFoundStreamResponse();
            }
            throw new RequestProcessingException(snfe);
        }
        if (null != connHttpStatsCollector) {
            connHttpStatsCollector.registerStreamResponse(System.currentTimeMillis() - start);
            globalHttpStatsCollector.merge(connHttpStatsCollector);
            connHttpStatsCollector.reset();
        } else if (null != globalHttpStatsCollector) {
            globalHttpStatsCollector.registerStreamResponse(System.currentTimeMillis() - start);
        }
    } catch (InvalidRequestParamValueException e) {
        HttpStatisticsCollector globalHttpStatsCollector = _relay.getHttpStatisticsCollector();
        if (null != globalHttpStatsCollector) {
            globalHttpStatsCollector.registerInvalidStreamRequest();
        }
        throw e;
    }
    return request;
}