Examples with InLongMsg org.apache.inlong.common.msg.InLongMsg used on opensource projects

Example 16 with InLongMsg

use of org.apache.inlong.common.msg.InLongMsg in project incubator-inlong by apache.

the class SimpleMessageHandler method processMessage.

@Override
public void processMessage(Context context) throws MessageProcessException {
    String topic = "test";
    String topicValue = topic;
    String attr = "m=0";
    StringBuffer newAttrBuffer = new StringBuffer(attr);
    String groupId = (String) context.get(HttpSourceConstants.GROUP_ID);
    String streamId = (String) context.get(HttpSourceConstants.STREAM_ID);
    String dt = (String) context.get(HttpSourceConstants.DATA_TIME);
    String value = getTopic(groupId, streamId);
    if (null != value && !"".equals(value)) {
        topicValue = value.trim();
    }
    String mxValue = configManager.getMxProperties().get(groupId);
    if (null != mxValue) {
        newAttrBuffer = new StringBuffer(mxValue.trim());
    }
    newAttrBuffer.append("&groupId=").append(groupId).append("&streamId=").append(streamId).append("&dt=").append(dt);
    HttpServletRequest request = (HttpServletRequest) context.get(HttpSourceConstants.HTTP_REQUEST);
    String strRemoteIP = request.getRemoteAddr();
    newAttrBuffer.append("&NodeIP=").append(strRemoteIP);
    String msgCount = request.getParameter(HttpSourceConstants.MESSAGE_COUNT);
    if (msgCount == null || "".equals(msgCount)) {
        msgCount = "1";
    }
    InLongMsg inLongMsg = InLongMsg.newInLongMsg(true);
    String charset = (String) context.get(HttpSourceConstants.CHARSET);
    if (charset == null || "".equals(charset)) {
        charset = "UTF-8";
    }
    String body = (String) context.get(HttpSourceConstants.BODY);
    try {
        inLongMsg.addMsg(newAttrBuffer.toString(), body.getBytes(charset));
    } catch (UnsupportedEncodingException e) {
        throw new MessageProcessException(e);
    }
    Map<String, String> headers = new HashMap<String, String>();
    headers.put(HttpSourceConstants.DATA_TIME, dt);
    headers.put(ConfigConstants.TOPIC_KEY, topicValue);
    headers.put(AttributeConstants.STREAM_ID, streamId);
    headers.put(ConfigConstants.REMOTE_IP_KEY, strRemoteIP);
    headers.put(ConfigConstants.REMOTE_IDC_KEY, DEFAULT_REMOTE_IDC_VALUE);
    headers.put(ConfigConstants.MSG_COUNTER_KEY, msgCount);
    byte[] data = inLongMsg.buildArray();
    headers.put(ConfigConstants.TOTAL_LEN, String.valueOf(data.length));
    String pkgTime = dateFormator.get().format(inLongMsg.getCreatetime());
    headers.put(ConfigConstants.PKG_TIME_KEY, pkgTime);
    Event event = EventBuilder.withBody(data, headers);
    String counterExtKey = topicValue + "#" + 0 + "#" + strRemoteIP + "#time#" + pkgTime;
    counterGroupExt.addAndGet(counterExtKey, Long.valueOf(msgCount));
    long dtten = 0;
    try {
        dtten = Long.parseLong(dt);
    } catch (NumberFormatException e1) {
        throw new MessageProcessException(new Throwable("attribute dt=" + dt + " has error," + " detail is: " + newAttrBuffer));
    }
    dtten = dtten / 1000 / 60 / 10;
    dtten = dtten * 1000 * 60 * 10;
    StringBuilder newbase = new StringBuilder();
    newbase.append("http").append(SEPARATOR).append(topicValue).append(SEPARATOR).append(streamId).append(SEPARATOR).append(strRemoteIP).append(SEPARATOR).append(NetworkUtils.getLocalIp()).append(SEPARATOR).append(new SimpleDateFormat("yyyyMMddHHmm").format(dtten)).append(SEPARATOR).append(pkgTime);
    if (isNewMetricOn) {
        monitorIndex.addAndGet(new String(newbase), Integer.parseInt(msgCount), 1, data.length, 0);
    }
    inLongMsg.reset();
    long beginTime = System.currentTimeMillis();
    try {
        processor.processEvent(event);
        counterGroup.incrementAndGet(StatConstants.EVENT_SUCCESS);
        monitorIndexExt.incrementAndGet("EVENT_SUCCESS");
    } catch (ChannelException ex) {
        counterGroup.incrementAndGet(StatConstants.EVENT_DROPPED);
        monitorIndexExt.incrementAndGet("EVENT_DROPPED");
        if (isNewMetricOn) {
            monitorIndex.addAndGet(new String(newbase), 0, 0, 0, Integer.parseInt(msgCount));
        }
        logCounter++;
        if (logCounter == 1 || logCounter % 1000 == 0) {
            LOG.error("Error writting to channel,and will retry after 1s,ex={}," + "logCounter = {}, spend time={} ms", new Object[] { ex.toString(), logCounter, System.currentTimeMillis() - beginTime });
            if (logCounter > Long.MAX_VALUE - 10) {
                logCounter = 0;
                LOG.info("logCounter will reverse.");
            }
        }
        throw ex;
    }
    channelTrace++;
    if (channelTrace % 600000 == 0) {
        LOG.info("processor.processEvent spend time={} ms", System.currentTimeMillis() - beginTime);
    }
    if (channelTrace > Long.MAX_VALUE - 10) {
        channelTrace = 0;
        LOG.info("channelTrace will reverse.");
    }
}