Examples with ProxyMessage org.apache.inlong.dataproxy.base.ProxyMessage used on opensource projects

Example 1 with ProxyMessage

use of org.apache.inlong.dataproxy.base.ProxyMessage in project incubator-inlong by apache.

the class ServerMessageHandler method updateMsgList.

private void updateMsgList(List<ProxyMessage> msgList, Map<String, String> commonAttrMap, Map<String, HashMap<String, List<ProxyMessage>>> messageMap, String strRemoteIP, MsgType msgType) {
    for (ProxyMessage message : msgList) {
        Map<String, String> attrMap = message.getAttributeMap();
        String topic = this.defaultTopic;
        AtomicReference<String> topicInfo = new AtomicReference<>(topic);
        checkGroupIdInfo(message, commonAttrMap, attrMap, topicInfo);
        topic = topicInfo.get();
        // if(groupId==null)groupId="b_test";//default groupId
        message.setTopic(topic);
        commonAttrMap.put(AttributeConstants.NODE_IP, strRemoteIP);
        String groupId = message.getGroupId();
        String streamId = message.getStreamId();
        // whether sla
        if (SLA_METRIC_GROUPID.equals(groupId)) {
            commonAttrMap.put(SLA_METRIC_DATA, "true");
            message.setTopic(SLA_METRIC_DATA);
        }
        if (groupId != null && streamId != null) {
            String tubeSwtichKey = groupId + SEPARATOR + streamId;
            if (configManager.getTubeSwitchProperties().get(tubeSwtichKey) != null && "false".equals(configManager.getTubeSwitchProperties().get(tubeSwtichKey).trim())) {
                continue;
            }
        }
        if (!"pb".equals(attrMap.get(AttributeConstants.MESSAGE_TYPE)) && !MsgType.MSG_MULTI_BODY.equals(msgType) && !MsgType.MSG_MULTI_BODY_ATTR.equals(msgType)) {
            byte[] data = message.getData();
            if (data[data.length - 1] == '\n') {
                int tripDataLen = data.length - 1;
                if (data[data.length - 2] == '\r') {
                    tripDataLen = data.length - 2;
                }
                byte[] tripData = new byte[tripDataLen];
                System.arraycopy(data, 0, tripData, 0, tripDataLen);
                message.setData(tripData);
            }
        }
        if (streamId == null) {
            streamId = "";
        }
        HashMap<String, List<ProxyMessage>> streamIdMsgMap = messageMap.computeIfAbsent(topic, k -> new HashMap<>());
        List<ProxyMessage> streamIdMsgList = streamIdMsgMap.computeIfAbsent(streamId, k -> new ArrayList<>());
        streamIdMsgList.add(message);
    }
}

Example 2 with ProxyMessage

use of org.apache.inlong.dataproxy.base.ProxyMessage in project incubator-inlong by apache.

the class SimpleMessageHandler method formatMessagesAndSend.

/**
 * formatMessagesAndSend
 *
 * @param  commonAttrMap
 * @param  messageMap
 * @param  strRemoteIP
 * @param  msgType
 * @throws MessageIDException
 */
private void formatMessagesAndSend(Map<String, String> commonAttrMap, Map<String, HashMap<String, List<ProxyMessage>>> messageMap, String strRemoteIP, MsgType msgType) throws MessageIDException {
    int inLongMsgVer = 1;
    if (MsgType.MSG_MULTI_BODY_ATTR.equals(msgType)) {
        inLongMsgVer = 3;
    } else if (MsgType.MSG_BIN_MULTI_BODY.equals(msgType)) {
        inLongMsgVer = 4;
    }
    for (Map.Entry<String, HashMap<String, List<ProxyMessage>>> topicEntry : messageMap.entrySet()) {
        for (Map.Entry<String, List<ProxyMessage>> streamIdEntry : topicEntry.getValue().entrySet()) {
            InLongMsg inLongMsg = InLongMsg.newInLongMsg(this.isCompressed, inLongMsgVer);
            Map<String, String> headers = new HashMap<String, String>();
            for (ProxyMessage message : streamIdEntry.getValue()) {
                if (MsgType.MSG_MULTI_BODY_ATTR.equals(msgType) || MsgType.MSG_MULTI_BODY.equals(msgType)) {
                    message.getAttributeMap().put(AttributeConstants.MESSAGE_COUNT, String.valueOf(1));
                    inLongMsg.addMsg(mapJoiner.join(message.getAttributeMap()), message.getData());
                } else if (MsgType.MSG_BIN_MULTI_BODY.equals(msgType)) {
                    inLongMsg.addMsg(message.getData());
                } else {
                    inLongMsg.addMsg(mapJoiner.join(message.getAttributeMap()), message.getData());
                }
            }
            long pkgTimeInMillis = inLongMsg.getCreatetime();
            String pkgTimeStr = dateFormator.get().format(pkgTimeInMillis);
            if (inLongMsgVer == 4) {
                if (commonAttrMap.containsKey(ConfigConstants.PKG_TIME_KEY)) {
                    pkgTimeStr = commonAttrMap.get(ConfigConstants.PKG_TIME_KEY);
                } else {
                    pkgTimeStr = dateFormator.get().format(System.currentTimeMillis());
                }
            }
            long dtTime = NumberUtils.toLong(commonAttrMap.get(AttributeConstants.DATA_TIME), System.currentTimeMillis());
            headers.put(AttributeConstants.DATA_TIME, String.valueOf(dtTime));
            headers.put(ConfigConstants.TOPIC_KEY, topicEntry.getKey());
            headers.put(AttributeConstants.STREAM_ID, streamIdEntry.getKey());
            headers.put(ConfigConstants.REMOTE_IP_KEY, strRemoteIP);
            headers.put(ConfigConstants.REMOTE_IDC_KEY, DEFAULT_REMOTE_IDC_VALUE);
            // every message share the same msg cnt? what if msgType = 5
            String proxyMetricMsgCnt = commonAttrMap.get(AttributeConstants.MESSAGE_COUNT);
            headers.put(ConfigConstants.MSG_COUNTER_KEY, proxyMetricMsgCnt);
            byte[] data = inLongMsg.buildArray();
            headers.put(ConfigConstants.TOTAL_LEN, String.valueOf(data.length));
            String sequenceId = commonAttrMap.get(AttributeConstants.SEQUENCE_ID);
            if (StringUtils.isNotEmpty(sequenceId)) {
                StringBuilder sidBuilder = new StringBuilder();
                sidBuilder.append(topicEntry.getKey()).append(SEPARATOR).append(streamIdEntry.getKey()).append(SEPARATOR).append(sequenceId);
                headers.put(ConfigConstants.SEQUENCE_ID, sidBuilder.toString());
            }
            headers.put(ConfigConstants.PKG_TIME_KEY, pkgTimeStr);
            // process proxy message list
            this.processProxyMessageList(headers, streamIdEntry.getValue());
        }
    }
}

Example 3 with ProxyMessage

use of org.apache.inlong.dataproxy.base.ProxyMessage in project incubator-inlong by apache.

the class SimpleMessageHandler method updateMsgList.

private void updateMsgList(List<ProxyMessage> msgList, Map<String, String> commonAttrMap, Map<String, HashMap<String, List<ProxyMessage>>> messageMap, String strRemoteIP, MsgType msgType) {
    for (ProxyMessage message : msgList) {
        Map<String, String> attrMap = message.getAttributeMap();
        String topic = this.defaultTopic;
        AtomicReference<String> topicInfo = new AtomicReference<>(topic);
        checkGroupIdInfo(message, commonAttrMap, attrMap, topicInfo);
        topic = topicInfo.get();
        // if(groupId==null)groupId="b_test";//default groupId
        message.setTopic(topic);
        commonAttrMap.put(AttributeConstants.NODE_IP, strRemoteIP);
        String groupId = message.getGroupId();
        String streamId = message.getStreamId();
        // whether sla
        if (SLA_METRIC_GROUPID.equals(groupId)) {
            commonAttrMap.put(SLA_METRIC_DATA, "true");
            message.setTopic(SLA_METRIC_DATA);
        }
        if (groupId != null && streamId != null) {
            String tubeSwtichKey = groupId + SEPARATOR + streamId;
            if (configManager.getTubeSwitchProperties().get(tubeSwtichKey) != null && "false".equals(configManager.getTubeSwitchProperties().get(tubeSwtichKey).trim())) {
                continue;
            }
        }
        if (!"pb".equals(attrMap.get(AttributeConstants.MESSAGE_TYPE)) && !MsgType.MSG_MULTI_BODY.equals(msgType) && !MsgType.MSG_MULTI_BODY_ATTR.equals(msgType)) {
            byte[] data = message.getData();
            if (data[data.length - 1] == '\n') {
                int tripDataLen = data.length - 1;
                if (data[data.length - 2] == '\r') {
                    tripDataLen = data.length - 2;
                }
                byte[] tripData = new byte[tripDataLen];
                System.arraycopy(data, 0, tripData, 0, tripDataLen);
                message.setData(tripData);
            }
        }
        if (streamId == null) {
            streamId = "";
        }
        HashMap<String, List<ProxyMessage>> streamIdMsgMap = messageMap.computeIfAbsent(topic, k -> new HashMap<>());
        List<ProxyMessage> streamIdMsgList = streamIdMsgMap.computeIfAbsent(streamId, k -> new ArrayList<>());
        streamIdMsgList.add(message);
    }
}

Example 4 with ProxyMessage

use of org.apache.inlong.dataproxy.base.ProxyMessage in project incubator-inlong by apache.

the class SimpleMessageHandler method processProxyMessageList.

/**
 * processProxyMessageList
 *
 * @param commonHeaders
 * @param proxyMessages
 */
private void processProxyMessageList(Map<String, String> commonHeaders, List<ProxyMessage> proxyMessages) {
    for (ProxyMessage message : proxyMessages) {
        Event event = this.parseProxyMessage2Event(commonHeaders, message);
        try {
            processor.processEvent(event);
            this.addMetric(true, event.getBody().length, event);
        } catch (Throwable ex) {
            logger.error("Error writting to channel,data will discard.", ex);
            this.addMetric(false, event.getBody().length, event);
            throw new ChannelException("ProcessEvent error can't write event to channel.");
        }
    }
}

Example 5 with ProxyMessage

use of org.apache.inlong.dataproxy.base.ProxyMessage in project incubator-inlong by apache.

the class DefaultServiceDecoder method extractDefaultData.

/**
 * extract bin data, message type less than 7
 * @param resultMap
 * @param cb
 * @param channel
 * @param totalDataLen
 * @param msgType
 * @return
 *
 * @throws Exception
 */
private Map<String, Object> extractDefaultData(Map<String, Object> resultMap, ByteBuf cb, Channel channel, int totalDataLen, MsgType msgType) throws Exception {
    int bodyLen = cb.readInt();
    if (bodyLen == 0) {
        throw new Exception(new Throwable("err msg,  bodyLen is empty" + ";" + "Connection info:" + channel.toString()));
    }
    // that means an invalid message, reject it.
    if (bodyLen > totalDataLen - 5) {
        throw new Exception(new Throwable("err msg, firstLen > totalDataLen, and bodyLen=" + bodyLen + ",totalDataLen=" + totalDataLen + ";Connection info:" + channel.toString()));
    }
    // extract body bytes
    byte[] bodyData = new byte[bodyLen];
    cb.readBytes(bodyData, 0, bodyLen);
    resultMap.put(ConfigConstants.DECODER_BODY, bodyData);
    int attrLen = cb.readInt();
    // 9 means bodyLen bytes(4) + message type bytes(1) + attrLen bytes(4)
    if (totalDataLen != 9 + attrLen + bodyLen) {
        throw new Exception(new Throwable("err msg, totalDataLen != 9 + bodyLen + attrLen,and bodyLen=" + bodyLen + ",totalDataLen=" + totalDataLen + ",attrDataLen=" + attrLen + ";Connection info:" + channel.toString()));
    }
    // extract attr bytes
    byte[] attrData = new byte[attrLen];
    cb.readBytes(attrData, 0, attrLen);
    String strAttr = new String(attrData, StandardCharsets.UTF_8);
    resultMap.put(ConfigConstants.DECODER_ATTRS, strAttr);
    // convert attr bytes to map
    Map<String, String> commonAttrMap = null;
    try {
        commonAttrMap = new HashMap<String, String>(mapSplitter.split(strAttr));
    } catch (Exception e) {
        throw new Exception(new Throwable("Parse commonAttrMap error.commonAttrString is: " + strAttr + " ,channel is :" + channel.toString()));
    }
    resultMap.put(ConfigConstants.COMMON_ATTR_MAP, commonAttrMap);
    // decompress body data if compress type exists.
    String compressType = commonAttrMap.get(AttributeConstants.COMPRESS_TYPE);
    resultMap.put(ConfigConstants.COMPRESS_TYPE, compressType);
    if (StringUtils.isNotBlank(compressType)) {
        byte[] unCompressedData = processUnCompress(bodyData, compressType);
        if (unCompressedData == null || unCompressedData.length == 0) {
            throw new Exception(new Throwable("Uncompress data error!compress type:" + compressType + ";data:" + new String(bodyData, StandardCharsets.UTF_8) + ";attr:" + strAttr + ";channel:" + channel.toString()));
        }
        bodyData = unCompressedData;
    }
    // fill up attr map with some keys.
    commonAttrMap.put(AttributeConstants.RCV_TIME, String.valueOf(System.currentTimeMillis()));
    String groupId = commonAttrMap.get(AttributeConstants.GROUP_ID);
    String streamId = commonAttrMap.get(AttributeConstants.STREAM_ID);
    // add message count attr
    String cntStr = commonAttrMap.get(AttributeConstants.MESSAGE_COUNT);
    int msgCnt = cntStr != null ? Integer.parseInt(cntStr) : 1;
    commonAttrMap.put(AttributeConstants.MESSAGE_COUNT, String.valueOf(msgCnt));
    // extract data from bodyData and if message type is 5, convert data into list.
    List<ProxyMessage> msgList = null;
    ByteBuffer bodyBuffer = ByteBuffer.wrap(bodyData);
    if (MsgType.MSG_MULTI_BODY.equals(msgType)) {
        msgList = new ArrayList<>(msgCnt);
        while (bodyBuffer.remaining() > 0) {
            int singleMsgLen = bodyBuffer.getInt();
            if (singleMsgLen <= 0 || singleMsgLen > bodyBuffer.remaining()) {
                throw new Exception(new Throwable("[Malformed Data]Invalid data len!channel is " + channel.toString()));
            }
            byte[] record = new byte[singleMsgLen];
            bodyBuffer.get(record);
            ProxyMessage message = new ProxyMessage(groupId, streamId, commonAttrMap, record);
            msgList.add(message);
        }
    } else {
        msgList = new ArrayList<>(1);
        msgList.add(new ProxyMessage(groupId, streamId, commonAttrMap, bodyData));
    }
    resultMap.put(ConfigConstants.MSG_LIST, msgList);
    return resultMap;
}