Examples with InLongMsg org.apache.inlong.common.msg.InLongMsg used on opensource projects

Example 11 with InLongMsg

use of org.apache.inlong.common.msg.InLongMsg in project incubator-inlong by apache.

the class InLongMsgCsvFormatDeserializerTest method testIgnoreAttributeErrors.

@Test
public void testIgnoreAttributeErrors() throws Exception {
    InLongMsgCsvFormatDeserializer deserializer = new InLongMsgCsvFormatDeserializer(TEST_ROW_INFO, DEFAULT_TIME_FIELD_NAME, DEFAULT_ATTRIBUTES_FIELD_NAME, Charset.defaultCharset().name(), TableFormatConstants.DEFAULT_DELIMITER, null, null, null, true, true);
    InLongMsg inLongMsg = InLongMsg.newInLongMsg();
    String attrs = "m=0&&&&";
    String body1 = "123,field11,field12,field13";
    inLongMsg.addMsg(attrs, body1.getBytes());
    testRowDeserialization(deserializer, inLongMsg.buildArray(), Collections.emptyList());
}

Example 12 with InLongMsg

use of org.apache.inlong.common.msg.InLongMsg in project incubator-inlong by apache.

the class InLongMsgCsvFormatDeserializerTest method testNoPredefinedFields.

@Test
public void testNoPredefinedFields() throws Exception {
    final InLongMsgCsvFormatDeserializer deserializer = new InLongMsgCsvFormatDeserializer(TEST_ROW_INFO);
    InLongMsg inLongMsg = InLongMsg.newInLongMsg();
    String attrs = "m=0&" + INLONGMSG_ATTR_STREAM_ID + "=testInterfaceId&t=20200322";
    String body1 = "1,2,123,field11,field12,";
    String body2 = "1,2,123,field21,,field23";
    inLongMsg.addMsg(attrs, body1.getBytes());
    inLongMsg.addMsg(attrs, body2.getBytes());
    Map<String, String> expectedAttributes = new HashMap<>();
    expectedAttributes.put("m", "0");
    expectedAttributes.put(INLONGMSG_ATTR_STREAM_ID, "testInterfaceId");
    expectedAttributes.put("t", "20200322");
    Row expectedRow1 = Row.of(Timestamp.valueOf("2020-03-22 00:00:00"), expectedAttributes, 1, 2, 123, "field11", "field12", "");
    Row expectedRow2 = Row.of(Timestamp.valueOf("2020-03-22 00:00:00"), expectedAttributes, 1, 2, 123, "field21", "", "field23");
    testRowDeserialization(deserializer, inLongMsg.buildArray(), Arrays.asList(expectedRow1, expectedRow2));
}

Example 13 with InLongMsg

use of org.apache.inlong.common.msg.InLongMsg in project incubator-inlong by apache.

the class InLongMsgCsvFormatDeserializerTest method testDeleteHeadDelimiter.

@Test
public void testDeleteHeadDelimiter() throws Exception {
    final InLongMsgCsvFormatDeserializer deserializer = new InLongMsgCsvFormatDeserializer(TEST_ROW_INFO, DEFAULT_TIME_FIELD_NAME, DEFAULT_ATTRIBUTES_FIELD_NAME, Charset.defaultCharset().name(), TableFormatConstants.DEFAULT_DELIMITER, null, null, null, true, true);
    InLongMsg inLongMsg = InLongMsg.newInLongMsg();
    String attrs = "m=0&" + INLONGMSG_ATTR_STREAM_ID + "=testInterfaceId&t=20200322";
    String body = ",1,2,3,field1,field2,field3";
    inLongMsg.addMsg(attrs, body.getBytes());
    Map<String, String> expectedAttributes = new HashMap<>();
    expectedAttributes.put("m", "0");
    expectedAttributes.put(INLONGMSG_ATTR_STREAM_ID, "testInterfaceId");
    expectedAttributes.put("t", "20200322");
    Row expectedRow = Row.of(Timestamp.valueOf("2020-03-22 00:00:00"), expectedAttributes, 1, 2, 3, "field1", "field2", "field3");
    testRowDeserialization(deserializer, inLongMsg.buildArray(), Collections.singletonList(expectedRow));
}

Example 14 with InLongMsg

use of org.apache.inlong.common.msg.InLongMsg in project incubator-inlong by apache.

the class ServerMessageHandler method formatMessagesAndSend.

private void formatMessagesAndSend(ChannelHandlerContext ctx, Map<String, String> commonAttrMap, Map<String, HashMap<String, List<ProxyMessage>>> messageMap, String strRemoteIP, MsgType msgType) throws MessageIDException {
    int inLongMsgVer = 1;
    if (MsgType.MSG_MULTI_BODY_ATTR.equals(msgType)) {
        inLongMsgVer = 3;
    } else if (MsgType.MSG_BIN_MULTI_BODY.equals(msgType)) {
        inLongMsgVer = 4;
    }
    for (Map.Entry<String, HashMap<String, List<ProxyMessage>>> topicEntry : messageMap.entrySet()) {
        for (Map.Entry<String, List<ProxyMessage>> streamIdEntry : topicEntry.getValue().entrySet()) {
            InLongMsg inLongMsg = InLongMsg.newInLongMsg(this.isCompressed, inLongMsgVer);
            Map<String, String> headers = new HashMap<String, String>();
            for (ProxyMessage message : streamIdEntry.getValue()) {
                if (MsgType.MSG_MULTI_BODY_ATTR.equals(msgType) || MsgType.MSG_MULTI_BODY.equals(msgType)) {
                    message.getAttributeMap().put(AttributeConstants.MESSAGE_COUNT, String.valueOf(1));
                    inLongMsg.addMsg(mapJoiner.join(message.getAttributeMap()), message.getData());
                } else if (MsgType.MSG_BIN_MULTI_BODY.equals(msgType)) {
                    inLongMsg.addMsg(message.getData());
                } else {
                    inLongMsg.addMsg(mapJoiner.join(message.getAttributeMap()), message.getData());
                }
            }
            long pkgTimeInMillis = inLongMsg.getCreatetime();
            String pkgTimeStr = dateFormator.get().format(pkgTimeInMillis);
            if (inLongMsgVer == 4) {
                if (commonAttrMap.containsKey(ConfigConstants.PKG_TIME_KEY)) {
                    pkgTimeStr = commonAttrMap.get(ConfigConstants.PKG_TIME_KEY);
                } else {
                    pkgTimeStr = dateFormator.get().format(System.currentTimeMillis());
                }
            }
            if (commonAttrMap.get(AttributeConstants.DATA_TIME) != null) {
                headers.put(AttributeConstants.DATA_TIME, commonAttrMap.get(AttributeConstants.DATA_TIME));
            } else {
                headers.put(AttributeConstants.DATA_TIME, String.valueOf(System.currentTimeMillis()));
            }
            String syncSend = commonAttrMap.get(AttributeConstants.MESSAGE_SYNC_SEND);
            if (StringUtils.isNotEmpty(syncSend)) {
                headers.put(AttributeConstants.MESSAGE_SYNC_SEND, syncSend);
            }
            String partitionKey = commonAttrMap.get(AttributeConstants.MESSAGE_PARTITION_KEY);
            if (StringUtils.isNotEmpty(partitionKey)) {
                headers.put(AttributeConstants.MESSAGE_PARTITION_KEY, partitionKey);
            }
            headers.put(ConfigConstants.TOPIC_KEY, topicEntry.getKey());
            headers.put(AttributeConstants.GROUP_ID, streamIdEntry.getValue().get(0).getGroupId());
            headers.put(AttributeConstants.STREAM_ID, streamIdEntry.getKey());
            headers.put(ConfigConstants.REMOTE_IP_KEY, strRemoteIP);
            headers.put(ConfigConstants.REMOTE_IDC_KEY, DEFAULT_REMOTE_IDC_VALUE);
            // every message share the same msg cnt? what if msgType = 5
            String proxyMetricMsgCnt = commonAttrMap.get(AttributeConstants.MESSAGE_COUNT);
            headers.put(ConfigConstants.MSG_COUNTER_KEY, proxyMetricMsgCnt);
            byte[] data = inLongMsg.buildArray();
            headers.put(ConfigConstants.TOTAL_LEN, String.valueOf(data.length));
            headers.put(AttributeConstants.UNIQ_ID, commonAttrMap.get(AttributeConstants.UNIQ_ID));
            String sequenceId = commonAttrMap.get(AttributeConstants.SEQUENCE_ID);
            if (StringUtils.isNotEmpty(sequenceId)) {
                StringBuilder sidBuilder = new StringBuilder();
                sidBuilder.append(topicEntry.getKey()).append(SEPARATOR).append(streamIdEntry.getKey()).append(SEPARATOR).append(sequenceId);
                headers.put(ConfigConstants.SEQUENCE_ID, sidBuilder.toString());
            }
            headers.put(ConfigConstants.PKG_TIME_KEY, pkgTimeStr);
            Event event = EventBuilder.withBody(data, headers);
            if (MessageUtils.isSyncSendForOrder(event)) {
                event = new OrderEvent(ctx, event);
            }
            long dtten = 0;
            try {
                dtten = Long.parseLong(headers.get(AttributeConstants.DATA_TIME));
            } catch (Exception e1) {
                long uniqVal = Long.parseLong(commonAttrMap.get(AttributeConstants.UNIQ_ID));
                throw new MessageIDException(uniqVal, ErrorCode.DT_ERROR, new Throwable("attribute dt=" + headers.get(AttributeConstants.DATA_TIME + " has error, detail is: topic=" + topicEntry.getKey() + "&streamId=" + streamIdEntry.getKey() + "&NodeIP=" + strRemoteIP), e1));
            }
            dtten = dtten / 1000 / 60 / 10;
            dtten = dtten * 1000 * 60 * 10;
            StringBuilder newbase = new StringBuilder();
            newbase.append(protocolType).append(SEPARATOR).append(topicEntry.getKey()).append(SEPARATOR).append(streamIdEntry.getKey()).append(SEPARATOR).append(strRemoteIP).append(SEPARATOR).append(NetworkUtils.getLocalIp()).append(SEPARATOR).append(new SimpleDateFormat("yyyyMMddHHmm").format(dtten)).append(SEPARATOR).append(pkgTimeStr);
            try {
                processor.processEvent(event);
                monitorIndexExt.incrementAndGet("EVENT_SUCCESS");
                this.addMetric(true, data.length, event);
                monitorIndex.addAndGet(new String(newbase), Integer.parseInt(proxyMetricMsgCnt), 1, data.length, 0);
            } catch (Throwable ex) {
                logger.error("Error writting to channel,data will discard.", ex);
                monitorIndexExt.incrementAndGet("EVENT_DROPPED");
                monitorIndex.addAndGet(new String(newbase), 0, 0, 0, Integer.parseInt(proxyMetricMsgCnt));
                this.addMetric(false, data.length, event);
                throw new ChannelException("ProcessEvent error can't write event to channel.");
            }
        }
    }
}

Example 15 with InLongMsg

use of org.apache.inlong.common.msg.InLongMsg in project incubator-inlong by apache.

the class MultiTenancyInLongMsgMixedDeserializerTest method testDeserialize.

@Test
public void testDeserialize() throws Exception {
    final MultiTenancyInLongMsgMixedDeserializer deserializer = new MultiTenancyInLongMsgMixedDeserializer();
    final FieldInfo stringField = new FieldInfo("not_important", new StringFormatInfo());
    final FieldInfo longField = new FieldInfo("id", new LongFormatInfo());
    final TubeSourceInfo tubeSourceInfo = new TubeSourceInfo("topic", "address", null, new InLongMsgCsvDeserializationInfo("tid", '|', false), new FieldInfo[] { stringField, longField });
    final EmptySinkInfo sinkInfo = new EmptySinkInfo();
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, tubeSourceInfo, sinkInfo);
    deserializer.addDataFlow(dataFlowInfo);
    final InLongMsg inLongMsg = InLongMsg.newInLongMsg();
    final String attrs = "m=0&" + InLongMsgUtils.INLONGMSG_ATTR_STREAM_ID + "=tid&t=20210513";
    final String body1 = "tianqiwan|29";
    inLongMsg.addMsg(attrs, body1.getBytes());
    final TestingCollector<Record> collector = new TestingCollector<>();
    deserializer.deserialize(new InLongMsgMixedSerializedRecord("topic", 0, inLongMsg.buildArray()), collector);
    assertEquals(1, collector.results.size());
    assertEquals(1L, collector.results.get(0).getDataflowId());
    assertEquals(4, collector.results.get(0).getRow().getArity());
    final long time = new SimpleDateFormat("yyyyMMdd").parse("20210513").getTime();
    assertEquals(new Timestamp(time), collector.results.get(0).getRow().getField(0));
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("m", "0");
    attributes.put(InLongMsgUtils.INLONGMSG_ATTR_STREAM_ID, "tid");
    attributes.put("t", "20210513");
    assertEquals(attributes, collector.results.get(0).getRow().getField(1));
    assertEquals("tianqiwan", collector.results.get(0).getRow().getField(2));
    assertEquals(29L, collector.results.get(0).getRow().getField(3));
}