Examples with InLongMsg org.apache.inlong.common.msg.InLongMsg used on opensource projects

Example 1 with InLongMsg

use of org.apache.inlong.common.msg.InLongMsg in project incubator-inlong by apache.

the class DeserializationFunction method processElement.

@Override
public void processElement(SerializedRecord value, ProcessFunction<SerializedRecord, Row>.Context ctx, Collector<Row> out) throws Exception {
    InLongMsg inLongMsg = InLongMsg.parseFrom(value.getData());
    for (String attr : inLongMsg.getAttrs()) {
        Iterator<byte[]> iterator = inLongMsg.getIterator(attr);
        if (iterator == null) {
            continue;
        }
        while (iterator.hasNext()) {
            byte[] bodyBytes = iterator.next();
            if (bodyBytes == null || bodyBytes.length == 0) {
                continue;
            }
            deserializationSchema.deserialize(bodyBytes, new CallbackCollector<>(inputRow -> {
                if (appendAttributes) {
                    inputRow = Row.join(Row.of(new HashMap<>()), inputRow);
                }
                out.collect(fieldMappingTransformer.transform(inputRow, value.getTimestampMillis()));
            }));
        }
    }
}

Example 2 with InLongMsg

use of org.apache.inlong.common.msg.InLongMsg in project incubator-inlong by apache.

the class InLongMsgCsvFormatDeserializerTest method testUnmatchedFields1.

@Test
public void testUnmatchedFields1() throws Exception {
    final InLongMsgCsvFormatDeserializer deserializer = new InLongMsgCsvFormatDeserializer(TEST_ROW_INFO);
    InLongMsg inLongMsg = InLongMsg.newInLongMsg();
    String attrs = "m=0&" + INLONGMSG_ATTR_STREAM_ID + "=testInterfaceId&t=20200322&__addcol1__=1&__addcol2__=2";
    String body1 = "123,field11,field12";
    String body2 = "123,field21,field22,field23,field24";
    inLongMsg.addMsg(attrs, body1.getBytes());
    inLongMsg.addMsg(attrs, body2.getBytes());
    Map<String, String> expectedAttributes = new HashMap<>();
    expectedAttributes.put("m", "0");
    expectedAttributes.put(INLONGMSG_ATTR_STREAM_ID, "testInterfaceId");
    expectedAttributes.put("t", "20200322");
    expectedAttributes.put("__addcol1__", "1");
    expectedAttributes.put("__addcol2__", "2");
    Row expectedRow1 = Row.of(Timestamp.valueOf("2020-03-22 00:00:00"), expectedAttributes, 1, 2, 123, "field11", "field12", null);
    Row expectedRow2 = Row.of(Timestamp.valueOf("2020-03-22 00:00:00"), expectedAttributes, 1, 2, 123, "field21", "field22", "field23");
    testRowDeserialization(deserializer, inLongMsg.buildArray(), Arrays.asList(expectedRow1, expectedRow2));
}

Example 3 with InLongMsg

use of org.apache.inlong.common.msg.InLongMsg in project incubator-inlong by apache.

the class InLongMsgCsvFormatDeserializerTest method testUnmatchedFields2.

@Test
public void testUnmatchedFields2() throws Exception {
    final InLongMsgCsvFormatDeserializer deserializer = new InLongMsgCsvFormatDeserializer(TEST_ROW_INFO);
    InLongMsg inLongMsg = InLongMsg.newInLongMsg();
    String attrs = "m=0&" + INLONGMSG_ATTR_STREAM_ID + "=testInterfaceId&t=20200322&__addcol1__=1&" + "__addcol2__=2&__addcol3__=3&__addcol4__=4&__addcol5__=5&__addcol6__=6&__addcol7__=7";
    String body = "field11,field12";
    inLongMsg.addMsg(attrs, body.getBytes());
    Map<String, String> expectedAttributes = new HashMap<>();
    expectedAttributes.put("m", "0");
    expectedAttributes.put(INLONGMSG_ATTR_STREAM_ID, "testInterfaceId");
    expectedAttributes.put("t", "20200322");
    expectedAttributes.put("__addcol1__", "1");
    expectedAttributes.put("__addcol2__", "2");
    expectedAttributes.put("__addcol3__", "3");
    expectedAttributes.put("__addcol4__", "4");
    expectedAttributes.put("__addcol5__", "5");
    expectedAttributes.put("__addcol6__", "6");
    expectedAttributes.put("__addcol7__", "7");
    Row expectedRow = Row.of(Timestamp.valueOf("2020-03-22 00:00:00"), expectedAttributes, 1, 2, 3, "4", "5", "6");
    testRowDeserialization(deserializer, inLongMsg.buildArray(), Collections.singletonList(expectedRow));
}

Example 4 with InLongMsg

use of org.apache.inlong.common.msg.InLongMsg in project incubator-inlong by apache.

the class InLongMsgCsvFormatDeserializerTest method testIgnoreBodyErrors.

@Test
public void testIgnoreBodyErrors() throws Exception {
    final InLongMsgCsvFormatDeserializer deserializer = new InLongMsgCsvFormatDeserializer(TEST_ROW_INFO, DEFAULT_TIME_FIELD_NAME, DEFAULT_ATTRIBUTES_FIELD_NAME, Charset.defaultCharset().name(), TableFormatConstants.DEFAULT_DELIMITER, null, null, null, true, true);
    InLongMsg inLongMsg = InLongMsg.newInLongMsg();
    String attrs = "m=0&" + INLONGMSG_ATTR_STREAM_ID + "=testInterfaceId&t=20200322&__addcol1__=1&__addcol2__=2";
    String body1 = "aaa,field11,field12,field13";
    String body2 = "123,field21,field22,field23";
    inLongMsg.addMsg(attrs, body1.getBytes());
    inLongMsg.addMsg(attrs, body2.getBytes());
    Map<String, String> expectedAttributes = new HashMap<>();
    expectedAttributes.put("m", "0");
    expectedAttributes.put(INLONGMSG_ATTR_STREAM_ID, "testInterfaceId");
    expectedAttributes.put("t", "20200322");
    expectedAttributes.put("__addcol1__", "1");
    expectedAttributes.put("__addcol2__", "2");
    Row expectedRow2 = Row.of(Timestamp.valueOf("2020-03-22 00:00:00"), expectedAttributes, 1, 2, 123, "field21", "field22", "field23");
    testRowDeserialization(deserializer, inLongMsg.buildArray(), Collections.singletonList(expectedRow2));
}

Example 5 with InLongMsg

use of org.apache.inlong.common.msg.InLongMsg in project incubator-inlong by apache.

the class InLongMsgCsvFormatDeserializerTest method testNormal.

@Test
public void testNormal() throws Exception {
    final InLongMsgCsvFormatDeserializer deserializer = new InLongMsgCsvFormatDeserializer(TEST_ROW_INFO);
    InLongMsg inLongMsg = InLongMsg.newInLongMsg();
    String attrs = "m=0&" + INLONGMSG_ATTR_STREAM_ID + "=testInterfaceId&t=20200322&__addcol1__=1&__addcol2__=2";
    String body1 = "123,field11,field12,field13";
    String body2 = "123,field21,field22,field23";
    inLongMsg.addMsg(attrs, body1.getBytes());
    inLongMsg.addMsg(attrs, body2.getBytes());
    Map<String, String> expectedAttributes = new HashMap<>();
    expectedAttributes.put("m", "0");
    expectedAttributes.put(INLONGMSG_ATTR_STREAM_ID, "testInterfaceId");
    expectedAttributes.put("t", "20200322");
    expectedAttributes.put("__addcol1__", "1");
    expectedAttributes.put("__addcol2__", "2");
    Row expectedRow1 = Row.of(Timestamp.valueOf("2020-03-22 00:00:00"), expectedAttributes, 1, 2, 123, "field11", "field12", "field13");
    Row expectedRow2 = Row.of(Timestamp.valueOf("2020-03-22 00:00:00"), expectedAttributes, 1, 2, 123, "field21", "field22", "field23");
    testRowDeserialization(deserializer, inLongMsg.buildArray(), Arrays.asList(expectedRow1, expectedRow2));
}