Examples with TubeSourceInfo org.apache.inlong.sort.protocol.source.TubeSourceInfo used on opensource projects

Example 6 with TubeSourceInfo

use of org.apache.inlong.sort.protocol.source.TubeSourceInfo in project incubator-inlong by apache.

the class MultiTenancyInLongMsgMixedDeserializerTest method testDeserialize.

@Test
public void testDeserialize() throws Exception {
    final MultiTenancyInLongMsgMixedDeserializer deserializer = new MultiTenancyInLongMsgMixedDeserializer();
    final FieldInfo stringField = new FieldInfo("not_important", new StringFormatInfo());
    final FieldInfo longField = new FieldInfo("id", new LongFormatInfo());
    final TubeSourceInfo tubeSourceInfo = new TubeSourceInfo("topic", "address", null, new InLongMsgCsvDeserializationInfo("tid", '|', false), new FieldInfo[] { stringField, longField });
    final EmptySinkInfo sinkInfo = new EmptySinkInfo();
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, tubeSourceInfo, sinkInfo);
    deserializer.addDataFlow(dataFlowInfo);
    final InLongMsg inLongMsg = InLongMsg.newInLongMsg();
    final String attrs = "m=0&" + InLongMsgUtils.INLONGMSG_ATTR_STREAM_ID + "=tid&t=20210513";
    final String body1 = "tianqiwan|29";
    inLongMsg.addMsg(attrs, body1.getBytes());
    final TestingCollector<Record> collector = new TestingCollector<>();
    deserializer.deserialize(new InLongMsgMixedSerializedRecord("topic", 0, inLongMsg.buildArray()), collector);
    assertEquals(1, collector.results.size());
    assertEquals(1L, collector.results.get(0).getDataflowId());
    assertEquals(4, collector.results.get(0).getRow().getArity());
    final long time = new SimpleDateFormat("yyyyMMdd").parse("20210513").getTime();
    assertEquals(new Timestamp(time), collector.results.get(0).getRow().getField(0));
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("m", "0");
    attributes.put(InLongMsgUtils.INLONGMSG_ATTR_STREAM_ID, "tid");
    attributes.put("t", "20210513");
    assertEquals(attributes, collector.results.get(0).getRow().getField(1));
    assertEquals("tianqiwan", collector.results.get(0).getRow().getField(2));
    assertEquals(29L, collector.results.get(0).getRow().getField(3));
}

Example 7 with TubeSourceInfo

use of org.apache.inlong.sort.protocol.source.TubeSourceInfo in project incubator-inlong by apache.

the class MultiTenancyInLongMsgMixedDeserializer method removeDataFlow.

@Override
public void removeDataFlow(DataFlowInfo dataFlowInfo) {
    if (!isInLongMsgDataFlow(dataFlowInfo)) {
        return;
    }
    final TubeSourceInfo tubeSourceInfo = (TubeSourceInfo) dataFlowInfo.getSourceInfo();
    final InLongMsgDeserializationInfo inLongMsgDeserializationInfo = (InLongMsgDeserializationInfo) tubeSourceInfo.getDeserializationInfo();
    final InLongMsgMixedDeserializer mixedDeserializer = mixedDeserializerMap.get(tubeSourceInfo.getTopic());
    if (mixedDeserializer != null) {
        mixedDeserializer.removeDataFlow(dataFlowInfo.getId(), inLongMsgDeserializationInfo.getTid());
        if (mixedDeserializer.isEmpty()) {
            mixedDeserializerMap.remove(tubeSourceInfo.getTopic());
        }
    }
}