Examples with TubeSourceInfo org.apache.inlong.sort.protocol.source.TubeSourceInfo used on opensource projects

Example 1 with TubeSourceInfo

use of org.apache.inlong.sort.protocol.source.TubeSourceInfo in project incubator-inlong by apache.

the class MultiTenancyInLongMsgMixedDeserializerTest method testIsInLongMsgDataFlow.

@Test
public void testIsInLongMsgDataFlow() {
    final TubeSourceInfo tubeSourceInfo = new TubeSourceInfo("topic", "address", null, new InLongMsgCsvDeserializationInfo("tid", ',', false), new FieldInfo[0]);
    final EmptySinkInfo sinkInfo = new EmptySinkInfo();
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, tubeSourceInfo, sinkInfo);
    assertTrue(MultiTenancyInLongMsgMixedDeserializer.isInLongMsgDataFlow(dataFlowInfo));
    final DataFlowInfo nonInLongMsgDataFlow = new DataFlowInfo(2L, new EmptySourceInfo(), sinkInfo);
    assertFalse(MultiTenancyInLongMsgMixedDeserializer.isInLongMsgDataFlow(nonInLongMsgDataFlow));
}

Example 2 with TubeSourceInfo

use of org.apache.inlong.sort.protocol.source.TubeSourceInfo in project incubator-inlong by apache.

the class TubeSubscriptionDescriptionTest method testCopyConstructor.

@Test
public void testCopyConstructor() {
    final TubeSubscriptionDescription description = new TubeSubscriptionDescription(topic, masterAddress, null);
    final TubeSourceInfo tubeSourceInfo = new TubeSourceInfo(topic, masterAddress, null, new InLongMsgCsvDeserializationInfo(tid, ',', true), new FieldInfo[0]);
    description.addDataFlow(dataFlowId, tubeSourceInfo);
    final TubeSubscriptionDescription copied = new TubeSubscriptionDescription(description);
    assertEquals(description, copied);
    copied.removeDataFlow(dataFlowId);
    assertNotEquals(description, copied);
}

Example 3 with TubeSourceInfo

use of org.apache.inlong.sort.protocol.source.TubeSourceInfo in project incubator-inlong by apache.

the class MultiTenancyInLongMsgMixedDeserializer method updateDataFlow.

@Override
public void updateDataFlow(DataFlowInfo dataFlowInfo) {
    if (!isInLongMsgDataFlow(dataFlowInfo)) {
        return;
    }
    final InLongMsgDeserializationInfo inLongMsgDeserializationInfo = (InLongMsgDeserializationInfo) dataFlowInfo.getSourceInfo().getDeserializationInfo();
    Pair<AbstractInLongMsgMixedFormatDeserializer, InLongMsgMixedFormatConverter> allDeserializer = generateDeserializer(dataFlowInfo.getSourceInfo().getFields(), inLongMsgDeserializationInfo);
    final AbstractInLongMsgMixedFormatDeserializer preDeserializer = allDeserializer.getLeft();
    final InLongMsgMixedFormatConverter deserializer = allDeserializer.getRight();
    final String topic;
    if (dataFlowInfo.getSourceInfo() instanceof TubeSourceInfo) {
        topic = ((TubeSourceInfo) dataFlowInfo.getSourceInfo()).getTopic();
    } else if (dataFlowInfo.getSourceInfo() instanceof PulsarSourceInfo) {
        topic = ((PulsarSourceInfo) dataFlowInfo.getSourceInfo()).getTopic();
    } else {
        throw new UnsupportedOperationException("Unknown source type " + dataFlowInfo.getSourceInfo());
    }
    final InLongMsgMixedDeserializer mixedDeserializer = mixedDeserializerMap.computeIfAbsent(topic, key -> new InLongMsgMixedDeserializer());
    mixedDeserializer.updateDataFlow(dataFlowInfo.getId(), inLongMsgDeserializationInfo.getTid(), preDeserializer, deserializer);
}

Example 4 with TubeSourceInfo

use of org.apache.inlong.sort.protocol.source.TubeSourceInfo in project incubator-inlong by apache.

the class DeserializationInfoTest method testToJson.

@Test
public void testToJson() throws JsonProcessingException {
    DataFlowInfo dataFlowInfo = new DataFlowInfo(1, new TubeSourceInfo("topic" + System.currentTimeMillis(), "ma", "cg", new InLongMsgCsvDeserializationInfo("tid", ','), new FieldInfo[0]), new ClickHouseSinkInfo("url", "dn", "tn", "un", "pw", false, PartitionStrategy.HASH, "pk", new FieldInfo[0], new String[0], 100, 100, 100));
    ObjectMapper objectMapper = new ObjectMapper();
    System.out.println(objectMapper.writeValueAsString(dataFlowInfo));
}

Example 5 with TubeSourceInfo

use of org.apache.inlong.sort.protocol.source.TubeSourceInfo in project incubator-inlong by apache.

the class TubeSubscriptionDescriptionTest method testAddAndRemoveDataFlow.

@Test
public void testAddAndRemoveDataFlow() {
    final TubeSubscriptionDescription description = new TubeSubscriptionDescription(topic, masterAddress, null);
    final TubeSourceInfo tubeSourceInfo1 = new TubeSourceInfo(topic, masterAddress, null, new InLongMsgCsvDeserializationInfo(tid, ',', true), new FieldInfo[0]);
    description.addDataFlow(dataFlowId, tubeSourceInfo1);
    final String tid2 = "10002";
    final String consumerGroup = "consumerGroup";
    final TubeSourceInfo tubeSourceInfo2 = new TubeSourceInfo(topic, masterAddress, consumerGroup, new InLongMsgCsvDeserializationInfo(tid2, ',', true), new FieldInfo[0]);
    final long dataFlowId2 = 10086L;
    description.addDataFlow(dataFlowId2, tubeSourceInfo2);
    final List<String> expectedTids = new ArrayList<>();
    expectedTids.add(tid);
    expectedTids.add(tid2);
    Collections.sort(expectedTids);
    final List<String> tids = description.getTids();
    Collections.sort(tids);
    assertEquals(expectedTids, tids);
    description.removeDataFlow(dataFlowId);
    assertEquals(Collections.singletonList(tid2), description.getTids());
    description.removeDataFlow(dataFlowId2);
    try {
        description.getTids();
        fail("Should throw exception when get tids from an empty description");
    } catch (IllegalStateException expected) {
    }
}