Examples with InLongMsgMixedFormatConverter org.apache.inlong.sort.formats.inlongmsg.InLongMsgMixedFormatConverter used on opensource projects

Example 1 with InLongMsgMixedFormatConverter

use of org.apache.inlong.sort.formats.inlongmsg.InLongMsgMixedFormatConverter in project incubator-inlong by apache.

the class MultiTenancyInLongMsgMixedDeserializer method updateDataFlow.

@Override
public void updateDataFlow(DataFlowInfo dataFlowInfo) {
    if (!isInLongMsgDataFlow(dataFlowInfo)) {
        return;
    }
    final InLongMsgDeserializationInfo inLongMsgDeserializationInfo = (InLongMsgDeserializationInfo) dataFlowInfo.getSourceInfo().getDeserializationInfo();
    Pair<AbstractInLongMsgMixedFormatDeserializer, InLongMsgMixedFormatConverter> allDeserializer = generateDeserializer(dataFlowInfo.getSourceInfo().getFields(), inLongMsgDeserializationInfo);
    final AbstractInLongMsgMixedFormatDeserializer preDeserializer = allDeserializer.getLeft();
    final InLongMsgMixedFormatConverter deserializer = allDeserializer.getRight();
    final String topic;
    if (dataFlowInfo.getSourceInfo() instanceof TubeSourceInfo) {
        topic = ((TubeSourceInfo) dataFlowInfo.getSourceInfo()).getTopic();
    } else if (dataFlowInfo.getSourceInfo() instanceof PulsarSourceInfo) {
        topic = ((PulsarSourceInfo) dataFlowInfo.getSourceInfo()).getTopic();
    } else {
        throw new UnsupportedOperationException("Unknown source type " + dataFlowInfo.getSourceInfo());
    }
    final InLongMsgMixedDeserializer mixedDeserializer = mixedDeserializerMap.computeIfAbsent(topic, key -> new InLongMsgMixedDeserializer());
    mixedDeserializer.updateDataFlow(dataFlowInfo.getId(), inLongMsgDeserializationInfo.getTid(), preDeserializer, deserializer);
}

Example 2 with InLongMsgMixedFormatConverter

use of org.apache.inlong.sort.formats.inlongmsg.InLongMsgMixedFormatConverter in project incubator-inlong by apache.

the class MultiTenancyInLongMsgMixedDeserializer method generateDeserializer.

@VisibleForTesting
Pair<AbstractInLongMsgMixedFormatDeserializer, InLongMsgMixedFormatConverter> generateDeserializer(FieldInfo[] fields, InLongMsgDeserializationInfo inLongMsgDeserializationInfo) {
    final RowFormatInfo rowFormatInfo = CommonUtils.generateDeserializationRowFormatInfo(fields);
    final AbstractInLongMsgMixedFormatDeserializer preDeserializer;
    final InLongMsgMixedFormatConverter deserializer;
    if (inLongMsgDeserializationInfo instanceof InLongMsgCsvDeserializationInfo) {
        final InLongMsgCsvDeserializationInfo csvDeserializationInfo = (InLongMsgCsvDeserializationInfo) inLongMsgDeserializationInfo;
        preDeserializer = new InLongMsgCsvMixedFormatDeserializer(StandardCharsets.UTF_8.name(), csvDeserializationInfo.getDelimiter(), null, null, csvDeserializationInfo.isDeleteHeadDelimiter(), false);
        deserializer = new InLongMsgCsvMixedFormatConverter(rowFormatInfo, DEFAULT_TIME_FIELD_NAME, DEFAULT_ATTRIBUTES_FIELD_NAME, null, false);
    } else {
        throw new UnsupportedOperationException("Not supported yet " + inLongMsgDeserializationInfo.getClass().getSimpleName());
    }
    return Pair.of(preDeserializer, deserializer);
}