Examples with InLongMsgCsvDeserializationInfo org.apache.inlong.sort.protocol.deserialization.InLongMsgCsvDeserializationInfo used on opensource projects

Example 1 with InLongMsgCsvDeserializationInfo

use of org.apache.inlong.sort.protocol.deserialization.InLongMsgCsvDeserializationInfo in project incubator-inlong by apache.

the class PushSortConfigListener method getSourceInfo.

/**
 * Get source info for DataFlowInfo
 */
@Deprecated
private SourceInfo getSourceInfo(InlongGroupInfo groupInfo, String streamId, List<StreamSinkFieldEntity> fieldList) {
    DeserializationInfo deserializationInfo = null;
    String groupId = groupInfo.getInlongGroupId();
    InlongStreamInfo streamInfo = streamService.get(groupId, streamId);
    boolean isDbType = Constant.DATA_SOURCE_DB.equals(streamInfo.getDataType());
    if (!isDbType) {
        // FILE and auto push source, the data format is TEXT or KEY-VALUE, temporarily use InLongMsgCsv
        String dataType = streamInfo.getDataType();
        if (Constant.DATA_TYPE_TEXT.equalsIgnoreCase(dataType) || Constant.DATA_TYPE_KEY_VALUE.equalsIgnoreCase(dataType)) {
            // Use the field separator from the inlong stream
            char separator = (char) Integer.parseInt(streamInfo.getDataSeparator());
            // TODO support escape
            /*Character escape = null;
                if (info.getDataEscapeChar() != null) {
                    escape = info.getDataEscapeChar().charAt(0);
                }*/
            // Whether to delete the first separator, the default is false for the time being
            deserializationInfo = new InLongMsgCsvDeserializationInfo(streamId, separator);
        }
    }
    // The number and order of the source fields must be the same as the target fields
    return null;
}

Example 2 with InLongMsgCsvDeserializationInfo

use of org.apache.inlong.sort.protocol.deserialization.InLongMsgCsvDeserializationInfo in project incubator-inlong by apache.

the class MultiTenancyInLongMsgMixedDeserializerTest method testIsInLongMsgDataFlow.

@Test
public void testIsInLongMsgDataFlow() {
    final TubeSourceInfo tubeSourceInfo = new TubeSourceInfo("topic", "address", null, new InLongMsgCsvDeserializationInfo("tid", ',', false), new FieldInfo[0]);
    final EmptySinkInfo sinkInfo = new EmptySinkInfo();
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(1L, tubeSourceInfo, sinkInfo);
    assertTrue(MultiTenancyInLongMsgMixedDeserializer.isInLongMsgDataFlow(dataFlowInfo));
    final DataFlowInfo nonInLongMsgDataFlow = new DataFlowInfo(2L, new EmptySourceInfo(), sinkInfo);
    assertFalse(MultiTenancyInLongMsgMixedDeserializer.isInLongMsgDataFlow(nonInLongMsgDataFlow));
}

Example 3 with InLongMsgCsvDeserializationInfo

use of org.apache.inlong.sort.protocol.deserialization.InLongMsgCsvDeserializationInfo in project incubator-inlong by apache.

the class TubeSubscriptionDescriptionTest method testCopyConstructor.

@Test
public void testCopyConstructor() {
    final TubeSubscriptionDescription description = new TubeSubscriptionDescription(topic, masterAddress, null);
    final TubeSourceInfo tubeSourceInfo = new TubeSourceInfo(topic, masterAddress, null, new InLongMsgCsvDeserializationInfo(tid, ',', true), new FieldInfo[0]);
    description.addDataFlow(dataFlowId, tubeSourceInfo);
    final TubeSubscriptionDescription copied = new TubeSubscriptionDescription(description);
    assertEquals(description, copied);
    copied.removeDataFlow(dataFlowId);
    assertNotEquals(description, copied);
}

Example 4 with InLongMsgCsvDeserializationInfo

use of org.apache.inlong.sort.protocol.deserialization.InLongMsgCsvDeserializationInfo in project incubator-inlong by apache.

the class MultiTenancyInLongMsgMixedDeserializer method generateDeserializer.

@VisibleForTesting
Pair<AbstractInLongMsgMixedFormatDeserializer, InLongMsgMixedFormatConverter> generateDeserializer(FieldInfo[] fields, InLongMsgDeserializationInfo inLongMsgDeserializationInfo) {
    final RowFormatInfo rowFormatInfo = CommonUtils.generateDeserializationRowFormatInfo(fields);
    final AbstractInLongMsgMixedFormatDeserializer preDeserializer;
    final InLongMsgMixedFormatConverter deserializer;
    if (inLongMsgDeserializationInfo instanceof InLongMsgCsvDeserializationInfo) {
        final InLongMsgCsvDeserializationInfo csvDeserializationInfo = (InLongMsgCsvDeserializationInfo) inLongMsgDeserializationInfo;
        preDeserializer = new InLongMsgCsvMixedFormatDeserializer(StandardCharsets.UTF_8.name(), csvDeserializationInfo.getDelimiter(), null, null, csvDeserializationInfo.isDeleteHeadDelimiter(), false);
        deserializer = new InLongMsgCsvMixedFormatConverter(rowFormatInfo, DEFAULT_TIME_FIELD_NAME, DEFAULT_ATTRIBUTES_FIELD_NAME, null, false);
    } else {
        throw new UnsupportedOperationException("Not supported yet " + inLongMsgDeserializationInfo.getClass().getSimpleName());
    }
    return Pair.of(preDeserializer, deserializer);
}

Example 5 with InLongMsgCsvDeserializationInfo

use of org.apache.inlong.sort.protocol.deserialization.InLongMsgCsvDeserializationInfo in project incubator-inlong by apache.

the class MultiTenancyDeserializer method generateDeserializer.

@VisibleForTesting
Deserializer<SerializedRecord, Record> generateDeserializer(FieldInfo[] fields, DeserializationInfo deserializationInfo) {
    final RowFormatInfo rowFormatInfo = CommonUtils.generateDeserializationRowFormatInfo(fields);
    final Deserializer<SerializedRecord, Record> deserializer;
    if (deserializationInfo instanceof InLongMsgCsvDeserializationInfo) {
        InLongMsgCsvDeserializationInfo inLongMsgCsvDeserializationInfo = (InLongMsgCsvDeserializationInfo) deserializationInfo;
        InLongMsgCsvFormatDeserializer inLongMsgCsvFormatDeserializer = new InLongMsgCsvFormatDeserializer(rowFormatInfo, DEFAULT_TIME_FIELD_NAME, DEFAULT_ATTRIBUTES_FIELD_NAME, TableFormatConstants.DEFAULT_CHARSET, inLongMsgCsvDeserializationInfo.getDelimiter(), null, null, null, inLongMsgCsvDeserializationInfo.isDeleteHeadDelimiter(), TableFormatConstants.DEFAULT_IGNORE_ERRORS);
        deserializer = new InLongMsgDeserializer(inLongMsgCsvFormatDeserializer);
    } else {
        // TODO, support more formats here
        throw new UnsupportedOperationException("Not supported yet " + deserializationInfo.getClass().getSimpleName());
    }
    return deserializer;
}