Examples with CsvDeserializationInfo org.apache.inlong.sort.protocol.deserialization.CsvDeserializationInfo used on opensource projects

Example 1 with CsvDeserializationInfo

use of org.apache.inlong.sort.protocol.deserialization.CsvDeserializationInfo in project incubator-inlong by apache.

the class SerializationUtils method deserializeForKafka.

/**
 * Get deserialization info for Kafka
 */
private static DeserializationInfo deserializeForKafka(KafkaSourceResponse source, InlongStreamInfo stream) {
    String serializationType = source.getSerializationType();
    DataTypeEnum dataType = DataTypeEnum.forName(serializationType);
    switch(dataType) {
        case CSV:
            char separator = stream.getDataSeparator().toCharArray()[0];
            return new CsvDeserializationInfo(separator);
        case AVRO:
            return new AvroDeserializationInfo();
        case JSON:
            return new JsonDeserializationInfo();
        case CANAL:
            return new CanalDeserializationInfo(source.getDatabasePattern(), source.getTablePattern(), source.isIgnoreParseErrors(), source.getTimestampFormatStandard(), true);
        default:
            throw new IllegalArgumentException(String.format("Unsupported serializationType for Kafka source: %s", serializationType));
    }
}

Example 2 with CsvDeserializationInfo

use of org.apache.inlong.sort.protocol.deserialization.CsvDeserializationInfo in project incubator-inlong by apache.

the class DeserializationInfoTest method testCsvDeserializationInfo.

@Test
public void testCsvDeserializationInfo() throws IOException {
    ObjectMapper objectMapper = new ObjectMapper();
    String str = "{\"type\":\"csv\",\"splitter\":\"\\u0026\"}";
    CsvDeserializationInfo deserializationInfo = objectMapper.readValue(str.getBytes(), CsvDeserializationInfo.class);
    assertEquals('&', deserializationInfo.getSplitter());
}

Example 3 with CsvDeserializationInfo

use of org.apache.inlong.sort.protocol.deserialization.CsvDeserializationInfo in project incubator-inlong by apache.

the class PulsarTestMetaManagerUtil method prepareDataFlowInfo.

@Override
public DataFlowInfo prepareDataFlowInfo(long dataFlowId, String... args) {
    FieldInfo[] pulsarFields = new FieldInfo[] { new FieldInfo("f1", StringFormatInfo.INSTANCE), new FieldInfo("f2", StringFormatInfo.INSTANCE) };
    Map<String, Object> config = new HashMap<>();
    config.put("consumer.bootstrap-mode", "earliest");
    return new DataFlowInfo(dataFlowId, new PulsarSourceInfo(args[0], args[1], args[2], args[3], new CsvDeserializationInfo(','), pulsarFields, null), new HiveSinkInfo(new FieldInfo[0], "testServerJdbcUrl", "testDatabaseName", "testTableName", "testUsername", "testPassword", "testDataPath", new HivePartitionInfo[0], new TextFileFormat(',')), config);
}

Example 4 with CsvDeserializationInfo

use of org.apache.inlong.sort.protocol.deserialization.CsvDeserializationInfo in project incubator-inlong by apache.

the class SerializationUtils method deserializeForFile.

/**
 * Get deserialization info for File
 */
private static DeserializationInfo deserializeForFile(SourceResponse sourceResponse, InlongStreamInfo streamInfo) {
    String serializationType = sourceResponse.getSerializationType();
    DataTypeEnum dataType = DataTypeEnum.forName(serializationType);
    switch(dataType) {
        case CSV:
            char separator = streamInfo.getDataSeparator().toCharArray()[0];
            return new CsvDeserializationInfo(separator);
        case AVRO:
            return new AvroDeserializationInfo();
        case JSON:
            return new JsonDeserializationInfo();
        default:
            throw new IllegalArgumentException(String.format("Unsupported type for File source:%s", serializationType));
    }
}