Search in sources :

Example 1 with DateFormatInfo

use of org.apache.inlong.sort.formats.common.DateFormatInfo in project incubator-inlong by apache.

the class TableFormatUtils method deriveFormatInfo.

/**
 * Derive the format information for the given type.
 *
 * @param logicalType The type whose format is derived.
 * @return The format information for the given type.
 */
public static FormatInfo deriveFormatInfo(LogicalType logicalType) {
    if (logicalType instanceof VarCharType) {
        return StringFormatInfo.INSTANCE;
    } else if (logicalType instanceof BooleanType) {
        return BooleanFormatInfo.INSTANCE;
    } else if (logicalType instanceof TinyIntType) {
        return ByteFormatInfo.INSTANCE;
    } else if (logicalType instanceof SmallIntType) {
        return ShortFormatInfo.INSTANCE;
    } else if (logicalType instanceof IntType) {
        return IntFormatInfo.INSTANCE;
    } else if (logicalType instanceof BigIntType) {
        return LongFormatInfo.INSTANCE;
    } else if (logicalType instanceof FloatType) {
        return FloatFormatInfo.INSTANCE;
    } else if (logicalType instanceof DoubleType) {
        return DoubleFormatInfo.INSTANCE;
    } else if (logicalType instanceof DecimalType) {
        return DecimalFormatInfo.INSTANCE;
    } else if (logicalType instanceof DateType) {
        return new DateFormatInfo();
    } else if (logicalType instanceof TimeType) {
        return new TimeFormatInfo();
    } else if (logicalType instanceof TimestampType) {
        return new TimestampFormatInfo();
    } else if (logicalType instanceof LocalZonedTimestampType) {
        return new LocalZonedTimestampFormatInfo();
    } else if (logicalType instanceof ArrayType) {
        ArrayType arrayType = (ArrayType) logicalType;
        LogicalType elementType = arrayType.getElementType();
        FormatInfo elementFormatInfo = deriveFormatInfo(elementType);
        return new ArrayFormatInfo(elementFormatInfo);
    } else if (logicalType instanceof MapType) {
        MapType mapType = (MapType) logicalType;
        LogicalType keyType = mapType.getKeyType();
        LogicalType valueType = mapType.getValueType();
        FormatInfo keyFormatInfo = deriveFormatInfo(keyType);
        FormatInfo valueFormatInfo = deriveFormatInfo(valueType);
        return new MapFormatInfo(keyFormatInfo, valueFormatInfo);
    } else if (logicalType instanceof RowType) {
        RowType rowType = (RowType) logicalType;
        List<RowType.RowField> rowFields = rowType.getFields();
        String[] fieldNames = new String[rowFields.size()];
        FormatInfo[] fieldFormatInfos = new FormatInfo[rowFields.size()];
        for (int i = 0; i < rowFields.size(); ++i) {
            RowType.RowField rowField = rowFields.get(i);
            fieldNames[i] = rowField.getName();
            fieldFormatInfos[i] = deriveFormatInfo(rowField.getType());
        }
        return new RowFormatInfo(fieldNames, fieldFormatInfos);
    } else if (logicalType instanceof BinaryType) {
        return BinaryFormatInfo.INSTANCE;
    } else if (logicalType instanceof NullType) {
        return NullFormatInfo.INSTANCE;
    } else {
        throw new UnsupportedOperationException();
    }
}
Also used : MapFormatInfo(org.apache.inlong.sort.formats.common.MapFormatInfo) LocalZonedTimestampFormatInfo(org.apache.inlong.sort.formats.common.LocalZonedTimestampFormatInfo) TimestampFormatInfo(org.apache.inlong.sort.formats.common.TimestampFormatInfo) BigIntType(org.apache.flink.table.types.logical.BigIntType) LogicalType(org.apache.flink.table.types.logical.LogicalType) RowType(org.apache.flink.table.types.logical.RowType) ArrayFormatInfo(org.apache.inlong.sort.formats.common.ArrayFormatInfo) MapType(org.apache.flink.table.types.logical.MapType) TinyIntType(org.apache.flink.table.types.logical.TinyIntType) IntType(org.apache.flink.table.types.logical.IntType) BigIntType(org.apache.flink.table.types.logical.BigIntType) SmallIntType(org.apache.flink.table.types.logical.SmallIntType) FloatType(org.apache.flink.table.types.logical.FloatType) TimeType(org.apache.flink.table.types.logical.TimeType) ArrayType(org.apache.flink.table.types.logical.ArrayType) DateFormatInfo(org.apache.inlong.sort.formats.common.DateFormatInfo) LocalZonedTimestampFormatInfo(org.apache.inlong.sort.formats.common.LocalZonedTimestampFormatInfo) LocalZonedTimestampType(org.apache.flink.table.types.logical.LocalZonedTimestampType) TimestampType(org.apache.flink.table.types.logical.TimestampType) List(java.util.List) VarCharType(org.apache.flink.table.types.logical.VarCharType) DateType(org.apache.flink.table.types.logical.DateType) BinaryType(org.apache.flink.table.types.logical.BinaryType) BooleanType(org.apache.flink.table.types.logical.BooleanType) LocalZonedTimestampType(org.apache.flink.table.types.logical.LocalZonedTimestampType) TimeFormatInfo(org.apache.inlong.sort.formats.common.TimeFormatInfo) TinyIntType(org.apache.flink.table.types.logical.TinyIntType) SmallIntType(org.apache.flink.table.types.logical.SmallIntType) DoubleType(org.apache.flink.table.types.logical.DoubleType) RowFormatInfo(org.apache.inlong.sort.formats.common.RowFormatInfo) DecimalType(org.apache.flink.table.types.logical.DecimalType) NullType(org.apache.flink.table.types.logical.NullType) FormatInfo(org.apache.inlong.sort.formats.common.FormatInfo) BasicFormatInfo(org.apache.inlong.sort.formats.common.BasicFormatInfo) DoubleFormatInfo(org.apache.inlong.sort.formats.common.DoubleFormatInfo) BinaryFormatInfo(org.apache.inlong.sort.formats.common.BinaryFormatInfo) ArrayFormatInfo(org.apache.inlong.sort.formats.common.ArrayFormatInfo) BooleanFormatInfo(org.apache.inlong.sort.formats.common.BooleanFormatInfo) NullFormatInfo(org.apache.inlong.sort.formats.common.NullFormatInfo) IntFormatInfo(org.apache.inlong.sort.formats.common.IntFormatInfo) LocalZonedTimestampFormatInfo(org.apache.inlong.sort.formats.common.LocalZonedTimestampFormatInfo) DecimalFormatInfo(org.apache.inlong.sort.formats.common.DecimalFormatInfo) TimestampFormatInfo(org.apache.inlong.sort.formats.common.TimestampFormatInfo) ShortFormatInfo(org.apache.inlong.sort.formats.common.ShortFormatInfo) StringFormatInfo(org.apache.inlong.sort.formats.common.StringFormatInfo) FloatFormatInfo(org.apache.inlong.sort.formats.common.FloatFormatInfo) ByteFormatInfo(org.apache.inlong.sort.formats.common.ByteFormatInfo) TimeFormatInfo(org.apache.inlong.sort.formats.common.TimeFormatInfo) RowFormatInfo(org.apache.inlong.sort.formats.common.RowFormatInfo) MapFormatInfo(org.apache.inlong.sort.formats.common.MapFormatInfo) LongFormatInfo(org.apache.inlong.sort.formats.common.LongFormatInfo) DateFormatInfo(org.apache.inlong.sort.formats.common.DateFormatInfo)

Example 2 with DateFormatInfo

use of org.apache.inlong.sort.formats.common.DateFormatInfo in project incubator-inlong by apache.

the class CustomDateFormatDeserializationSchemaWrapperTest method testFromStringToDateAndTime.

@Test
public void testFromStringToDateAndTime() throws IOException, ClassNotFoundException, ParseException {
    FieldInfo[] fieldInfos = new FieldInfo[] { new FieldInfo("f1", new DateFormatInfo()), new FieldInfo("f2", new TimeFormatInfo()), new FieldInfo("f3", new TimestampFormatInfo()), new FieldInfo("f4", StringFormatInfo.INSTANCE) };
    DeserializationSchema<Row> stringSchema = DeserializationSchemaFactory.build(fieldInfos, null);
    CustomDateFormatDeserializationSchemaWrapper schemaWrapper = new CustomDateFormatDeserializationSchemaWrapper(stringSchema, extractFormatInfos(fieldInfos));
    Row testRow = Row.of("2022-02-15", "15:52:30", "2022-02-15 15:52:30", "don't convert");
    Row resultRow = schemaWrapper.fromStringToDateAndTime(testRow);
    assertTrue(resultRow.getField(0) instanceof Date);
    assertTrue(resultRow.getField(1) instanceof Time);
    assertTrue(resultRow.getField(2) instanceof Timestamp);
    final Row expectedRow = Row.of(Date.valueOf("2022-02-15"), Time.valueOf("15:52:30"), Timestamp.valueOf("2022-02-15 15:52:30"), "don't convert");
    assertEquals(expectedRow, resultRow);
}
Also used : DateFormatInfo(org.apache.inlong.sort.formats.common.DateFormatInfo) TimestampFormatInfo(org.apache.inlong.sort.formats.common.TimestampFormatInfo) TimeFormatInfo(org.apache.inlong.sort.formats.common.TimeFormatInfo) Time(java.sql.Time) Row(org.apache.flink.types.Row) Timestamp(java.sql.Timestamp) FieldInfo(org.apache.inlong.sort.protocol.FieldInfo) Date(java.sql.Date) Test(org.junit.Test)

Example 3 with DateFormatInfo

use of org.apache.inlong.sort.formats.common.DateFormatInfo in project incubator-inlong by apache.

the class CsvDeserializationSchemaTest method testNullLiteral.

@Test
public void testNullLiteral() throws Exception {
    String nullLiteral = "n/a";
    Consumer<CsvDeserializationSchema.Builder> config = builder -> builder.setNullLiteral(nullLiteral);
    testBasicDeserialization(config, StringFormatInfo.INSTANCE, null, nullLiteral);
    testBasicDeserialization(config, BooleanFormatInfo.INSTANCE, null, nullLiteral);
    testBasicDeserialization(config, ByteFormatInfo.INSTANCE, null, nullLiteral);
    testBasicDeserialization(config, ShortFormatInfo.INSTANCE, null, nullLiteral);
    testBasicDeserialization(config, IntFormatInfo.INSTANCE, null, nullLiteral);
    testBasicDeserialization(config, LongFormatInfo.INSTANCE, null, nullLiteral);
    testBasicDeserialization(config, FloatFormatInfo.INSTANCE, null, nullLiteral);
    testBasicDeserialization(config, DoubleFormatInfo.INSTANCE, null, nullLiteral);
    testBasicDeserialization(config, DecimalFormatInfo.INSTANCE, null, nullLiteral);
    testBasicDeserialization(config, new DateFormatInfo("dd/MM/yyyy"), null, nullLiteral);
    testBasicDeserialization(config, new TimeFormatInfo("ss/mm/hh"), null, nullLiteral);
    testBasicDeserialization(config, new TimestampFormatInfo("dd/MM/yyyy hh:mm:ss"), null, nullLiteral);
}
Also used : Time(java.sql.Time) FormatInfo(org.apache.inlong.sort.formats.common.FormatInfo) BasicFormatInfo(org.apache.inlong.sort.formats.common.BasicFormatInfo) TimestampFormatInfo(org.apache.inlong.sort.formats.common.TimestampFormatInfo) DoubleFormatInfo(org.apache.inlong.sort.formats.common.DoubleFormatInfo) ShortFormatInfo(org.apache.inlong.sort.formats.common.ShortFormatInfo) StringFormatInfo(org.apache.inlong.sort.formats.common.StringFormatInfo) BigDecimal(java.math.BigDecimal) BooleanFormatInfo(org.apache.inlong.sort.formats.common.BooleanFormatInfo) FloatFormatInfo(org.apache.inlong.sort.formats.common.FloatFormatInfo) ByteFormatInfo(org.apache.inlong.sort.formats.common.ByteFormatInfo) TimeFormatInfo(org.apache.inlong.sort.formats.common.TimeFormatInfo) Timestamp(java.sql.Timestamp) RowFormatInfo(org.apache.inlong.sort.formats.common.RowFormatInfo) IOException(java.io.IOException) Test(org.junit.Test) StandardCharsets(java.nio.charset.StandardCharsets) Date(java.sql.Date) IntFormatInfo(org.apache.inlong.sort.formats.common.IntFormatInfo) Consumer(java.util.function.Consumer) LongFormatInfo(org.apache.inlong.sort.formats.common.LongFormatInfo) Row(org.apache.flink.types.Row) DecimalFormatInfo(org.apache.inlong.sort.formats.common.DecimalFormatInfo) Assert.assertEquals(org.junit.Assert.assertEquals) DateFormatInfo(org.apache.inlong.sort.formats.common.DateFormatInfo) DateFormatInfo(org.apache.inlong.sort.formats.common.DateFormatInfo) TimestampFormatInfo(org.apache.inlong.sort.formats.common.TimestampFormatInfo) TimeFormatInfo(org.apache.inlong.sort.formats.common.TimeFormatInfo) Test(org.junit.Test)

Example 4 with DateFormatInfo

use of org.apache.inlong.sort.formats.common.DateFormatInfo in project incubator-inlong by apache.

the class CsvSerializationSchemaTest method testNormal.

@Test
public void testNormal() {
    Consumer<CsvSerializationSchema.Builder> config = builder -> {
    };
    testBasicSerialization(config, StringFormatInfo.INSTANCE, "hello", "hello");
    testBasicSerialization(config, BooleanFormatInfo.INSTANCE, true, "true");
    testBasicSerialization(config, ByteFormatInfo.INSTANCE, (byte) 124, "124");
    testBasicSerialization(config, ShortFormatInfo.INSTANCE, (short) 10000, "10000");
    testBasicSerialization(config, IntFormatInfo.INSTANCE, 1234567, "1234567");
    testBasicSerialization(config, LongFormatInfo.INSTANCE, 12345678910L, "12345678910");
    testBasicSerialization(config, FloatFormatInfo.INSTANCE, 0.33333334f, "0.33333334");
    testBasicSerialization(config, DoubleFormatInfo.INSTANCE, 0.33333333332, "0.33333333332");
    testBasicSerialization(config, DecimalFormatInfo.INSTANCE, new BigDecimal("1234.0000000000000000000000001"), "1234.0000000000000000000000001");
    testBasicSerialization(config, new DateFormatInfo("dd/MM/yyyy"), Date.valueOf("2020-03-22"), "22/03/2020");
    testBasicSerialization(config, new TimeFormatInfo("ss/mm/hh"), Time.valueOf("11:12:13"), "13/12/11");
    testBasicSerialization(config, new TimestampFormatInfo("dd/MM/yyyy hh:mm:ss"), Timestamp.valueOf("2020-03-22 11:12:13"), "22/03/2020 11:12:13");
}
Also used : Time(java.sql.Time) FormatInfo(org.apache.inlong.sort.formats.common.FormatInfo) BasicFormatInfo(org.apache.inlong.sort.formats.common.BasicFormatInfo) TimestampFormatInfo(org.apache.inlong.sort.formats.common.TimestampFormatInfo) DoubleFormatInfo(org.apache.inlong.sort.formats.common.DoubleFormatInfo) ShortFormatInfo(org.apache.inlong.sort.formats.common.ShortFormatInfo) StringFormatInfo(org.apache.inlong.sort.formats.common.StringFormatInfo) BigDecimal(java.math.BigDecimal) Assert.assertArrayEquals(org.junit.Assert.assertArrayEquals) BooleanFormatInfo(org.apache.inlong.sort.formats.common.BooleanFormatInfo) FloatFormatInfo(org.apache.inlong.sort.formats.common.FloatFormatInfo) ByteFormatInfo(org.apache.inlong.sort.formats.common.ByteFormatInfo) TimeFormatInfo(org.apache.inlong.sort.formats.common.TimeFormatInfo) Timestamp(java.sql.Timestamp) RowFormatInfo(org.apache.inlong.sort.formats.common.RowFormatInfo) Test(org.junit.Test) StandardCharsets(java.nio.charset.StandardCharsets) Date(java.sql.Date) IntFormatInfo(org.apache.inlong.sort.formats.common.IntFormatInfo) Consumer(java.util.function.Consumer) LongFormatInfo(org.apache.inlong.sort.formats.common.LongFormatInfo) Row(org.apache.flink.types.Row) DecimalFormatInfo(org.apache.inlong.sort.formats.common.DecimalFormatInfo) Assert.assertEquals(org.junit.Assert.assertEquals) DateFormatInfo(org.apache.inlong.sort.formats.common.DateFormatInfo) DateFormatInfo(org.apache.inlong.sort.formats.common.DateFormatInfo) TimestampFormatInfo(org.apache.inlong.sort.formats.common.TimestampFormatInfo) TimeFormatInfo(org.apache.inlong.sort.formats.common.TimeFormatInfo) BigDecimal(java.math.BigDecimal) Test(org.junit.Test)

Example 5 with DateFormatInfo

use of org.apache.inlong.sort.formats.common.DateFormatInfo in project incubator-inlong by apache.

the class CsvSerializationSchemaTest method testNullLiteral.

@Test
public void testNullLiteral() {
    String nullLiteral = "n/a";
    Consumer<CsvSerializationSchema.Builder> config = builder -> builder.setNullLiteral(nullLiteral);
    testBasicSerialization(config, StringFormatInfo.INSTANCE, null, nullLiteral);
    testBasicSerialization(config, BooleanFormatInfo.INSTANCE, null, nullLiteral);
    testBasicSerialization(config, ByteFormatInfo.INSTANCE, null, nullLiteral);
    testBasicSerialization(config, ShortFormatInfo.INSTANCE, null, nullLiteral);
    testBasicSerialization(config, IntFormatInfo.INSTANCE, null, nullLiteral);
    testBasicSerialization(config, LongFormatInfo.INSTANCE, null, nullLiteral);
    testBasicSerialization(config, FloatFormatInfo.INSTANCE, null, nullLiteral);
    testBasicSerialization(config, DoubleFormatInfo.INSTANCE, null, nullLiteral);
    testBasicSerialization(config, DecimalFormatInfo.INSTANCE, null, nullLiteral);
    testBasicSerialization(config, new DateFormatInfo("dd/MM/yyyy"), null, nullLiteral);
    testBasicSerialization(config, new TimeFormatInfo("ss/mm/hh"), null, nullLiteral);
    testBasicSerialization(config, new TimestampFormatInfo("dd/MM/yyyy hh:mm:ss"), null, nullLiteral);
}
Also used : Time(java.sql.Time) FormatInfo(org.apache.inlong.sort.formats.common.FormatInfo) BasicFormatInfo(org.apache.inlong.sort.formats.common.BasicFormatInfo) TimestampFormatInfo(org.apache.inlong.sort.formats.common.TimestampFormatInfo) DoubleFormatInfo(org.apache.inlong.sort.formats.common.DoubleFormatInfo) ShortFormatInfo(org.apache.inlong.sort.formats.common.ShortFormatInfo) StringFormatInfo(org.apache.inlong.sort.formats.common.StringFormatInfo) BigDecimal(java.math.BigDecimal) Assert.assertArrayEquals(org.junit.Assert.assertArrayEquals) BooleanFormatInfo(org.apache.inlong.sort.formats.common.BooleanFormatInfo) FloatFormatInfo(org.apache.inlong.sort.formats.common.FloatFormatInfo) ByteFormatInfo(org.apache.inlong.sort.formats.common.ByteFormatInfo) TimeFormatInfo(org.apache.inlong.sort.formats.common.TimeFormatInfo) Timestamp(java.sql.Timestamp) RowFormatInfo(org.apache.inlong.sort.formats.common.RowFormatInfo) Test(org.junit.Test) StandardCharsets(java.nio.charset.StandardCharsets) Date(java.sql.Date) IntFormatInfo(org.apache.inlong.sort.formats.common.IntFormatInfo) Consumer(java.util.function.Consumer) LongFormatInfo(org.apache.inlong.sort.formats.common.LongFormatInfo) Row(org.apache.flink.types.Row) DecimalFormatInfo(org.apache.inlong.sort.formats.common.DecimalFormatInfo) Assert.assertEquals(org.junit.Assert.assertEquals) DateFormatInfo(org.apache.inlong.sort.formats.common.DateFormatInfo) DateFormatInfo(org.apache.inlong.sort.formats.common.DateFormatInfo) TimestampFormatInfo(org.apache.inlong.sort.formats.common.TimestampFormatInfo) TimeFormatInfo(org.apache.inlong.sort.formats.common.TimeFormatInfo) Test(org.junit.Test)

Aggregations

DateFormatInfo (org.apache.inlong.sort.formats.common.DateFormatInfo)12 TimeFormatInfo (org.apache.inlong.sort.formats.common.TimeFormatInfo)12 TimestampFormatInfo (org.apache.inlong.sort.formats.common.TimestampFormatInfo)12 Row (org.apache.flink.types.Row)10 BasicFormatInfo (org.apache.inlong.sort.formats.common.BasicFormatInfo)10 BooleanFormatInfo (org.apache.inlong.sort.formats.common.BooleanFormatInfo)10 ByteFormatInfo (org.apache.inlong.sort.formats.common.ByteFormatInfo)10 DecimalFormatInfo (org.apache.inlong.sort.formats.common.DecimalFormatInfo)10 DoubleFormatInfo (org.apache.inlong.sort.formats.common.DoubleFormatInfo)10 FloatFormatInfo (org.apache.inlong.sort.formats.common.FloatFormatInfo)10 FormatInfo (org.apache.inlong.sort.formats.common.FormatInfo)10 IntFormatInfo (org.apache.inlong.sort.formats.common.IntFormatInfo)10 LongFormatInfo (org.apache.inlong.sort.formats.common.LongFormatInfo)10 RowFormatInfo (org.apache.inlong.sort.formats.common.RowFormatInfo)10 ShortFormatInfo (org.apache.inlong.sort.formats.common.ShortFormatInfo)10 StringFormatInfo (org.apache.inlong.sort.formats.common.StringFormatInfo)10 Test (org.junit.Test)10 Date (java.sql.Date)9 Time (java.sql.Time)9 Timestamp (java.sql.Timestamp)9