Examples with TimestampFormatInfo org.apache.inlong.sort.formats.common.TimestampFormatInfo used on opensource projects

Example 11 with TimestampFormatInfo

use of org.apache.inlong.sort.formats.common.TimestampFormatInfo in project incubator-inlong by apache.

the class KvSerializationSchemaTest method testNullIteral.

@Test
public void testNullIteral() {
    String nullLiteral = "n/a";
    String nullField = "f=n/a";
    Consumer<KvSerializationSchema.Builder> config = builder -> builder.setNullLiteral(nullLiteral);
    testBasicSerialization(config, StringFormatInfo.INSTANCE, null, nullField);
    testBasicSerialization(config, BooleanFormatInfo.INSTANCE, null, nullField);
    testBasicSerialization(config, ByteFormatInfo.INSTANCE, null, nullField);
    testBasicSerialization(config, ShortFormatInfo.INSTANCE, null, nullField);
    testBasicSerialization(config, IntFormatInfo.INSTANCE, null, nullField);
    testBasicSerialization(config, LongFormatInfo.INSTANCE, null, nullField);
    testBasicSerialization(config, FloatFormatInfo.INSTANCE, null, nullField);
    testBasicSerialization(config, DoubleFormatInfo.INSTANCE, null, nullField);
    testBasicSerialization(config, DecimalFormatInfo.INSTANCE, null, nullField);
    testBasicSerialization(config, new DateFormatInfo("dd/MM/yyyy"), null, nullField);
    testBasicSerialization(config, new TimeFormatInfo("ss/mm/hh"), null, nullField);
    testBasicSerialization(config, new TimestampFormatInfo("dd/MM/yyyy hh:mm:ss"), null, nullField);
}

Example 12 with TimestampFormatInfo

use of org.apache.inlong.sort.formats.common.TimestampFormatInfo in project incubator-inlong by apache.

the class KvSerializationSchemaTest method testNormal.

@Test
public void testNormal() {
    Consumer<KvSerializationSchema.Builder> config = builder -> {
    };
    testBasicSerialization(config, StringFormatInfo.INSTANCE, "hello", "f=hello");
    testBasicSerialization(config, BooleanFormatInfo.INSTANCE, true, "f=true");
    testBasicSerialization(config, ByteFormatInfo.INSTANCE, (byte) 124, "f=124");
    testBasicSerialization(config, ShortFormatInfo.INSTANCE, (short) 10000, "f=10000");
    testBasicSerialization(config, IntFormatInfo.INSTANCE, 1234567, "f=1234567");
    testBasicSerialization(config, LongFormatInfo.INSTANCE, 12345678910L, "f=12345678910");
    testBasicSerialization(config, FloatFormatInfo.INSTANCE, 0.33333334f, "f=0.33333334");
    testBasicSerialization(config, DoubleFormatInfo.INSTANCE, 0.33333333332, "f=0.33333333332");
    testBasicSerialization(config, DecimalFormatInfo.INSTANCE, new BigDecimal("1234.0000000000000000000000001"), "f=1234.0000000000000000000000001");
    testBasicSerialization(config, new DateFormatInfo("dd/MM/yyyy"), Date.valueOf("2020-03-22"), "f=22/03/2020");
    testBasicSerialization(config, new TimeFormatInfo("ss/mm/hh"), Time.valueOf("11:12:13"), "f=13/12/11");
    testBasicSerialization(config, new TimestampFormatInfo("dd/MM/yyyy hh:mm:ss"), Timestamp.valueOf("2020-03-22 11:12:13"), "f=22/03/2020 11:12:13");
}

Example 13 with TimestampFormatInfo

use of org.apache.inlong.sort.formats.common.TimestampFormatInfo in project incubator-inlong by apache.

the class CsvDeserializationSchemaTest method testNormal.

@Test
public void testNormal() throws Exception {
    Consumer<CsvDeserializationSchema.Builder> config = builder -> {
    };
    testBasicDeserialization(config, StringFormatInfo.INSTANCE, "hello", "hello");
    testBasicDeserialization(config, BooleanFormatInfo.INSTANCE, true, "true");
    testBasicDeserialization(config, ByteFormatInfo.INSTANCE, (byte) 124, "124");
    testBasicDeserialization(config, ShortFormatInfo.INSTANCE, (short) 10000, "10000");
    testBasicDeserialization(config, IntFormatInfo.INSTANCE, 1234567, "1234567");
    testBasicDeserialization(config, LongFormatInfo.INSTANCE, 12345678910L, "12345678910");
    testBasicDeserialization(config, FloatFormatInfo.INSTANCE, 0.33333334f, "0.33333334");
    testBasicDeserialization(config, DoubleFormatInfo.INSTANCE, 0.33333333332, "0.33333333332");
    testBasicDeserialization(config, DecimalFormatInfo.INSTANCE, new BigDecimal("1234.0000000000000000000000001"), "1234.0000000000000000000000001");
    testBasicDeserialization(config, new DateFormatInfo("dd/MM/yyyy"), Date.valueOf("2020-03-22"), "22/03/2020");
    testBasicDeserialization(config, new TimeFormatInfo("ss/mm/hh"), Time.valueOf("11:12:13"), "13/12/11");
    testBasicDeserialization(config, new TimestampFormatInfo("dd/MM/yyyy hh:mm:ss"), Timestamp.valueOf("2020-03-22 11:12:13"), "22/03/2020 11:12:13");
}

Example 14 with TimestampFormatInfo

use of org.apache.inlong.sort.formats.common.TimestampFormatInfo in project incubator-inlong by apache.

the class FieldMappingTransformerTest method testTransformWithDt.

@Test
public void testTransformWithDt() throws Exception {
    final FieldInfo fieldInfo = new FieldInfo("id", new LongFormatInfo());
    final FieldInfo dtFieldInfo = new BuiltInFieldInfo("dt", new TimestampFormatInfo(), BuiltInField.DATA_TIME);
    final SourceInfo sourceInfo = new TestingSourceInfo(new FieldInfo[] { fieldInfo, dtFieldInfo });
    final SinkInfo sinkInfo = new TestingSinkInfo(new FieldInfo[] { fieldInfo, dtFieldInfo });
    final long dataFlowId = 1L;
    final DataFlowInfo dataFlowInfo = new DataFlowInfo(dataFlowId, sourceInfo, sinkInfo);
    final FieldMappingTransformer transformer = new FieldMappingTransformer(new Configuration());
    transformer.addDataFlow(dataFlowInfo);
    // should be 3 fields (1 origin fields + time + attr)
    final Row sourceRow = new Row(1 + SOURCE_FIELD_SKIP_STEP);
    final long dt = System.currentTimeMillis();
    sourceRow.setField(0, dt);
    sourceRow.setField(1, "attr");
    sourceRow.setField(2, 9527L);
    final Record sourceRecord = new Record(dataFlowId, dt, sourceRow);
    final Record sinkRecord = transformer.transform(sourceRecord);
    assertEquals(dataFlowId, sinkRecord.getDataflowId());
    assertEquals(2, sinkRecord.getRow().getArity());
    assertEquals(9527L, sinkRecord.getRow().getField(0));
    assertEquals(new Timestamp(dt), sinkRecord.getRow().getField(1));
}

Example 15 with TimestampFormatInfo

use of org.apache.inlong.sort.formats.common.TimestampFormatInfo in project incubator-inlong by apache.

the class TableFormatUtils method deriveLogicalType.

/**
 * Derive the LogicalType for the given FormatInfo.
 */
public static LogicalType deriveLogicalType(FormatInfo formatInfo) {
    if (formatInfo instanceof StringFormatInfo) {
        return new VarCharType();
    } else if (formatInfo instanceof BooleanFormatInfo) {
        return new BooleanType();
    } else if (formatInfo instanceof ByteFormatInfo) {
        return new TinyIntType();
    } else if (formatInfo instanceof ShortFormatInfo) {
        return new SmallIntType();
    } else if (formatInfo instanceof IntFormatInfo) {
        return new IntType();
    } else if (formatInfo instanceof LongFormatInfo) {
        return new BigIntType();
    } else if (formatInfo instanceof FloatFormatInfo) {
        return new FloatType();
    } else if (formatInfo instanceof DoubleFormatInfo) {
        return new DoubleType();
    } else if (formatInfo instanceof DecimalFormatInfo) {
        return new DecimalType();
    } else if (formatInfo instanceof TimeFormatInfo) {
        return new TimeType();
    } else if (formatInfo instanceof DateFormatInfo) {
        return new DateType();
    } else if (formatInfo instanceof TimestampFormatInfo) {
        return new TimestampType(DEFAULT_PRECISION_FOR_TIMESTAMP);
    } else if (formatInfo instanceof LocalZonedTimestampFormatInfo) {
        return new LocalZonedTimestampType();
    } else if (formatInfo instanceof ArrayFormatInfo) {
        FormatInfo elementFormatInfo = ((ArrayFormatInfo) formatInfo).getElementFormatInfo();
        return new ArrayType(deriveLogicalType(elementFormatInfo));
    } else if (formatInfo instanceof MapFormatInfo) {
        MapFormatInfo mapFormatInfo = (MapFormatInfo) formatInfo;
        FormatInfo keyFormatInfo = mapFormatInfo.getKeyFormatInfo();
        FormatInfo valueFormatInfo = mapFormatInfo.getValueFormatInfo();
        return new MapType(deriveLogicalType(keyFormatInfo), deriveLogicalType(valueFormatInfo));
    } else if (formatInfo instanceof RowFormatInfo) {
        RowFormatInfo rowFormatInfo = (RowFormatInfo) formatInfo;
        FormatInfo[] formatInfos = rowFormatInfo.getFieldFormatInfos();
        int formatInfosSize = formatInfos.length;
        LogicalType[] logicalTypes = new LogicalType[formatInfosSize];
        for (int i = 0; i < formatInfosSize; ++i) {
            logicalTypes[i] = deriveLogicalType(formatInfos[i]);
        }
        return RowType.of(logicalTypes, rowFormatInfo.getFieldNames());
    } else if (formatInfo instanceof BinaryFormatInfo) {
        return new BinaryType();
    } else if (formatInfo instanceof NullFormatInfo) {
        return new NullType();
    } else {
        throw new UnsupportedOperationException();
    }
}