Examples with TimestampType org.apache.flink.table.types.logical.TimestampType used on opensource projects

Example 11 with TimestampType

use of org.apache.flink.table.types.logical.TimestampType in project flink by apache.

the class DynamicTableSourceSpecSerdeTest method testDynamicTableSinkSpecSerde.

public static Stream<DynamicTableSourceSpec> testDynamicTableSinkSpecSerde() {
    Map<String, String> options1 = new HashMap<>();
    options1.put("connector", FileSystemTableFactory.IDENTIFIER);
    options1.put("format", TestCsvFormatFactory.IDENTIFIER);
    options1.put("path", "/tmp");
    final ResolvedSchema resolvedSchema1 = new ResolvedSchema(Collections.singletonList(Column.physical("a", DataTypes.BIGINT())), Collections.emptyList(), null);
    final CatalogTable catalogTable1 = CatalogTable.of(Schema.newBuilder().fromResolvedSchema(resolvedSchema1).build(), null, Collections.emptyList(), options1);
    DynamicTableSourceSpec spec1 = new DynamicTableSourceSpec(ContextResolvedTable.temporary(ObjectIdentifier.of(DEFAULT_BUILTIN_CATALOG, DEFAULT_BUILTIN_DATABASE, "MyTable"), new ResolvedCatalogTable(catalogTable1, resolvedSchema1)), null);
    Map<String, String> options2 = new HashMap<>();
    options2.put("connector", TestValuesTableFactory.IDENTIFIER);
    options2.put("disable-lookup", "true");
    options2.put("enable-watermark-push-down", "true");
    options2.put("filterable-fields", "b");
    options2.put("bounded", "false");
    options2.put("readable-metadata", "m1:INT, m2:STRING");
    final ResolvedSchema resolvedSchema2 = new ResolvedSchema(Arrays.asList(Column.physical("a", DataTypes.BIGINT()), Column.physical("b", DataTypes.INT()), Column.physical("c", DataTypes.STRING()), Column.physical("p", DataTypes.STRING()), Column.metadata("m1", DataTypes.INT(), null, false), Column.metadata("m2", DataTypes.STRING(), null, false), Column.physical("ts", DataTypes.TIMESTAMP(3))), Collections.emptyList(), null);
    final CatalogTable catalogTable2 = CatalogTable.of(Schema.newBuilder().fromResolvedSchema(resolvedSchema2).build(), null, Collections.emptyList(), options2);
    FlinkTypeFactory factory = FlinkTypeFactory.INSTANCE();
    RexBuilder rexBuilder = new RexBuilder(factory);
    DynamicTableSourceSpec spec2 = new DynamicTableSourceSpec(ContextResolvedTable.temporary(ObjectIdentifier.of(DEFAULT_BUILTIN_CATALOG, DEFAULT_BUILTIN_DATABASE, "MyTable"), new ResolvedCatalogTable(catalogTable2, resolvedSchema2)), Arrays.asList(new ProjectPushDownSpec(new int[][] { { 0 }, { 1 }, { 4 }, { 6 } }, RowType.of(new LogicalType[] { new BigIntType(), new IntType(), new IntType(), new TimestampType(3) }, new String[] { "a", "b", "m1", "ts" })), new ReadingMetadataSpec(Arrays.asList("m1", "m2"), RowType.of(new LogicalType[] { new BigIntType(), new IntType(), new IntType(), new TimestampType(3) }, new String[] { "a", "b", "m1", "ts" })), new FilterPushDownSpec(Collections.singletonList(// b >= 10
    rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, rexBuilder.makeInputRef(factory.createSqlType(SqlTypeName.INTEGER), 1), rexBuilder.makeExactLiteral(new BigDecimal(10))))), new WatermarkPushDownSpec(rexBuilder.makeCall(SqlStdOperatorTable.MINUS, rexBuilder.makeInputRef(factory.createSqlType(SqlTypeName.TIMESTAMP, 3), 3), rexBuilder.makeIntervalLiteral(BigDecimal.valueOf(1000), new SqlIntervalQualifier(TimeUnit.SECOND, 2, TimeUnit.SECOND, 6, SqlParserPos.ZERO))), 5000, RowType.of(new BigIntType(), new IntType(), new IntType(), new TimestampType(false, TimestampKind.ROWTIME, 3))), new SourceWatermarkSpec(true, RowType.of(new BigIntType(), new IntType(), new IntType(), new TimestampType(false, TimestampKind.ROWTIME, 3))), new LimitPushDownSpec(100), new PartitionPushDownSpec(Arrays.asList(new HashMap<String, String>() {

        {
            put("p", "A");
        }
    }, new HashMap<String, String>() {

        {
            put("p", "B");
        }
    }))));
    return Stream.of(spec1, spec2);
}

Example 12 with TimestampType

use of org.apache.flink.table.types.logical.TimestampType in project flink by apache.

the class LogicalTypeJsonSerdeTest method testLogicalTypeSerde.

private static List<LogicalType> testLogicalTypeSerde() {
    final List<LogicalType> types = Arrays.asList(new BooleanType(), new TinyIntType(), new SmallIntType(), new IntType(), new BigIntType(), new FloatType(), new DoubleType(), new DecimalType(10), new DecimalType(15, 5), CharType.ofEmptyLiteral(), new CharType(), new CharType(5), VarCharType.ofEmptyLiteral(), new VarCharType(), new VarCharType(5), BinaryType.ofEmptyLiteral(), new BinaryType(), new BinaryType(100), VarBinaryType.ofEmptyLiteral(), new VarBinaryType(), new VarBinaryType(100), new DateType(), new TimeType(), new TimeType(3), new TimestampType(), new TimestampType(3), new LocalZonedTimestampType(false, TimestampKind.PROCTIME, 3), new TimestampType(false, TimestampKind.ROWTIME, 3), new ZonedTimestampType(), new ZonedTimestampType(3), new ZonedTimestampType(false, TimestampKind.ROWTIME, 3), new LocalZonedTimestampType(), new LocalZonedTimestampType(3), new LocalZonedTimestampType(false, TimestampKind.PROCTIME, 3), new LocalZonedTimestampType(false, TimestampKind.ROWTIME, 3), new DayTimeIntervalType(DayTimeIntervalType.DayTimeResolution.DAY_TO_HOUR), new DayTimeIntervalType(false, DayTimeIntervalType.DayTimeResolution.DAY_TO_HOUR, 3, 6), new YearMonthIntervalType(YearMonthIntervalType.YearMonthResolution.YEAR_TO_MONTH), new YearMonthIntervalType(false, YearMonthIntervalType.YearMonthResolution.MONTH, 2), new ZonedTimestampType(), new LocalZonedTimestampType(), new LocalZonedTimestampType(false, TimestampKind.PROCTIME, 3), new SymbolType<>(), new ArrayType(new IntType(false)), new ArrayType(new LocalZonedTimestampType(false, TimestampKind.ROWTIME, 3)), new ArrayType(new ZonedTimestampType(false, TimestampKind.ROWTIME, 3)), new ArrayType(new TimestampType()), new ArrayType(CharType.ofEmptyLiteral()), new ArrayType(VarCharType.ofEmptyLiteral()), new ArrayType(BinaryType.ofEmptyLiteral()), new ArrayType(VarBinaryType.ofEmptyLiteral()), new MapType(new BigIntType(), new IntType(false)), new MapType(new TimestampType(false, TimestampKind.ROWTIME, 3), new ZonedTimestampType()), new MapType(CharType.ofEmptyLiteral(), CharType.ofEmptyLiteral()), new MapType(VarCharType.ofEmptyLiteral(), VarCharType.ofEmptyLiteral()), new MapType(BinaryType.ofEmptyLiteral(), BinaryType.ofEmptyLiteral()), new MapType(VarBinaryType.ofEmptyLiteral(), VarBinaryType.ofEmptyLiteral()), new MultisetType(new IntType(false)), new MultisetType(new TimestampType()), new MultisetType(new TimestampType(true, TimestampKind.ROWTIME, 3)), new MultisetType(CharType.ofEmptyLiteral()), new MultisetType(VarCharType.ofEmptyLiteral()), new MultisetType(BinaryType.ofEmptyLiteral()), new MultisetType(VarBinaryType.ofEmptyLiteral()), RowType.of(new BigIntType(), new IntType(false), new VarCharType(200)), RowType.of(new LogicalType[] { new BigIntType(), new IntType(false), new VarCharType(200) }, new String[] { "f1", "f2", "f3" }), RowType.of(new TimestampType(false, TimestampKind.ROWTIME, 3), new TimestampType(false, TimestampKind.REGULAR, 3), new ZonedTimestampType(false, TimestampKind.ROWTIME, 3), new ZonedTimestampType(false, TimestampKind.REGULAR, 3), new LocalZonedTimestampType(false, TimestampKind.ROWTIME, 3), new LocalZonedTimestampType(false, TimestampKind.PROCTIME, 3), new LocalZonedTimestampType(false, TimestampKind.REGULAR, 3)), RowType.of(CharType.ofEmptyLiteral(), VarCharType.ofEmptyLiteral(), BinaryType.ofEmptyLiteral(), VarBinaryType.ofEmptyLiteral()), // registered structured type
    StructuredType.newBuilder(ObjectIdentifier.of("cat", "db", "structuredType"), PojoClass.class).attributes(Arrays.asList(new StructuredType.StructuredAttribute("f0", new IntType(true)), new StructuredType.StructuredAttribute("f1", new BigIntType(true)), new StructuredType.StructuredAttribute("f2", new VarCharType(200), "desc"))).comparison(StructuredType.StructuredComparison.FULL).setFinal(false).setInstantiable(false).superType(StructuredType.newBuilder(ObjectIdentifier.of("cat", "db", "structuredType2")).attributes(Collections.singletonList(new StructuredType.StructuredAttribute("f0", new BigIntType(false)))).build()).description("description for StructuredType").build(), // unregistered structured type
    StructuredType.newBuilder(PojoClass.class).attributes(Arrays.asList(new StructuredType.StructuredAttribute("f0", new IntType(true)), new StructuredType.StructuredAttribute("f1", new BigIntType(true)), new StructuredType.StructuredAttribute("f2", new VarCharType(200), "desc"))).build(), // registered distinct type
    DistinctType.newBuilder(ObjectIdentifier.of("cat", "db", "distinctType"), new VarCharType(5)).build(), DistinctType.newBuilder(ObjectIdentifier.of("cat", "db", "distinctType"), new VarCharType(false, 5)).build(), // custom RawType
    new RawType<>(LocalDateTime.class, LocalDateTimeSerializer.INSTANCE), // external RawType
    new RawType<>(Row.class, ExternalSerializer.of(DataTypes.ROW(DataTypes.INT(), DataTypes.STRING()))));
    final List<LogicalType> mutableTypes = new ArrayList<>(types);
    // RawType for MapView
    addRawTypesForMapView(mutableTypes, new VarCharType(100), new VarCharType(100));
    addRawTypesForMapView(mutableTypes, new VarCharType(100), new BigIntType());
    addRawTypesForMapView(mutableTypes, new BigIntType(), new VarCharType(100));
    addRawTypesForMapView(mutableTypes, new BigIntType(), new BigIntType());
    // RawType for ListView
    addRawTypesForListView(mutableTypes, new VarCharType(100));
    addRawTypesForListView(mutableTypes, new BigIntType());
    // RawType for custom MapView
    mutableTypes.add(DataViewUtils.adjustDataViews(MapView.newMapViewDataType(DataTypes.STRING().toInternal(), DataTypes.STRING().bridgedTo(byte[].class)), false).getLogicalType());
    final List<LogicalType> allTypes = new ArrayList<>();
    // consider nullable
    for (LogicalType type : mutableTypes) {
        allTypes.add(type.copy(true));
        allTypes.add(type.copy(false));
    }
    // ignore nullable for NullType
    allTypes.add(new NullType());
    return allTypes;
}

Example 13 with TimestampType

use of org.apache.flink.table.types.logical.TimestampType in project flink by apache.

the class SortCodeGeneratorTest method generateValues.

private Object[] generateValues(LogicalType type) {
    Random rnd = new Random();
    int seedNum = RECORD_NUM / 5;
    Object[] seeds = new Object[seedNum];
    seeds[0] = null;
    seeds[1] = value1(type, rnd);
    seeds[2] = value2(type, rnd);
    seeds[3] = value3(type, rnd);
    for (int i = 4; i < seeds.length; i++) {
        switch(type.getTypeRoot()) {
            case BOOLEAN:
                seeds[i] = rnd.nextBoolean();
                break;
            case TINYINT:
                seeds[i] = (byte) rnd.nextLong();
                break;
            case SMALLINT:
                seeds[i] = (short) rnd.nextLong();
                break;
            case INTEGER:
                seeds[i] = rnd.nextInt();
                break;
            case BIGINT:
                seeds[i] = rnd.nextLong();
                break;
            case FLOAT:
                seeds[i] = rnd.nextFloat() * rnd.nextLong();
                break;
            case DOUBLE:
                seeds[i] = rnd.nextDouble() * rnd.nextLong();
                break;
            case VARCHAR:
                seeds[i] = StringData.fromString(RandomStringUtils.random(rnd.nextInt(20)));
                break;
            case DECIMAL:
                DecimalType decimalType = (DecimalType) type;
                BigDecimal decimal = new BigDecimal(rnd.nextInt()).divide(new BigDecimal(ThreadLocalRandom.current().nextInt(1, 256)), ThreadLocalRandom.current().nextInt(1, 30), BigDecimal.ROUND_HALF_EVEN);
                seeds[i] = DecimalData.fromBigDecimal(decimal, decimalType.getPrecision(), decimalType.getScale());
                break;
            case TIMESTAMP_WITHOUT_TIME_ZONE:
                TimestampType timestampType = (TimestampType) type;
                if (timestampType.getPrecision() <= 3) {
                    seeds[i] = TimestampData.fromEpochMillis(rnd.nextLong());
                } else {
                    seeds[i] = TimestampData.fromEpochMillis(rnd.nextLong(), rnd.nextInt(1000000));
                }
                break;
            case ARRAY:
            case VARBINARY:
                byte[] bytes = new byte[rnd.nextInt(16) + 1];
                rnd.nextBytes(bytes);
                seeds[i] = type instanceof VarBinaryType ? bytes : BinaryArrayData.fromPrimitiveArray(bytes);
                break;
            case ROW:
                RowType rowType = (RowType) type;
                if (rowType.getFields().get(0).getType().getTypeRoot() == INTEGER) {
                    seeds[i] = GenericRowData.of(rnd.nextInt());
                } else {
                    seeds[i] = GenericRowData.of(GenericRowData.of(rnd.nextInt()));
                }
                break;
            case RAW:
                seeds[i] = RawValueData.fromObject(rnd.nextInt());
                break;
            default:
                throw new RuntimeException("Not support!");
        }
    }
    // result values
    Object[] results = new Object[RECORD_NUM];
    for (int i = 0; i < RECORD_NUM; i++) {
        results[i] = seeds[rnd.nextInt(seedNum)];
    }
    return results;
}

Example 14 with TimestampType

use of org.apache.flink.table.types.logical.TimestampType in project flink by apache.

the class LogicalTypeJsonDeserializer method deserializeTimestamp.

private static LogicalType deserializeTimestamp(LogicalTypeRoot typeRoot, JsonNode logicalTypeNode) {
    final int precision = logicalTypeNode.get(FIELD_NAME_PRECISION).asInt();
    final TimestampKind kind = TimestampKind.valueOf(logicalTypeNode.get(FIELD_NAME_TIMESTAMP_KIND).asText());
    switch(typeRoot) {
        case TIMESTAMP_WITHOUT_TIME_ZONE:
            return new TimestampType(true, kind, precision);
        case TIMESTAMP_WITH_TIME_ZONE:
            return new ZonedTimestampType(true, kind, precision);
        case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
            return new LocalZonedTimestampType(true, kind, precision);
        default:
            throw new TableException("Timestamp type root expected.");
    }
}

Example 15 with TimestampType

use of org.apache.flink.table.types.logical.TimestampType in project flink by apache.

the class LogicalTypeJsonSerializer method serializeTypeWithGenericSerialization.

// --------------------------------------------------------------------------------------------
// Generic Serialization
// --------------------------------------------------------------------------------------------
private static void serializeTypeWithGenericSerialization(LogicalType logicalType, JsonGenerator jsonGenerator, SerializerProvider serializerProvider, boolean serializeCatalogObjects) throws IOException {
    jsonGenerator.writeStartObject();
    jsonGenerator.writeStringField(FIELD_NAME_TYPE_NAME, logicalType.getTypeRoot().name());
    if (!logicalType.isNullable()) {
        jsonGenerator.writeBooleanField(FIELD_NAME_NULLABLE, false);
    }
    switch(logicalType.getTypeRoot()) {
        case CHAR:
        case VARCHAR:
        case BINARY:
        case VARBINARY:
            serializeZeroLengthString(jsonGenerator);
            break;
        case TIMESTAMP_WITHOUT_TIME_ZONE:
            final TimestampType timestampType = (TimestampType) logicalType;
            serializeTimestamp(timestampType.getPrecision(), timestampType.getKind(), jsonGenerator, serializerProvider);
            break;
        case TIMESTAMP_WITH_TIME_ZONE:
            final ZonedTimestampType zonedTimestampType = (ZonedTimestampType) logicalType;
            serializeTimestamp(zonedTimestampType.getPrecision(), zonedTimestampType.getKind(), jsonGenerator, serializerProvider);
            break;
        case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
            final LocalZonedTimestampType localZonedTimestampType = (LocalZonedTimestampType) logicalType;
            serializeTimestamp(localZonedTimestampType.getPrecision(), localZonedTimestampType.getKind(), jsonGenerator, serializerProvider);
            break;
        case ARRAY:
            serializeCollection(((ArrayType) logicalType).getElementType(), jsonGenerator, serializerProvider, serializeCatalogObjects);
            break;
        case MULTISET:
            serializeCollection(((MultisetType) logicalType).getElementType(), jsonGenerator, serializerProvider, serializeCatalogObjects);
            break;
        case MAP:
            serializeMap((MapType) logicalType, jsonGenerator, serializerProvider, serializeCatalogObjects);
            break;
        case ROW:
            serializeRow((RowType) logicalType, jsonGenerator, serializerProvider, serializeCatalogObjects);
            break;
        case DISTINCT_TYPE:
            serializeDistinctType((DistinctType) logicalType, jsonGenerator, serializerProvider, serializeCatalogObjects);
            break;
        case STRUCTURED_TYPE:
            serializeStructuredType((StructuredType) logicalType, jsonGenerator, serializerProvider, serializeCatalogObjects);
            break;
        case SYMBOL:
            // type root is enough
            break;
        case RAW:
            if (logicalType instanceof RawType) {
                serializeSpecializedRaw((RawType<?>) logicalType, jsonGenerator, serializerProvider);
                break;
            }
        // fall through
        default:
            throw new ValidationException(String.format("Unable to serialize logical type '%s'. Please check the documentation for supported types.", logicalType.asSummaryString()));
    }
    jsonGenerator.writeEndObject();
}