Examples with VarBinaryType org.apache.flink.table.types.logical.VarBinaryType used on opensource projects

Example 1 with VarBinaryType

use of org.apache.flink.table.types.logical.VarBinaryType in project flink by apache.

the class LogicalTypesTest method testEmptyStringLiterals.

@Test
public void testEmptyStringLiterals() {
    final CharType charType = CharType.ofEmptyLiteral();
    final VarCharType varcharType = VarCharType.ofEmptyLiteral();
    final BinaryType binaryType = BinaryType.ofEmptyLiteral();
    final VarBinaryType varBinaryType = VarBinaryType.ofEmptyLiteral();
    // make the types nullable for testing
    assertThat(charType.copy(true)).satisfies(nonEqualityCheckWithOtherType(new CharType(1)));
    assertThat(varcharType.copy(true)).satisfies(nonEqualityCheckWithOtherType(new VarCharType(1)));
    assertThat(binaryType.copy(true)).satisfies(nonEqualityCheckWithOtherType(new BinaryType(1)));
    assertThat(varBinaryType.copy(true)).satisfies(nonEqualityCheckWithOtherType(new VarBinaryType(1)));
    assertThat(charType).hasSummaryString("CHAR(0) NOT NULL");
    assertThat(varcharType).hasSummaryString("VARCHAR(0) NOT NULL");
    assertThat(binaryType).hasSummaryString("BINARY(0) NOT NULL");
    assertThat(varBinaryType).hasSummaryString("VARBINARY(0) NOT NULL");
    assertThat(charType).hasNoSerializableString();
    assertThat(varcharType).hasNoSerializableString();
    assertThat(binaryType).hasNoSerializableString();
    assertThat(varBinaryType).hasNoSerializableString();
}

Example 2 with VarBinaryType

use of org.apache.flink.table.types.logical.VarBinaryType in project flink by apache.

the class LogicalTypeJsonSerdeTest method testLogicalTypeSerde.

private static List<LogicalType> testLogicalTypeSerde() {
    final List<LogicalType> types = Arrays.asList(new BooleanType(), new TinyIntType(), new SmallIntType(), new IntType(), new BigIntType(), new FloatType(), new DoubleType(), new DecimalType(10), new DecimalType(15, 5), CharType.ofEmptyLiteral(), new CharType(), new CharType(5), VarCharType.ofEmptyLiteral(), new VarCharType(), new VarCharType(5), BinaryType.ofEmptyLiteral(), new BinaryType(), new BinaryType(100), VarBinaryType.ofEmptyLiteral(), new VarBinaryType(), new VarBinaryType(100), new DateType(), new TimeType(), new TimeType(3), new TimestampType(), new TimestampType(3), new LocalZonedTimestampType(false, TimestampKind.PROCTIME, 3), new TimestampType(false, TimestampKind.ROWTIME, 3), new ZonedTimestampType(), new ZonedTimestampType(3), new ZonedTimestampType(false, TimestampKind.ROWTIME, 3), new LocalZonedTimestampType(), new LocalZonedTimestampType(3), new LocalZonedTimestampType(false, TimestampKind.PROCTIME, 3), new LocalZonedTimestampType(false, TimestampKind.ROWTIME, 3), new DayTimeIntervalType(DayTimeIntervalType.DayTimeResolution.DAY_TO_HOUR), new DayTimeIntervalType(false, DayTimeIntervalType.DayTimeResolution.DAY_TO_HOUR, 3, 6), new YearMonthIntervalType(YearMonthIntervalType.YearMonthResolution.YEAR_TO_MONTH), new YearMonthIntervalType(false, YearMonthIntervalType.YearMonthResolution.MONTH, 2), new ZonedTimestampType(), new LocalZonedTimestampType(), new LocalZonedTimestampType(false, TimestampKind.PROCTIME, 3), new SymbolType<>(), new ArrayType(new IntType(false)), new ArrayType(new LocalZonedTimestampType(false, TimestampKind.ROWTIME, 3)), new ArrayType(new ZonedTimestampType(false, TimestampKind.ROWTIME, 3)), new ArrayType(new TimestampType()), new ArrayType(CharType.ofEmptyLiteral()), new ArrayType(VarCharType.ofEmptyLiteral()), new ArrayType(BinaryType.ofEmptyLiteral()), new ArrayType(VarBinaryType.ofEmptyLiteral()), new MapType(new BigIntType(), new IntType(false)), new MapType(new TimestampType(false, TimestampKind.ROWTIME, 3), new ZonedTimestampType()), new MapType(CharType.ofEmptyLiteral(), CharType.ofEmptyLiteral()), new MapType(VarCharType.ofEmptyLiteral(), VarCharType.ofEmptyLiteral()), new MapType(BinaryType.ofEmptyLiteral(), BinaryType.ofEmptyLiteral()), new MapType(VarBinaryType.ofEmptyLiteral(), VarBinaryType.ofEmptyLiteral()), new MultisetType(new IntType(false)), new MultisetType(new TimestampType()), new MultisetType(new TimestampType(true, TimestampKind.ROWTIME, 3)), new MultisetType(CharType.ofEmptyLiteral()), new MultisetType(VarCharType.ofEmptyLiteral()), new MultisetType(BinaryType.ofEmptyLiteral()), new MultisetType(VarBinaryType.ofEmptyLiteral()), RowType.of(new BigIntType(), new IntType(false), new VarCharType(200)), RowType.of(new LogicalType[] { new BigIntType(), new IntType(false), new VarCharType(200) }, new String[] { "f1", "f2", "f3" }), RowType.of(new TimestampType(false, TimestampKind.ROWTIME, 3), new TimestampType(false, TimestampKind.REGULAR, 3), new ZonedTimestampType(false, TimestampKind.ROWTIME, 3), new ZonedTimestampType(false, TimestampKind.REGULAR, 3), new LocalZonedTimestampType(false, TimestampKind.ROWTIME, 3), new LocalZonedTimestampType(false, TimestampKind.PROCTIME, 3), new LocalZonedTimestampType(false, TimestampKind.REGULAR, 3)), RowType.of(CharType.ofEmptyLiteral(), VarCharType.ofEmptyLiteral(), BinaryType.ofEmptyLiteral(), VarBinaryType.ofEmptyLiteral()), // registered structured type
    StructuredType.newBuilder(ObjectIdentifier.of("cat", "db", "structuredType"), PojoClass.class).attributes(Arrays.asList(new StructuredType.StructuredAttribute("f0", new IntType(true)), new StructuredType.StructuredAttribute("f1", new BigIntType(true)), new StructuredType.StructuredAttribute("f2", new VarCharType(200), "desc"))).comparison(StructuredType.StructuredComparison.FULL).setFinal(false).setInstantiable(false).superType(StructuredType.newBuilder(ObjectIdentifier.of("cat", "db", "structuredType2")).attributes(Collections.singletonList(new StructuredType.StructuredAttribute("f0", new BigIntType(false)))).build()).description("description for StructuredType").build(), // unregistered structured type
    StructuredType.newBuilder(PojoClass.class).attributes(Arrays.asList(new StructuredType.StructuredAttribute("f0", new IntType(true)), new StructuredType.StructuredAttribute("f1", new BigIntType(true)), new StructuredType.StructuredAttribute("f2", new VarCharType(200), "desc"))).build(), // registered distinct type
    DistinctType.newBuilder(ObjectIdentifier.of("cat", "db", "distinctType"), new VarCharType(5)).build(), DistinctType.newBuilder(ObjectIdentifier.of("cat", "db", "distinctType"), new VarCharType(false, 5)).build(), // custom RawType
    new RawType<>(LocalDateTime.class, LocalDateTimeSerializer.INSTANCE), // external RawType
    new RawType<>(Row.class, ExternalSerializer.of(DataTypes.ROW(DataTypes.INT(), DataTypes.STRING()))));
    final List<LogicalType> mutableTypes = new ArrayList<>(types);
    // RawType for MapView
    addRawTypesForMapView(mutableTypes, new VarCharType(100), new VarCharType(100));
    addRawTypesForMapView(mutableTypes, new VarCharType(100), new BigIntType());
    addRawTypesForMapView(mutableTypes, new BigIntType(), new VarCharType(100));
    addRawTypesForMapView(mutableTypes, new BigIntType(), new BigIntType());
    // RawType for ListView
    addRawTypesForListView(mutableTypes, new VarCharType(100));
    addRawTypesForListView(mutableTypes, new BigIntType());
    // RawType for custom MapView
    mutableTypes.add(DataViewUtils.adjustDataViews(MapView.newMapViewDataType(DataTypes.STRING().toInternal(), DataTypes.STRING().bridgedTo(byte[].class)), false).getLogicalType());
    final List<LogicalType> allTypes = new ArrayList<>();
    // consider nullable
    for (LogicalType type : mutableTypes) {
        allTypes.add(type.copy(true));
        allTypes.add(type.copy(false));
    }
    // ignore nullable for NullType
    allTypes.add(new NullType());
    return allTypes;
}

Example 3 with VarBinaryType

use of org.apache.flink.table.types.logical.VarBinaryType in project flink by apache.

the class SortCodeGeneratorTest method generateValues.

private Object[] generateValues(LogicalType type) {
    Random rnd = new Random();
    int seedNum = RECORD_NUM / 5;
    Object[] seeds = new Object[seedNum];
    seeds[0] = null;
    seeds[1] = value1(type, rnd);
    seeds[2] = value2(type, rnd);
    seeds[3] = value3(type, rnd);
    for (int i = 4; i < seeds.length; i++) {
        switch(type.getTypeRoot()) {
            case BOOLEAN:
                seeds[i] = rnd.nextBoolean();
                break;
            case TINYINT:
                seeds[i] = (byte) rnd.nextLong();
                break;
            case SMALLINT:
                seeds[i] = (short) rnd.nextLong();
                break;
            case INTEGER:
                seeds[i] = rnd.nextInt();
                break;
            case BIGINT:
                seeds[i] = rnd.nextLong();
                break;
            case FLOAT:
                seeds[i] = rnd.nextFloat() * rnd.nextLong();
                break;
            case DOUBLE:
                seeds[i] = rnd.nextDouble() * rnd.nextLong();
                break;
            case VARCHAR:
                seeds[i] = StringData.fromString(RandomStringUtils.random(rnd.nextInt(20)));
                break;
            case DECIMAL:
                DecimalType decimalType = (DecimalType) type;
                BigDecimal decimal = new BigDecimal(rnd.nextInt()).divide(new BigDecimal(ThreadLocalRandom.current().nextInt(1, 256)), ThreadLocalRandom.current().nextInt(1, 30), BigDecimal.ROUND_HALF_EVEN);
                seeds[i] = DecimalData.fromBigDecimal(decimal, decimalType.getPrecision(), decimalType.getScale());
                break;
            case TIMESTAMP_WITHOUT_TIME_ZONE:
                TimestampType timestampType = (TimestampType) type;
                if (timestampType.getPrecision() <= 3) {
                    seeds[i] = TimestampData.fromEpochMillis(rnd.nextLong());
                } else {
                    seeds[i] = TimestampData.fromEpochMillis(rnd.nextLong(), rnd.nextInt(1000000));
                }
                break;
            case ARRAY:
            case VARBINARY:
                byte[] bytes = new byte[rnd.nextInt(16) + 1];
                rnd.nextBytes(bytes);
                seeds[i] = type instanceof VarBinaryType ? bytes : BinaryArrayData.fromPrimitiveArray(bytes);
                break;
            case ROW:
                RowType rowType = (RowType) type;
                if (rowType.getFields().get(0).getType().getTypeRoot() == INTEGER) {
                    seeds[i] = GenericRowData.of(rnd.nextInt());
                } else {
                    seeds[i] = GenericRowData.of(GenericRowData.of(rnd.nextInt()));
                }
                break;
            case RAW:
                seeds[i] = RawValueData.fromObject(rnd.nextInt());
                break;
            default:
                throw new RuntimeException("Not support!");
        }
    }
    // result values
    Object[] results = new Object[RECORD_NUM];
    for (int i = 0; i < RECORD_NUM; i++) {
        results[i] = seeds[rnd.nextInt(seedNum)];
    }
    return results;
}

Example 4 with VarBinaryType

use of org.apache.flink.table.types.logical.VarBinaryType in project flink by apache.

the class ParquetSplitReaderUtil method createVectorFromConstant.

public static ColumnVector createVectorFromConstant(LogicalType type, Object value, int batchSize) {
    switch(type.getTypeRoot()) {
        case CHAR:
        case VARCHAR:
        case BINARY:
        case VARBINARY:
            HeapBytesVector bsv = new HeapBytesVector(batchSize);
            if (value == null) {
                bsv.fillWithNulls();
            } else {
                bsv.fill(value instanceof byte[] ? (byte[]) value : value.toString().getBytes(StandardCharsets.UTF_8));
            }
            return bsv;
        case BOOLEAN:
            HeapBooleanVector bv = new HeapBooleanVector(batchSize);
            if (value == null) {
                bv.fillWithNulls();
            } else {
                bv.fill((boolean) value);
            }
            return bv;
        case TINYINT:
            HeapByteVector byteVector = new HeapByteVector(batchSize);
            if (value == null) {
                byteVector.fillWithNulls();
            } else {
                byteVector.fill(((Number) value).byteValue());
            }
            return byteVector;
        case SMALLINT:
            HeapShortVector sv = new HeapShortVector(batchSize);
            if (value == null) {
                sv.fillWithNulls();
            } else {
                sv.fill(((Number) value).shortValue());
            }
            return sv;
        case INTEGER:
            HeapIntVector iv = new HeapIntVector(batchSize);
            if (value == null) {
                iv.fillWithNulls();
            } else {
                iv.fill(((Number) value).intValue());
            }
            return iv;
        case BIGINT:
            HeapLongVector lv = new HeapLongVector(batchSize);
            if (value == null) {
                lv.fillWithNulls();
            } else {
                lv.fill(((Number) value).longValue());
            }
            return lv;
        case DECIMAL:
            DecimalType decimalType = (DecimalType) type;
            int precision = decimalType.getPrecision();
            int scale = decimalType.getScale();
            DecimalData decimal = value == null ? null : Preconditions.checkNotNull(DecimalData.fromBigDecimal((BigDecimal) value, precision, scale));
            ColumnVector internalVector;
            if (ParquetSchemaConverter.is32BitDecimal(precision)) {
                internalVector = createVectorFromConstant(new IntType(), decimal == null ? null : (int) decimal.toUnscaledLong(), batchSize);
            } else if (ParquetSchemaConverter.is64BitDecimal(precision)) {
                internalVector = createVectorFromConstant(new BigIntType(), decimal == null ? null : decimal.toUnscaledLong(), batchSize);
            } else {
                internalVector = createVectorFromConstant(new VarBinaryType(), decimal == null ? null : decimal.toUnscaledBytes(), batchSize);
            }
            return new ParquetDecimalVector(internalVector);
        case FLOAT:
            HeapFloatVector fv = new HeapFloatVector(batchSize);
            if (value == null) {
                fv.fillWithNulls();
            } else {
                fv.fill(((Number) value).floatValue());
            }
            return fv;
        case DOUBLE:
            HeapDoubleVector dv = new HeapDoubleVector(batchSize);
            if (value == null) {
                dv.fillWithNulls();
            } else {
                dv.fill(((Number) value).doubleValue());
            }
            return dv;
        case DATE:
            if (value instanceof LocalDate) {
                value = Date.valueOf((LocalDate) value);
            }
            return createVectorFromConstant(new IntType(), value == null ? null : toInternal((Date) value), batchSize);
        case TIMESTAMP_WITHOUT_TIME_ZONE:
            HeapTimestampVector tv = new HeapTimestampVector(batchSize);
            if (value == null) {
                tv.fillWithNulls();
            } else {
                tv.fill(TimestampData.fromLocalDateTime((LocalDateTime) value));
            }
            return tv;
        default:
            throw new UnsupportedOperationException("Unsupported type: " + type);
    }
}

Example 5 with VarBinaryType

use of org.apache.flink.table.types.logical.VarBinaryType in project flink by apache.

the class ArrowReaderWriterTest method init.

@BeforeClass
public static void init() {
    fieldTypes.add(new TinyIntType());
    fieldTypes.add(new SmallIntType());
    fieldTypes.add(new IntType());
    fieldTypes.add(new BigIntType());
    fieldTypes.add(new BooleanType());
    fieldTypes.add(new FloatType());
    fieldTypes.add(new DoubleType());
    fieldTypes.add(new VarCharType());
    fieldTypes.add(new VarBinaryType());
    fieldTypes.add(new DecimalType(10, 3));
    fieldTypes.add(new DateType());
    fieldTypes.add(new TimeType(0));
    fieldTypes.add(new TimeType(2));
    fieldTypes.add(new TimeType(4));
    fieldTypes.add(new TimeType(8));
    fieldTypes.add(new LocalZonedTimestampType(0));
    fieldTypes.add(new LocalZonedTimestampType(2));
    fieldTypes.add(new LocalZonedTimestampType(4));
    fieldTypes.add(new LocalZonedTimestampType(8));
    fieldTypes.add(new TimestampType(0));
    fieldTypes.add(new TimestampType(2));
    fieldTypes.add(new TimestampType(4));
    fieldTypes.add(new TimestampType(8));
    fieldTypes.add(new ArrayType(new VarCharType()));
    rowFieldType = new RowType(Arrays.asList(new RowType.RowField("a", new IntType()), new RowType.RowField("b", new VarCharType()), new RowType.RowField("c", new ArrayType(new VarCharType())), new RowType.RowField("d", new TimestampType(2)), new RowType.RowField("e", new RowType(Arrays.asList(new RowType.RowField("e1", new IntType()), new RowType.RowField("e2", new VarCharType()))))));
    fieldTypes.add(rowFieldType);
    List<RowType.RowField> rowFields = new ArrayList<>();
    for (int i = 0; i < fieldTypes.size(); i++) {
        rowFields.add(new RowType.RowField("f" + i, fieldTypes.get(i)));
    }
    rowType = new RowType(rowFields);
    allocator = ArrowUtils.getRootAllocator().newChildAllocator("stdout", 0, Long.MAX_VALUE);
}