Examples with TimestampType io.prestosql.spi.type.TimestampType used on opensource projects

Example 6 with TimestampType

use of io.prestosql.spi.type.TimestampType in project boostkit-bigdata by kunpengcompute.

the class HivePartitionManager method getFilteredPartitionNames.

private List<String> getFilteredPartitionNames(SemiTransactionalHiveMetastore metastore, HiveIdentity identity, SchemaTableName tableName, List<HiveColumnHandle> partitionKeys, TupleDomain<ColumnHandle> effectivePredicate, Table table) {
    checkArgument(effectivePredicate.getDomains().isPresent());
    List<String> filter = new ArrayList<>();
    for (HiveColumnHandle partitionKey : partitionKeys) {
        Domain domain = effectivePredicate.getDomains().get().get(partitionKey);
        if (domain != null && domain.isNullableSingleValue()) {
            Object value = domain.getNullableSingleValue();
            Type type = domain.getType();
            if (value == null) {
                filter.add(HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION);
            } else if (type instanceof CharType) {
                Slice slice = (Slice) value;
                filter.add(padSpaces(slice, (CharType) type).toStringUtf8());
            } else if (type instanceof VarcharType) {
                Slice slice = (Slice) value;
                filter.add(slice.toStringUtf8());
            } else // unless we know that all partition values use the canonical Java representation.
            if (!assumeCanonicalPartitionKeys) {
                filter.add(PARTITION_VALUE_WILDCARD);
            } else if (type instanceof DecimalType && !((DecimalType) type).isShort()) {
                Slice slice = (Slice) value;
                filter.add(Decimals.toString(slice, ((DecimalType) type).getScale()));
            } else if (type instanceof DecimalType && ((DecimalType) type).isShort()) {
                filter.add(Decimals.toString((long) value, ((DecimalType) type).getScale()));
            } else if (type instanceof DateType) {
                DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.date().withZoneUTC();
                filter.add(dateTimeFormatter.print(TimeUnit.DAYS.toMillis((long) value)));
            } else if (type instanceof TimestampType) {
                // we don't have time zone info, so just add a wildcard
                filter.add(PARTITION_VALUE_WILDCARD);
            } else if (type instanceof TinyintType || type instanceof SmallintType || type instanceof IntegerType || type instanceof BigintType || type instanceof DoubleType || type instanceof RealType || type instanceof BooleanType) {
                filter.add(value.toString());
            } else {
                throw new PrestoException(NOT_SUPPORTED, format("Unsupported partition key type: %s", type.getDisplayName()));
            }
        } else {
            filter.add(PARTITION_VALUE_WILDCARD);
        }
    }
    // fetch the partition names
    return metastore.getPartitionNamesByParts(identity, tableName.getSchemaName(), tableName.getTableName(), filter, table).orElseThrow(() -> new TableNotFoundException(tableName));
}

Example 7 with TimestampType

use of io.prestosql.spi.type.TimestampType in project boostkit-bigdata by kunpengcompute.

the class TestHiveBucketing method testHashingCompare.

@Test
public void testHashingCompare() {
    assertBucketEquals("boolean", null, 0, 0);
    assertBucketEquals("boolean", true, 1, 1);
    assertBucketEquals("boolean", false, 0, 0);
    assertBucketEquals("tinyint", null, 0, 0);
    assertBucketEquals("tinyint", (byte) 5, 5, 5);
    assertBucketEquals("tinyint", Byte.MIN_VALUE, -128, -128);
    assertBucketEquals("tinyint", Byte.MAX_VALUE, 127, 127);
    assertBucketEquals("smallint", null, 0, 0);
    assertBucketEquals("smallint", (short) 300, 300, 2107031704);
    assertBucketEquals("smallint", Short.MIN_VALUE, -32768, 1342976838);
    assertBucketEquals("smallint", Short.MAX_VALUE, 32767, -684075052);
    assertBucketEquals("int", null, 0, 0);
    assertBucketEquals("int", 300_000, 300000, -678663480);
    assertBucketEquals("int", Integer.MIN_VALUE, -2147483648, 1194881028);
    assertBucketEquals("int", Integer.MAX_VALUE, 2147483647, 1133859967);
    assertBucketEquals("bigint", null, 0, 0);
    assertBucketEquals("bigint", 300_000_000_000L, -647710651, -888935297);
    assertBucketEquals("bigint", Long.MIN_VALUE, -2147483648, 1728983947);
    assertBucketEquals("bigint", Long.MAX_VALUE, -2147483648, -536577852);
    assertBucketEquals("float", null, 0, 0);
    assertBucketEquals("float", 12.34F, 1095069860, -381747602);
    assertBucketEquals("float", -Float.MAX_VALUE, -8388609, 470252243);
    assertBucketEquals("float", Float.MIN_VALUE, 1, 1206721797);
    assertBucketEquals("float", Float.POSITIVE_INFINITY, 2139095040, -292175804);
    assertBucketEquals("float", Float.NEGATIVE_INFINITY, -8388608, -1433270801);
    assertBucketEquals("float", Float.NaN, 2143289344, -480354314);
    // also a NaN
    assertBucketEquals("float", intBitsToFloat(0xffc00000), 2143289344, -480354314);
    // also a NaN
    assertBucketEquals("float", intBitsToFloat(0x7fc00000), 2143289344, -480354314);
    // also a NaN
    assertBucketEquals("float", intBitsToFloat(0x7fc01234), 2143289344, -480354314);
    // also a NaN
    assertBucketEquals("float", intBitsToFloat(0xffc01234), 2143289344, -480354314);
    assertBucketEquals("double", null, 0, 0);
    assertBucketEquals("double", 12.34, 986311098, -2070733568);
    assertBucketEquals("double", -Double.MAX_VALUE, 1048576, 14392725);
    assertBucketEquals("double", Double.MIN_VALUE, 1, -8838199);
    assertBucketEquals("double", Double.POSITIVE_INFINITY, 2146435072, 1614292060);
    assertBucketEquals("double", Double.NEGATIVE_INFINITY, -1048576, 141388605);
    assertBucketEquals("double", Double.NaN, 2146959360, 1138026565);
    // also a NaN
    assertBucketEquals("double", longBitsToDouble(0xfff8000000000000L), 2146959360, 1138026565);
    // also a NaN
    assertBucketEquals("double", longBitsToDouble(0x7ff8123412341234L), 2146959360, 1138026565);
    // also a NaN
    assertBucketEquals("double", longBitsToDouble(0xfff8123412341234L), 2146959360, 1138026565);
    assertBucketEquals("varchar(15)", null, 0, 0);
    assertBucketEquals("varchar(15)", "", 1, -965378730);
    assertBucketEquals("varchar(15)", "test string", -189841218, -138301454);
    // 3-byte UTF-8 sequences (in Basic Plane, i.e. Plane 0)
    assertBucketEquals("varchar(15)", "\u5f3a\u5927\u7684Hetu\u5f15\u64ce", 481023052, 1436831192);
    // 4 code points: 20FFC - 20FFF. 4-byte UTF-8 sequences in Supplementary Plane 2
    assertBucketEquals("varchar(15)", "\uD843\uDFFC\uD843\uDFFD\uD843\uDFFE\uD843\uDFFF", -457487557, -697348811);
    assertBucketEquals("string", null, 0, 0);
    assertBucketEquals("string", "", 0, -965378730);
    assertBucketEquals("string", "test string", -318923937, -138301454);
    // 3-byte UTF-8 sequences (in Basic Plane, i.e. Plane 0)
    assertBucketEquals("string", "\u5f3a\u5927\u7684Hetu\u5f15\u64ce", 889847277, 1436831192);
    // 4 code points: 20FFC - 20FFF. 4-byte UTF-8 sequences in Supplementary Plane 2
    assertBucketEquals("string", "\uD843\uDFFC\uD843\uDFFD\uD843\uDFFE\uD843\uDFFF", -1810797254, -697348811);
    assertBucketEquals("char(6)", null, 0, 0);
    assertBucketEquals("char(6)", "", 1, -965378730);
    assertBucketEquals("char(6)", "test_1", 10333957, 1284522943);
    assertBucketEquals("date", null, 0, 0);
    assertBucketEquals("date", Date.valueOf("1970-01-01"), 0, 1362653161);
    assertBucketEquals("date", Date.valueOf("2015-11-19"), 16758, 8542395);
    assertBucketEquals("date", Date.valueOf("1950-11-19"), -6983, -431619185);
    for (BucketingVersion version : BucketingVersion.values()) {
        List<TypeInfo> typeInfos = ImmutableList.of(timestampTypeInfo);
        assertThatThrownBy(() -> getBucketHashCode(version, typeInfos, new Object[] { 0 })).hasMessage("Computation of Hive bucket hashCode is not supported for Hive primitive category: TIMESTAMP");
        TimestampType timestampType = TimestampType.TIMESTAMP;
        BlockBuilder builder = timestampType.createBlockBuilder(null, 1);
        timestampType.writeLong(builder, 0);
        Page page = new Page(builder.build());
        assertThatThrownBy(() -> getBucketHashCode(version, typeInfos, page, 0)).hasMessage("Computation of Hive bucket hashCode is not supported for Hive primitive category: TIMESTAMP");
    }
    assertBucketEquals("array<double>", null, 0, 0);
    assertBucketEquals("array<boolean>", ImmutableList.of(), 0, 0);
    assertBucketEquals("array<smallint>", ImmutableList.of((short) 5, (short) 8, (short) 13), 5066, -905011156);
    assertBucketEquals("array<string>", ImmutableList.of("test1", "test2", "test3", "test4"), 957612994, 1305539282);
    assertBucketEquals("array<array<bigint>>", ImmutableList.of(ImmutableList.of(10L, 20L), ImmutableList.of(-10L, -20L), asList((Object) null)), 326368, 611324477);
    assertBucketEquals("map<float,date>", null, 0, 0);
    assertBucketEquals("map<double,timestamp>", ImmutableMap.of(), 0, 0);
    assertBucketEquals("map<string,bigint>", ImmutableMap.of("key", 123L, "key2", 123456789L, "key3", -123456L), 127880789, -1910999650);
    assertBucketEquals("map<array<double>,map<int,string>>", ImmutableMap.of(ImmutableList.of(12.3, 45.7), ImmutableMap.of(123, "test99")), -34001111, -1565874874);
    // multiple bucketing columns
    assertBucketEquals(ImmutableList.of("float", "array<smallint>", "map<string,bigint>"), ImmutableList.of(12.34F, ImmutableList.of((short) 5, (short) 8, (short) 13), ImmutableMap.of("key", 123L)), 95411006, 932898434);
    assertBucketEquals(ImmutableList.of("double", "array<smallint>", "boolean", "map<string,bigint>", "tinyint"), asList(null, ImmutableList.of((short) 5, (short) 8, (short) 13), null, ImmutableMap.of("key", 123L), null), 154207826, -1120812524);
}

Example 8 with TimestampType

use of io.prestosql.spi.type.TimestampType in project incubator-pulsar by apache.

the class PulsarPrimitiveRowDecoder method decodeRow.

@Override
public Optional<Map<DecoderColumnHandle, FieldValueProvider>> decodeRow(ByteBuf byteBuf) {
    if (columnHandle == null) {
        return Optional.empty();
    }
    Object value = schema.decode(byteBuf);
    Map<DecoderColumnHandle, FieldValueProvider> primitiveColumn = new HashMap<>();
    if (value == null) {
        primitiveColumn.put(columnHandle, FieldValueProviders.nullValueProvider());
    } else {
        Type type = columnHandle.getType();
        if (type instanceof BooleanType) {
            primitiveColumn.put(columnHandle, booleanValueProvider(Boolean.valueOf((Boolean) value)));
        } else if (type instanceof TinyintType || type instanceof SmallintType || type instanceof IntegerType || type instanceof BigintType) {
            primitiveColumn.put(columnHandle, longValueProvider(Long.parseLong(value.toString())));
        } else if (type instanceof DoubleType) {
            primitiveColumn.put(columnHandle, doubleValueProvider(Double.parseDouble(value.toString())));
        } else if (type instanceof RealType) {
            primitiveColumn.put(columnHandle, longValueProvider(Float.floatToIntBits((Float.parseFloat(value.toString())))));
        } else if (type instanceof VarbinaryType) {
            primitiveColumn.put(columnHandle, bytesValueProvider((byte[]) value));
        } else if (type instanceof VarcharType) {
            primitiveColumn.put(columnHandle, bytesValueProvider(value.toString().getBytes()));
        } else if (type instanceof DateType) {
            primitiveColumn.put(columnHandle, longValueProvider(((Date) value).getTime()));
        } else if (type instanceof TimeType) {
            primitiveColumn.put(columnHandle, longValueProvider(((Time) value).getTime()));
        } else if (type instanceof TimestampType) {
            primitiveColumn.put(columnHandle, longValueProvider(((Timestamp) value).getTime()));
        } else {
            primitiveColumn.put(columnHandle, bytesValueProvider(value.toString().getBytes()));
        }
    }
    return Optional.of(primitiveColumn);
}

Example 9 with TimestampType

use of io.prestosql.spi.type.TimestampType in project pulsar by apache.

the class PulsarPrimitiveRowDecoder method decodeRow.

@Override
public Optional<Map<DecoderColumnHandle, FieldValueProvider>> decodeRow(ByteBuf byteBuf) {
    if (columnHandle == null) {
        return Optional.empty();
    }
    Object value = schema.decode(byteBuf);
    Map<DecoderColumnHandle, FieldValueProvider> primitiveColumn = new HashMap<>();
    if (value == null) {
        primitiveColumn.put(columnHandle, FieldValueProviders.nullValueProvider());
    } else {
        Type type = columnHandle.getType();
        if (type instanceof BooleanType) {
            primitiveColumn.put(columnHandle, booleanValueProvider(Boolean.valueOf((Boolean) value)));
        } else if (type instanceof TinyintType || type instanceof SmallintType || type instanceof IntegerType || type instanceof BigintType) {
            primitiveColumn.put(columnHandle, longValueProvider(Long.parseLong(value.toString())));
        } else if (type instanceof DoubleType) {
            primitiveColumn.put(columnHandle, doubleValueProvider(Double.parseDouble(value.toString())));
        } else if (type instanceof RealType) {
            primitiveColumn.put(columnHandle, longValueProvider(Float.floatToIntBits((Float.parseFloat(value.toString())))));
        } else if (type instanceof VarbinaryType) {
            primitiveColumn.put(columnHandle, bytesValueProvider((byte[]) value));
        } else if (type instanceof VarcharType) {
            primitiveColumn.put(columnHandle, bytesValueProvider(value.toString().getBytes()));
        } else if (type instanceof DateType) {
            primitiveColumn.put(columnHandle, longValueProvider(((Date) value).getTime()));
        } else if (type instanceof TimeType) {
            primitiveColumn.put(columnHandle, longValueProvider(((Time) value).getTime()));
        } else if (type instanceof TimestampType) {
            primitiveColumn.put(columnHandle, longValueProvider(((Timestamp) value).getTime()));
        } else {
            primitiveColumn.put(columnHandle, bytesValueProvider(value.toString().getBytes()));
        }
    }
    return Optional.of(primitiveColumn);
}

Example 10 with TimestampType

use of io.prestosql.spi.type.TimestampType in project hetu-core by openlookeng.

the class HivePartitionManager method getFilteredPartitionNames.

private List<String> getFilteredPartitionNames(SemiTransactionalHiveMetastore metastore, HiveIdentity identity, SchemaTableName tableName, List<HiveColumnHandle> partitionKeys, TupleDomain<ColumnHandle> effectivePredicate, Table table) {
    checkArgument(effectivePredicate.getDomains().isPresent());
    List<String> filter = new ArrayList<>();
    for (HiveColumnHandle partitionKey : partitionKeys) {
        Domain domain = effectivePredicate.getDomains().get().get(partitionKey);
        if (domain != null && domain.isNullableSingleValue()) {
            Object value = domain.getNullableSingleValue();
            Type type = domain.getType();
            if (value == null) {
                filter.add(HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION);
            } else if (type instanceof CharType) {
                Slice slice = (Slice) value;
                filter.add(padSpaces(slice, (CharType) type).toStringUtf8());
            } else if (type instanceof VarcharType) {
                Slice slice = (Slice) value;
                filter.add(slice.toStringUtf8());
            } else // unless we know that all partition values use the canonical Java representation.
            if (!assumeCanonicalPartitionKeys) {
                filter.add(PARTITION_VALUE_WILDCARD);
            } else if (type instanceof DecimalType && !((DecimalType) type).isShort()) {
                Slice slice = (Slice) value;
                filter.add(Decimals.toString(slice, ((DecimalType) type).getScale()));
            } else if (type instanceof DecimalType && ((DecimalType) type).isShort()) {
                filter.add(Decimals.toString((long) value, ((DecimalType) type).getScale()));
            } else if (type instanceof DateType) {
                DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.date().withZoneUTC();
                filter.add(dateTimeFormatter.print(TimeUnit.DAYS.toMillis((long) value)));
            } else if (type instanceof TimestampType) {
                // we don't have time zone info, so just add a wildcard
                filter.add(PARTITION_VALUE_WILDCARD);
            } else if (type instanceof TinyintType || type instanceof SmallintType || type instanceof IntegerType || type instanceof BigintType || type instanceof DoubleType || type instanceof RealType || type instanceof BooleanType) {
                filter.add(value.toString());
            } else {
                throw new PrestoException(NOT_SUPPORTED, format("Unsupported partition key type: %s", type.getDisplayName()));
            }
        } else {
            filter.add(PARTITION_VALUE_WILDCARD);
        }
    }
    // fetch the partition names
    return metastore.getPartitionNamesByParts(identity, tableName.getSchemaName(), tableName.getTableName(), filter, table).orElseThrow(() -> new TableNotFoundException(tableName));
}