Examples with SqlDate com.facebook.presto.common.type.SqlDate used on opensource projects

Example 1 with SqlDate

use of com.facebook.presto.common.type.SqlDate in project presto by prestodb.

the class TestShardWriter method testWriter.

@Test(dataProvider = "useOptimizedOrcWriter")
public void testWriter(boolean useOptimizedOrcWriter) throws Exception {
    FunctionAndTypeManager functionAndTypeManager = createTestFunctionAndTypeManager();
    List<Long> columnIds = ImmutableList.of(1L, 2L, 4L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L);
    ArrayType arrayType = new ArrayType(BIGINT);
    ArrayType arrayOfArrayType = new ArrayType(arrayType);
    Type mapType = functionAndTypeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.of(createVarcharType(10).getTypeSignature()), TypeSignatureParameter.of(BOOLEAN.getTypeSignature())));
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(10), VARBINARY, DOUBLE, BOOLEAN, arrayType, mapType, arrayOfArrayType, TIMESTAMP, TIME, DATE);
    File file = new File(directory, System.nanoTime() + ".orc");
    byte[] bytes1 = octets(0x00, 0xFE, 0xFF);
    byte[] bytes3 = octets(0x01, 0x02, 0x19, 0x80);
    long timestampValue = sqlTimestampOf(2002, 4, 6, 7, 8, 9, 0, UTC, UTC_KEY, SESSION).getMillisUtc();
    long timeValue = new SqlTime(NANOSECONDS.toMillis(new DateTime(2004, 11, 29, 0, 0, 0, 0, UTC).toLocalTime().getMillisOfDay())).getMillis();
    DateTime date = new DateTime(2001, 11, 22, 0, 0, 0, 0, UTC);
    int dateValue = new SqlDate(Days.daysBetween(new DateTime(0, ISOChronology.getInstanceUTC()), date).getDays()).getDays();
    RowPagesBuilder rowPagesBuilder = RowPagesBuilder.rowPagesBuilder(columnTypes).row(123L, "hello", wrappedBuffer(bytes1), 123.456, true, arrayBlockOf(BIGINT, 1, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)), timestampValue, timeValue, dateValue).row(null, "world", null, Double.POSITIVE_INFINITY, null, arrayBlockOf(BIGINT, 3, null), mapBlockOf(createVarcharType(5), BOOLEAN, "k2", null), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7)), timestampValue, timeValue, dateValue).row(456L, "bye \u2603", wrappedBuffer(bytes3), Double.NaN, false, arrayBlockOf(BIGINT), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT)), timestampValue, timeValue, dateValue);
    try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(new EmptyClassLoader());
        FileWriter writer = createFileWriter(columnIds, columnTypes, file)) {
        writer.appendPages(rowPagesBuilder.build());
    }
    try (OrcDataSource dataSource = fileOrcDataSource(file)) {
        OrcBatchRecordReader reader = createReader(dataSource, columnIds, columnTypes);
        assertEquals(reader.getReaderRowCount(), 3);
        assertEquals(reader.getReaderPosition(), 0);
        assertEquals(reader.getFileRowCount(), reader.getReaderRowCount());
        assertEquals(reader.getFilePosition(), reader.getFilePosition());
        assertEquals(reader.nextBatch(), 3);
        assertEquals(reader.getReaderPosition(), 0);
        assertEquals(reader.getFilePosition(), reader.getFilePosition());
        Block column0 = reader.readBlock(0);
        assertEquals(column0.isNull(0), false);
        assertEquals(column0.isNull(1), true);
        assertEquals(column0.isNull(2), false);
        assertEquals(BIGINT.getLong(column0, 0), 123L);
        assertEquals(BIGINT.getLong(column0, 2), 456L);
        Block column1 = reader.readBlock(1);
        assertEquals(createVarcharType(10).getSlice(column1, 0), utf8Slice("hello"));
        assertEquals(createVarcharType(10).getSlice(column1, 1), utf8Slice("world"));
        assertEquals(createVarcharType(10).getSlice(column1, 2), utf8Slice("bye \u2603"));
        Block column2 = reader.readBlock(2);
        assertEquals(VARBINARY.getSlice(column2, 0), wrappedBuffer(bytes1));
        assertEquals(column2.isNull(1), true);
        assertEquals(VARBINARY.getSlice(column2, 2), wrappedBuffer(bytes3));
        Block column3 = reader.readBlock(3);
        assertEquals(column3.isNull(0), false);
        assertEquals(column3.isNull(1), false);
        assertEquals(column3.isNull(2), false);
        assertEquals(DOUBLE.getDouble(column3, 0), 123.456);
        assertEquals(DOUBLE.getDouble(column3, 1), Double.POSITIVE_INFINITY);
        assertEquals(DOUBLE.getDouble(column3, 2), Double.NaN);
        Block column4 = reader.readBlock(4);
        assertEquals(column4.isNull(0), false);
        assertEquals(column4.isNull(1), true);
        assertEquals(column4.isNull(2), false);
        assertEquals(BOOLEAN.getBoolean(column4, 0), true);
        assertEquals(BOOLEAN.getBoolean(column4, 2), false);
        Block column5 = reader.readBlock(5);
        assertEquals(column5.getPositionCount(), 3);
        assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 0), arrayBlockOf(BIGINT, 1, 2)));
        assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 1), arrayBlockOf(BIGINT, 3, null)));
        assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 2), arrayBlockOf(BIGINT)));
        Block column6 = reader.readBlock(6);
        assertEquals(column6.getPositionCount(), 3);
        assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column6, 0), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true)));
        Block object = arrayType.getObject(column6, 1);
        Block k2 = mapBlockOf(createVarcharType(5), BOOLEAN, "k2", null);
        assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, object, k2));
        assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column6, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false)));
        Block column7 = reader.readBlock(7);
        assertEquals(column7.getPositionCount(), 3);
        assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 0), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5))));
        assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 1), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7))));
        assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 2), arrayBlockOf(arrayType, arrayBlockOf(BIGINT))));
        Block column8 = reader.readBlock(8);
        assertEquals(TIMESTAMP.getLong(column8, 0), timestampValue);
        assertEquals(TIMESTAMP.getLong(column8, 1), timestampValue);
        assertEquals(TIMESTAMP.getLong(column8, 2), timestampValue);
        Block column9 = reader.readBlock(9);
        assertEquals(TIME.getLong(column9, 0), timeValue);
        assertEquals(TIME.getLong(column9, 1), timeValue);
        assertEquals(TIME.getLong(column9, 2), timeValue);
        Block column10 = reader.readBlock(10);
        assertEquals(DATE.getLong(column10, 0), dateValue);
        assertEquals(DATE.getLong(column10, 1), dateValue);
        assertEquals(DATE.getLong(column10, 2), dateValue);
        assertEquals(reader.nextBatch(), -1);
        assertEquals(reader.getReaderPosition(), 3);
        assertEquals(reader.getFilePosition(), reader.getFilePosition());
        OrcFileMetadata orcFileMetadata = METADATA_CODEC.fromJson(reader.getUserMetadata().get(OrcFileMetadata.KEY).getBytes());
        assertEquals(orcFileMetadata, new OrcFileMetadata(ImmutableMap.<Long, TypeSignature>builder().put(1L, BIGINT.getTypeSignature()).put(2L, createVarcharType(10).getTypeSignature()).put(4L, VARBINARY.getTypeSignature()).put(6L, DOUBLE.getTypeSignature()).put(7L, BOOLEAN.getTypeSignature()).put(8L, arrayType.getTypeSignature()).put(9L, mapType.getTypeSignature()).put(10L, arrayOfArrayType.getTypeSignature()).put(11L, TIMESTAMP.getTypeSignature()).put(12L, TIME.getTypeSignature()).put(13L, DATE.getTypeSignature()).build()));
    }
    File crcFile = new File(file.getParentFile(), "." + file.getName() + ".crc");
    assertFalse(crcFile.exists());
    // Test unsupported types
    for (Type type : ImmutableList.of(TIMESTAMP_WITH_TIME_ZONE, RowType.anonymous(ImmutableList.of(BIGINT, DOUBLE)))) {
        try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(new EmptyClassLoader())) {
            createFileWriter(ImmutableList.of(1L), ImmutableList.of(type), file);
            fail();
        } catch (PrestoException e) {
            assertTrue(e.getMessage().toLowerCase(ENGLISH).contains("type"));
        }
    }
}

Example 2 with SqlDate

use of com.facebook.presto.common.type.SqlDate in project presto by prestodb.

the class TestRaptorConnector method assertSplitShard.

private void assertSplitShard(Type temporalType, String min, String max, String userTimeZone, int expectedSplits) throws Exception {
    ConnectorSession session = new TestingConnectorSession("user", Optional.of("test"), Optional.empty(), getTimeZoneKey(userTimeZone), ENGLISH, System.currentTimeMillis(), new RaptorSessionProperties(new StorageManagerConfig()).getSessionProperties(), ImmutableMap.of(), true, Optional.empty(), ImmutableSet.of(), Optional.empty(), ImmutableMap.of());
    ConnectorTransactionHandle transaction = connector.beginTransaction(READ_COMMITTED, false);
    connector.getMetadata(transaction).createTable(SESSION, new ConnectorTableMetadata(new SchemaTableName("test", "test"), ImmutableList.of(new ColumnMetadata("id", BIGINT), new ColumnMetadata("time", temporalType)), ImmutableMap.of(TEMPORAL_COLUMN_PROPERTY, "time", TABLE_SUPPORTS_DELTA_DELETE, false)), false);
    connector.commit(transaction);
    ConnectorTransactionHandle txn1 = connector.beginTransaction(READ_COMMITTED, false);
    ConnectorTableHandle handle1 = getTableHandle(connector.getMetadata(txn1), "test");
    ConnectorInsertTableHandle insertTableHandle = connector.getMetadata(txn1).beginInsert(session, handle1);
    ConnectorPageSink raptorPageSink = connector.getPageSinkProvider().createPageSink(txn1, session, insertTableHandle, PageSinkContext.defaultContext());
    Object timestamp1 = null;
    Object timestamp2 = null;
    if (temporalType.equals(TIMESTAMP)) {
        timestamp1 = new SqlTimestamp(parseTimestampLiteral(getTimeZoneKey(userTimeZone), min), getTimeZoneKey(userTimeZone));
        timestamp2 = new SqlTimestamp(parseTimestampLiteral(getTimeZoneKey(userTimeZone), max), getTimeZoneKey(userTimeZone));
    } else if (temporalType.equals(DATE)) {
        timestamp1 = new SqlDate(parseDate(min));
        timestamp2 = new SqlDate(parseDate(max));
    }
    Page inputPage = MaterializedResult.resultBuilder(session, ImmutableList.of(BIGINT, temporalType)).row(1L, timestamp1).row(2L, timestamp2).build().toPage();
    raptorPageSink.appendPage(inputPage);
    Collection<Slice> shards = raptorPageSink.finish().get();
    assertEquals(shards.size(), expectedSplits);
    connector.getMetadata(txn1).dropTable(session, handle1);
    connector.commit(txn1);
}

Example 3 with SqlDate

use of com.facebook.presto.common.type.SqlDate in project presto by prestodb.

the class OrcTester method preprocessWriteValueHive.

private static Object preprocessWriteValueHive(Type type, Object value) {
    if (value == null) {
        return null;
    }
    if (type.equals(BOOLEAN)) {
        return value;
    } else if (type.equals(TINYINT)) {
        return ((Number) value).byteValue();
    } else if (type.equals(SMALLINT)) {
        return ((Number) value).shortValue();
    } else if (type.equals(INTEGER)) {
        return ((Number) value).intValue();
    } else if (type.equals(BIGINT)) {
        return ((Number) value).longValue();
    } else if (type.equals(REAL)) {
        return ((Number) value).floatValue();
    } else if (type.equals(DOUBLE)) {
        return ((Number) value).doubleValue();
    } else if (type instanceof VarcharType) {
        return value;
    } else if (type instanceof CharType) {
        return new HiveChar((String) value, ((CharType) type).getLength());
    } else if (type.equals(VARBINARY)) {
        return ((SqlVarbinary) value).getBytes();
    } else if (type.equals(DATE)) {
        int days = ((SqlDate) value).getDays();
        LocalDate localDate = LocalDate.ofEpochDay(days);
        ZonedDateTime zonedDateTime = localDate.atStartOfDay(ZoneId.systemDefault());
        long millis = SECONDS.toMillis(zonedDateTime.toEpochSecond());
        Date date = new Date(0);
        // millis must be set separately to avoid masking
        date.setTime(millis);
        return date;
    } else if (type.equals(TIMESTAMP)) {
        long millisUtc = (int) ((SqlTimestamp) value).getMillisUtc();
        return new Timestamp(millisUtc);
    } else if (type instanceof DecimalType) {
        return HiveDecimal.create(((SqlDecimal) value).toBigDecimal());
    } else if (type.getTypeSignature().getBase().equals(StandardTypes.ARRAY)) {
        Type elementType = type.getTypeParameters().get(0);
        return ((List<?>) value).stream().map(element -> preprocessWriteValueHive(elementType, element)).collect(toList());
    } else if (type.getTypeSignature().getBase().equals(StandardTypes.MAP)) {
        Type keyType = type.getTypeParameters().get(0);
        Type valueType = type.getTypeParameters().get(1);
        Map<Object, Object> newMap = new HashMap<>();
        for (Entry<?, ?> entry : ((Map<?, ?>) value).entrySet()) {
            newMap.put(preprocessWriteValueHive(keyType, entry.getKey()), preprocessWriteValueHive(valueType, entry.getValue()));
        }
        return newMap;
    } else if (type.getTypeSignature().getBase().equals(StandardTypes.ROW)) {
        List<?> fieldValues = (List<?>) value;
        List<Type> fieldTypes = type.getTypeParameters();
        List<Object> newStruct = new ArrayList<>();
        for (int fieldId = 0; fieldId < fieldValues.size(); fieldId++) {
            newStruct.add(preprocessWriteValueHive(fieldTypes.get(fieldId), fieldValues.get(fieldId)));
        }
        return newStruct;
    }
    throw new IllegalArgumentException("unsupported type: " + type);
}

Example 4 with SqlDate

use of com.facebook.presto.common.type.SqlDate in project presto by prestodb.

the class OrcTester method decodeRecordReaderValue.

private static Object decodeRecordReaderValue(Type type, Object actualValue) {
    if (actualValue instanceof OrcLazyObject) {
        try {
            actualValue = ((OrcLazyObject) actualValue).materialize();
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }
    if (actualValue instanceof BooleanWritable) {
        actualValue = ((BooleanWritable) actualValue).get();
    } else if (actualValue instanceof ByteWritable) {
        actualValue = ((ByteWritable) actualValue).get();
    } else if (actualValue instanceof BytesWritable) {
        actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes());
    } else if (actualValue instanceof DateWritable) {
        actualValue = new SqlDate(((DateWritable) actualValue).getDays());
    } else if (actualValue instanceof DoubleWritable) {
        actualValue = ((DoubleWritable) actualValue).get();
    } else if (actualValue instanceof FloatWritable) {
        actualValue = ((FloatWritable) actualValue).get();
    } else if (actualValue instanceof IntWritable) {
        actualValue = ((IntWritable) actualValue).get();
    } else if (actualValue instanceof HiveCharWritable) {
        actualValue = ((HiveCharWritable) actualValue).getPaddedValue().toString();
    } else if (actualValue instanceof LongWritable) {
        actualValue = ((LongWritable) actualValue).get();
    } else if (actualValue instanceof ShortWritable) {
        actualValue = ((ShortWritable) actualValue).get();
    } else if (actualValue instanceof HiveDecimalWritable) {
        DecimalType decimalType = (DecimalType) type;
        HiveDecimalWritable writable = (HiveDecimalWritable) actualValue;
        // writable messes with the scale so rescale the values to the Presto type
        BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale());
        actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale());
    } else if (actualValue instanceof Text) {
        actualValue = actualValue.toString();
    } else if (actualValue instanceof TimestampWritable) {
        TimestampWritable timestamp = (TimestampWritable) actualValue;
        actualValue = sqlTimestampOf((timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), SESSION);
    } else if (actualValue instanceof OrcStruct) {
        List<Object> fields = new ArrayList<>();
        OrcStruct structObject = (OrcStruct) actualValue;
        for (int fieldId = 0; fieldId < structObject.getNumFields(); fieldId++) {
            fields.add(OrcUtil.getFieldValue(structObject, fieldId));
        }
        actualValue = decodeRecordReaderStruct(type, fields);
    } else if (actualValue instanceof com.facebook.hive.orc.OrcStruct) {
        List<Object> fields = new ArrayList<>();
        com.facebook.hive.orc.OrcStruct structObject = (com.facebook.hive.orc.OrcStruct) actualValue;
        for (int fieldId = 0; fieldId < structObject.getNumFields(); fieldId++) {
            fields.add(structObject.getFieldValue(fieldId));
        }
        actualValue = decodeRecordReaderStruct(type, fields);
    } else if (actualValue instanceof List) {
        actualValue = decodeRecordReaderList(type, ((List<?>) actualValue));
    } else if (actualValue instanceof Map) {
        actualValue = decodeRecordReaderMap(type, (Map<?, ?>) actualValue);
    }
    return actualValue;
}

Example 5 with SqlDate

use of com.facebook.presto.common.type.SqlDate in project presto by prestodb.

the class LiteralEncoder method toExpression.

@Deprecated
public Expression toExpression(Object object, Type type, boolean typeOnly) {
    requireNonNull(type, "type is null");
    if (object instanceof Expression) {
        return (Expression) object;
    }
    if (object == null) {
        if (type.equals(UNKNOWN)) {
            return new NullLiteral();
        }
        return new Cast(new NullLiteral(), type.getTypeSignature().toString(), false, typeOnly);
    }
    if (type.equals(TINYINT)) {
        return new GenericLiteral("TINYINT", object.toString());
    }
    if (type.equals(SMALLINT)) {
        return new GenericLiteral("SMALLINT", object.toString());
    }
    if (type.equals(INTEGER)) {
        return new LongLiteral(object.toString());
    }
    if (type.equals(BIGINT)) {
        LongLiteral expression = new LongLiteral(object.toString());
        if (expression.getValue() >= Integer.MIN_VALUE && expression.getValue() <= Integer.MAX_VALUE) {
            return new GenericLiteral("BIGINT", object.toString());
        }
        return new LongLiteral(object.toString());
    }
    checkArgument(Primitives.wrap(type.getJavaType()).isInstance(object), "object.getClass (%s) and type.getJavaType (%s) do not agree", object.getClass(), type.getJavaType());
    if (type.equals(DOUBLE)) {
        Double value = (Double) object;
        // When changing this, don't forget about similar code for REAL below
        if (value.isNaN()) {
            return new FunctionCall(QualifiedName.of("nan"), ImmutableList.of());
        }
        if (value.equals(Double.NEGATIVE_INFINITY)) {
            return ArithmeticUnaryExpression.negative(new FunctionCall(QualifiedName.of("infinity"), ImmutableList.of()));
        }
        if (value.equals(Double.POSITIVE_INFINITY)) {
            return new FunctionCall(QualifiedName.of("infinity"), ImmutableList.of());
        }
        return new DoubleLiteral(object.toString());
    }
    if (type.equals(REAL)) {
        Float value = intBitsToFloat(((Long) object).intValue());
        // WARNING for ORC predicate code as above (for double)
        if (value.isNaN()) {
            return new Cast(new FunctionCall(QualifiedName.of("nan"), ImmutableList.of()), StandardTypes.REAL);
        }
        if (value.equals(Float.NEGATIVE_INFINITY)) {
            return ArithmeticUnaryExpression.negative(new Cast(new FunctionCall(QualifiedName.of("infinity"), ImmutableList.of()), StandardTypes.REAL));
        }
        if (value.equals(Float.POSITIVE_INFINITY)) {
            return new Cast(new FunctionCall(QualifiedName.of("infinity"), ImmutableList.of()), StandardTypes.REAL);
        }
        return new GenericLiteral("REAL", value.toString());
    }
    if (type instanceof DecimalType) {
        String string;
        if (isShortDecimal(type)) {
            string = Decimals.toString((long) object, ((DecimalType) type).getScale());
        } else {
            string = Decimals.toString((Slice) object, ((DecimalType) type).getScale());
        }
        return new Cast(new DecimalLiteral(string), type.getDisplayName());
    }
    if (type instanceof VarcharType) {
        VarcharType varcharType = (VarcharType) type;
        Slice value = (Slice) object;
        StringLiteral stringLiteral = new StringLiteral(value.toStringUtf8());
        if (!varcharType.isUnbounded() && varcharType.getLengthSafe() == SliceUtf8.countCodePoints(value)) {
            return stringLiteral;
        }
        return new Cast(stringLiteral, type.getDisplayName(), false, typeOnly);
    }
    if (type instanceof CharType) {
        StringLiteral stringLiteral = new StringLiteral(((Slice) object).toStringUtf8());
        return new Cast(stringLiteral, type.getDisplayName(), false, typeOnly);
    }
    if (type.equals(BOOLEAN)) {
        return new BooleanLiteral(object.toString());
    }
    if (type.equals(DATE)) {
        return new GenericLiteral("DATE", new SqlDate(toIntExact((Long) object)).toString());
    }
    if (object instanceof Block) {
        SliceOutput output = new DynamicSliceOutput(toIntExact(((Block) object).getSizeInBytes()));
        BlockSerdeUtil.writeBlock(blockEncodingSerde, output, (Block) object);
        object = output.slice();
    // This if condition will evaluate to true: object instanceof Slice && !type.equals(VARCHAR)
    }
    if (type instanceof EnumType) {
        return new EnumLiteral(type.getTypeSignature().toString(), object);
    }
    Signature signature = getMagicLiteralFunctionSignature(type);
    if (object instanceof Slice) {
        // HACK: we need to serialize VARBINARY in a format that can be embedded in an expression to be
        // able to encode it in the plan that gets sent to workers.
        // We do this by transforming the in-memory varbinary into a call to from_base64(<base64-encoded value>)
        FunctionCall fromBase64 = new FunctionCall(QualifiedName.of("from_base64"), ImmutableList.of(new StringLiteral(VarbinaryFunctions.toBase64((Slice) object).toStringUtf8())));
        return new FunctionCall(QualifiedName.of(signature.getNameSuffix()), ImmutableList.of(fromBase64));
    }
    Expression rawLiteral = toExpression(object, typeForMagicLiteral(type), typeOnly);
    return new FunctionCall(QualifiedName.of(signature.getNameSuffix()), ImmutableList.of(rawLiteral));
}