Search in sources :

Example 11 with SqlDate

use of io.prestosql.spi.type.SqlDate in project boostkit-bigdata by kunpengcompute.

the class AbstractTestHive method assertGetRecords.

protected void assertGetRecords(HiveStorageFormat hiveStorageFormat, ConnectorTableMetadata tableMetadata, HiveSplit hiveSplit, ConnectorPageSource pageSource, List<? extends ColumnHandle> columnHandles) throws IOException {
    try {
        MaterializedResult result = materializeSourceDataStream(newSession(), pageSource, getTypes(columnHandles));
        assertPageSourceType(pageSource, hiveStorageFormat);
        ImmutableMap<String, Integer> columnIndex = indexColumns(tableMetadata);
        long rowNumber = 0;
        long completedBytes = 0;
        for (MaterializedRow row : result) {
            try {
                assertValueTypes(row, tableMetadata.getColumns());
            } catch (RuntimeException e) {
                throw new RuntimeException("row " + rowNumber, e);
            }
            rowNumber++;
            Integer index;
            Object value;
            // STRING
            index = columnIndex.get("t_string");
            value = row.getField(index);
            if (rowNumber % 19 == 0) {
                assertNull(value);
            } else if (rowNumber % 19 == 1) {
                assertEquals(value, "");
            } else {
                assertEquals(value, "test");
            }
            // NUMBERS
            assertEquals(row.getField(columnIndex.get("t_tinyint")), (byte) (1 + rowNumber));
            assertEquals(row.getField(columnIndex.get("t_smallint")), (short) (2 + rowNumber));
            assertEquals(row.getField(columnIndex.get("t_int")), (int) (3 + rowNumber));
            index = columnIndex.get("t_bigint");
            if ((rowNumber % 13) == 0) {
                assertNull(row.getField(index));
            } else {
                assertEquals(row.getField(index), 4 + rowNumber);
            }
            assertEquals((Float) row.getField(columnIndex.get("t_float")), 5.1f + rowNumber, 0.001);
            assertEquals(row.getField(columnIndex.get("t_double")), 6.2 + rowNumber);
            // BOOLEAN
            index = columnIndex.get("t_boolean");
            if ((rowNumber % 3) == 2) {
                assertNull(row.getField(index));
            } else {
                assertEquals(row.getField(index), (rowNumber % 3) != 0);
            }
            // TIMESTAMP
            index = columnIndex.get("t_timestamp");
            if (index != null) {
                if ((rowNumber % 17) == 0) {
                    assertNull(row.getField(index));
                } else {
                    SqlTimestamp expected = sqlTimestampOf(2011, 5, 6, 7, 8, 9, 123);
                    assertEquals(row.getField(index), expected);
                }
            }
            // BINARY
            index = columnIndex.get("t_binary");
            if (index != null) {
                if ((rowNumber % 23) == 0) {
                    assertNull(row.getField(index));
                } else {
                    assertEquals(row.getField(index), new SqlVarbinary("test binary".getBytes(UTF_8)));
                }
            }
            // DATE
            index = columnIndex.get("t_date");
            if (index != null) {
                if ((rowNumber % 37) == 0) {
                    assertNull(row.getField(index));
                } else {
                    SqlDate expected = new SqlDate(toIntExact(MILLISECONDS.toDays(new DateTime(2013, 8, 9, 0, 0, 0, UTC).getMillis())));
                    assertEquals(row.getField(index), expected);
                }
            }
            // VARCHAR(50)
            index = columnIndex.get("t_varchar");
            if (index != null) {
                value = row.getField(index);
                if (rowNumber % 39 == 0) {
                    assertNull(value);
                } else if (rowNumber % 39 == 1) {
                    // RCBINARY reads empty VARCHAR as null
                    if (hiveStorageFormat == RCBINARY) {
                        assertNull(value);
                    } else {
                        assertEquals(value, "");
                    }
                } else {
                    assertEquals(value, "test varchar");
                }
            }
            // CHAR(25)
            index = columnIndex.get("t_char");
            if (index != null) {
                value = row.getField(index);
                if ((rowNumber % 41) == 0) {
                    assertNull(value);
                } else {
                    assertEquals(value, (rowNumber % 41) == 1 ? "                         " : "test char                ");
                }
            }
            // MAP<STRING, STRING>
            index = columnIndex.get("t_map");
            if (index != null) {
                if ((rowNumber % 27) == 0) {
                    assertNull(row.getField(index));
                } else {
                    assertEquals(row.getField(index), ImmutableMap.of("test key", "test value"));
                }
            }
            // ARRAY<STRING>
            index = columnIndex.get("t_array_string");
            if (index != null) {
                if ((rowNumber % 29) == 0) {
                    assertNull(row.getField(index));
                } else {
                    assertEquals(row.getField(index), ImmutableList.of("abc", "xyz", "data"));
                }
            }
            // ARRAY<TIMESTAMP>
            index = columnIndex.get("t_array_timestamp");
            if (index != null) {
                if ((rowNumber % 43) == 0) {
                    assertNull(row.getField(index));
                } else {
                    SqlTimestamp expected = sqlTimestampOf(LocalDateTime.of(2011, 5, 6, 7, 8, 9, 123_000_000));
                    assertEquals(row.getField(index), ImmutableList.of(expected));
                }
            }
            // ARRAY<STRUCT<s_string: STRING, s_double:DOUBLE>>
            index = columnIndex.get("t_array_struct");
            if (index != null) {
                if ((rowNumber % 31) == 0) {
                    assertNull(row.getField(index));
                } else {
                    List<Object> expected1 = ImmutableList.of("test abc", 0.1);
                    List<Object> expected2 = ImmutableList.of("test xyz", 0.2);
                    assertEquals(row.getField(index), ImmutableList.of(expected1, expected2));
                }
            }
            // STRUCT<s_string: STRING, s_double:DOUBLE>
            index = columnIndex.get("t_struct");
            if (index != null) {
                if ((rowNumber % 31) == 0) {
                    assertNull(row.getField(index));
                } else {
                    assertTrue(row.getField(index) instanceof List);
                    List<?> values = (List<?>) row.getField(index);
                    assertEquals(values.size(), 2);
                    assertEquals(values.get(0), "test abc");
                    assertEquals(values.get(1), 0.1);
                }
            }
            // MAP<INT, ARRAY<STRUCT<s_string: STRING, s_double:DOUBLE>>>
            index = columnIndex.get("t_complex");
            if (index != null) {
                if ((rowNumber % 33) == 0) {
                    assertNull(row.getField(index));
                } else {
                    List<Object> expected1 = ImmutableList.of("test abc", 0.1);
                    List<Object> expected2 = ImmutableList.of("test xyz", 0.2);
                    assertEquals(row.getField(index), ImmutableMap.of(1, ImmutableList.of(expected1, expected2)));
                }
            }
            // NEW COLUMN
            assertNull(row.getField(columnIndex.get("new_column")));
            long newCompletedBytes = pageSource.getCompletedBytes();
            assertTrue(newCompletedBytes >= completedBytes);
            assertTrue(newCompletedBytes <= hiveSplit.getLength());
            completedBytes = newCompletedBytes;
        }
        assertTrue(completedBytes <= hiveSplit.getLength());
        assertEquals(rowNumber, 100);
    } finally {
        pageSource.close();
    }
}
Also used : SqlVarbinary(io.prestosql.spi.type.SqlVarbinary) SqlTimestamp(io.prestosql.spi.type.SqlTimestamp) LocalDateTime(java.time.LocalDateTime) DateTime(org.joda.time.DateTime) SqlDate(io.prestosql.spi.type.SqlDate) Lists.newArrayList(com.google.common.collect.Lists.newArrayList) ImmutableList(com.google.common.collect.ImmutableList) Collectors.toList(java.util.stream.Collectors.toList) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) List(java.util.List) MaterializedResult(io.prestosql.testing.MaterializedResult) MaterializedRow(io.prestosql.testing.MaterializedRow)

Example 12 with SqlDate

use of io.prestosql.spi.type.SqlDate in project hetu-core by openlookeng.

the class LiteralEncoder method toExpression.

public Expression toExpression(Object inputObject, Type type) {
    requireNonNull(type, "type is null");
    Object object = inputObject;
    if (object instanceof Expression) {
        return (Expression) object;
    }
    if (object == null) {
        if (type.equals(UNKNOWN)) {
            return new NullLiteral();
        }
        return new Cast(new NullLiteral(), type.getTypeSignature().toString(), false, true);
    }
    // AbstractIntType internally uses long as javaType. So specially handled for AbstractIntType types.
    Class<?> wrap = Primitives.wrap(type.getJavaType());
    checkArgument(wrap.isInstance(object) || (type instanceof AbstractIntType && wrap == Long.class && Integer.class.isInstance(object)), "object.getClass (%s) and type.getJavaType (%s) do not agree", object.getClass(), type.getJavaType());
    if (type.equals(TINYINT)) {
        return new GenericLiteral("TINYINT", object.toString());
    }
    if (type.equals(SMALLINT)) {
        return new GenericLiteral("SMALLINT", object.toString());
    }
    if (type.equals(INTEGER)) {
        return new LongLiteral(object.toString());
    }
    if (type.equals(BIGINT)) {
        LongLiteral expression = new LongLiteral(object.toString());
        if (expression.getValue() >= Integer.MIN_VALUE && expression.getValue() <= Integer.MAX_VALUE) {
            return new GenericLiteral("BIGINT", object.toString());
        }
        return new LongLiteral(object.toString());
    }
    if (type.equals(DOUBLE)) {
        Double value = (Double) object;
        // When changing this, don't forget about similar code for REAL below
        if (value.isNaN()) {
            return new FunctionCallBuilder(metadata).setName(QualifiedName.of("nan")).build();
        }
        if (value.equals(Double.NEGATIVE_INFINITY)) {
            return ArithmeticUnaryExpression.negative(new FunctionCallBuilder(metadata).setName(QualifiedName.of("infinity")).build());
        }
        if (value.equals(Double.POSITIVE_INFINITY)) {
            return new FunctionCallBuilder(metadata).setName(QualifiedName.of("infinity")).build();
        }
        return new DoubleLiteral(object.toString());
    }
    if (type.equals(REAL)) {
        Float value = intBitsToFloat(((Long) object).intValue());
        // WARNING for ORC predicate code as above (for double)
        if (value.isNaN()) {
            return new Cast(new FunctionCallBuilder(metadata).setName(QualifiedName.of("nan")).build(), StandardTypes.REAL);
        }
        if (value.equals(Float.NEGATIVE_INFINITY)) {
            return ArithmeticUnaryExpression.negative(new Cast(new FunctionCallBuilder(metadata).setName(QualifiedName.of("infinity")).build(), StandardTypes.REAL));
        }
        if (value.equals(Float.POSITIVE_INFINITY)) {
            return new Cast(new FunctionCallBuilder(metadata).setName(QualifiedName.of("infinity")).build(), StandardTypes.REAL);
        }
        return new GenericLiteral("REAL", value.toString());
    }
    if (type instanceof DecimalType) {
        String string;
        if (isShortDecimal(type)) {
            string = Decimals.toString((long) object, ((DecimalType) type).getScale());
        } else {
            string = Decimals.toString((Slice) object, ((DecimalType) type).getScale());
        }
        return new Cast(new DecimalLiteral(string), type.getDisplayName());
    }
    if (type instanceof VarcharType) {
        VarcharType varcharType = (VarcharType) type;
        Slice value = (Slice) object;
        StringLiteral stringLiteral = new StringLiteral(value.toStringUtf8());
        if (!varcharType.isUnbounded() && varcharType.getBoundedLength() == SliceUtf8.countCodePoints(value)) {
            return stringLiteral;
        }
        return new Cast(stringLiteral, type.getDisplayName(), false, true);
    }
    if (type instanceof CharType) {
        StringLiteral stringLiteral = new StringLiteral(((Slice) object).toStringUtf8());
        return new Cast(stringLiteral, type.getDisplayName(), false, true);
    }
    if (type.equals(BOOLEAN)) {
        return new BooleanLiteral(object.toString());
    }
    if (type.equals(DATE)) {
        return new GenericLiteral("DATE", new SqlDate(toIntExact((Long) object)).toString());
    }
    if (type.equals(TIMESTAMP)) {
        return new GenericLiteral("TIMESTAMP", new SqlTimestamp((Long) object).toString());
    }
    if (object instanceof Block) {
        SliceOutput output = new DynamicSliceOutput(toIntExact(((Block) object).getSizeInBytes()));
        BlockSerdeUtil.writeBlock(metadata.getFunctionAndTypeManager().getBlockEncodingSerde(), output, (Block) object);
        object = output.slice();
    // This if condition will evaluate to true: object instanceof Slice && !type.equals(VARCHAR)
    }
    Signature signature = LiteralFunction.getLiteralFunctionSignature(type);
    Type argumentType = typeForLiteralFunctionArgument(type);
    Expression argument;
    if (object instanceof Slice) {
        // HACK: we need to serialize VARBINARY in a format that can be embedded in an expression to be
        // able to encode it in the plan that gets sent to workers.
        // We do this by transforming the in-memory varbinary into a call to from_base64(<base64-encoded value>)
        Slice encoded = VarbinaryFunctions.toBase64((Slice) object);
        argument = new FunctionCallBuilder(metadata).setName(QualifiedName.of("from_base64")).addArgument(VARCHAR, new StringLiteral(encoded.toStringUtf8())).build();
    } else {
        argument = toExpression(object, argumentType);
    }
    return new FunctionCallBuilder(metadata).setName(QualifiedName.of(signature.getNameSuffix())).addArgument(argumentType, argument).build();
}
Also used : Cast(io.prestosql.sql.tree.Cast) SliceOutput(io.airlift.slice.SliceOutput) DynamicSliceOutput(io.airlift.slice.DynamicSliceOutput) VarcharType(io.prestosql.spi.type.VarcharType) BooleanLiteral(io.prestosql.sql.tree.BooleanLiteral) SqlTimestamp(io.prestosql.spi.type.SqlTimestamp) GenericLiteral(io.prestosql.sql.tree.GenericLiteral) DecimalLiteral(io.prestosql.sql.tree.DecimalLiteral) DynamicSliceOutput(io.airlift.slice.DynamicSliceOutput) LongLiteral(io.prestosql.sql.tree.LongLiteral) AbstractIntType(io.prestosql.spi.type.AbstractIntType) Float.intBitsToFloat(java.lang.Float.intBitsToFloat) DecimalType(io.prestosql.spi.type.DecimalType) Type(io.prestosql.spi.type.Type) CharType(io.prestosql.spi.type.CharType) AbstractIntType(io.prestosql.spi.type.AbstractIntType) VarcharType(io.prestosql.spi.type.VarcharType) StringLiteral(io.prestosql.sql.tree.StringLiteral) ArithmeticUnaryExpression(io.prestosql.sql.tree.ArithmeticUnaryExpression) RowExpression(io.prestosql.spi.relation.RowExpression) Expression(io.prestosql.sql.tree.Expression) Slice(io.airlift.slice.Slice) SqlDate(io.prestosql.spi.type.SqlDate) TypeSignature(io.prestosql.spi.type.TypeSignature) Signature(io.prestosql.spi.function.Signature) DecimalType(io.prestosql.spi.type.DecimalType) Block(io.prestosql.spi.block.Block) DoubleLiteral(io.prestosql.sql.tree.DoubleLiteral) CharType(io.prestosql.spi.type.CharType) NullLiteral(io.prestosql.sql.tree.NullLiteral)

Example 13 with SqlDate

use of io.prestosql.spi.type.SqlDate in project hetu-core by openlookeng.

the class TestArrayOperators method sqlDate.

private static SqlDate sqlDate(String dateString) throws ParseException {
    DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
    dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
    return new SqlDate(toIntExact(TimeUnit.MILLISECONDS.toDays(dateFormat.parse(dateString).getTime())));
}
Also used : DateFormat(java.text.DateFormat) SimpleDateFormat(java.text.SimpleDateFormat) SqlDate(io.prestosql.spi.type.SqlDate) SimpleDateFormat(java.text.SimpleDateFormat)

Example 14 with SqlDate

use of io.prestosql.spi.type.SqlDate in project hetu-core by openlookeng.

the class TestDate method testLeast.

@Test
public void testLeast() {
    int days = (int) TimeUnit.MILLISECONDS.toDays(new DateTime(2012, 5, 23, 0, 0, UTC).getMillis());
    assertFunction("least(DATE '2013-03-30', DATE '2012-05-23')", DATE, new SqlDate(days));
    assertFunction("least(DATE '2013-03-30', DATE '2012-05-23', DATE '2012-06-01')", DATE, new SqlDate(days));
}
Also used : SqlDate(io.prestosql.spi.type.SqlDate) DateTime(org.joda.time.DateTime) Test(org.testng.annotations.Test)

Example 15 with SqlDate

use of io.prestosql.spi.type.SqlDate in project hetu-core by openlookeng.

the class TestDate method testLiteral.

@Test
public void testLiteral() {
    long millis = new DateTime(2001, 1, 22, 0, 0, UTC).getMillis();
    assertFunction("DATE '2001-1-22'", DATE, new SqlDate((int) TimeUnit.MILLISECONDS.toDays(millis)));
}
Also used : SqlDate(io.prestosql.spi.type.SqlDate) DateTime(org.joda.time.DateTime) Test(org.testng.annotations.Test)

Aggregations

SqlDate (io.prestosql.spi.type.SqlDate)19 SqlTimestamp (io.prestosql.spi.type.SqlTimestamp)10 SqlVarbinary (io.prestosql.spi.type.SqlVarbinary)10 DecimalType (io.prestosql.spi.type.DecimalType)9 Type (io.prestosql.spi.type.Type)8 ImmutableList (com.google.common.collect.ImmutableList)7 CharType (io.prestosql.spi.type.CharType)7 SqlDecimal (io.prestosql.spi.type.SqlDecimal)7 List (java.util.List)7 Slice (io.airlift.slice.Slice)6 VarcharType (io.prestosql.spi.type.VarcharType)6 Collectors.toList (java.util.stream.Collectors.toList)6 DateTime (org.joda.time.DateTime)6 BlockBuilder (io.prestosql.spi.block.BlockBuilder)5 ArrayList (java.util.ArrayList)5 Test (org.testng.annotations.Test)5 ImmutableMap (com.google.common.collect.ImmutableMap)4 Lists.newArrayList (com.google.common.collect.Lists.newArrayList)4 ArrayType (io.prestosql.spi.type.ArrayType)4 RowType (io.prestosql.spi.type.RowType)4