Examples with SqlVarbinary io.prestosql.spi.type.SqlVarbinary used on opensource projects

Example 26 with SqlVarbinary

use of io.prestosql.spi.type.SqlVarbinary in project hetu-core by openlookeng.

the class OrcTester method preprocessWriteValueHive.

private static Object preprocessWriteValueHive(Type type, Object value) {
    if (value == null) {
        return null;
    }
    if (type.equals(BOOLEAN)) {
        return value;
    }
    if (type.equals(TINYINT)) {
        return ((Number) value).byteValue();
    }
    if (type.equals(SMALLINT)) {
        return ((Number) value).shortValue();
    }
    if (type.equals(INTEGER)) {
        return ((Number) value).intValue();
    }
    if (type.equals(BIGINT)) {
        return ((Number) value).longValue();
    }
    if (type.equals(REAL)) {
        return ((Number) value).floatValue();
    }
    if (type.equals(DOUBLE)) {
        return ((Number) value).doubleValue();
    }
    if (type instanceof VarcharType) {
        return value;
    }
    if (type instanceof CharType) {
        return new HiveChar((String) value, ((CharType) type).getLength());
    }
    if (type.equals(VARBINARY)) {
        return ((SqlVarbinary) value).getBytes();
    }
    if (type.equals(DATE)) {
        return Date.ofEpochDay(((SqlDate) value).getDays());
    }
    if (type.equals(TIMESTAMP)) {
        return Timestamp.ofEpochMilli(((SqlTimestamp) value).getMillis());
    }
    if (type instanceof DecimalType) {
        return HiveDecimal.create(((SqlDecimal) value).toBigDecimal());
    }
    if (type.getTypeSignature().getBase().equals(StandardTypes.ARRAY)) {
        Type elementType = type.getTypeParameters().get(0);
        return ((List<?>) value).stream().map(element -> preprocessWriteValueHive(elementType, element)).collect(toList());
    }
    if (type.getTypeSignature().getBase().equals(StandardTypes.MAP)) {
        Type keyType = type.getTypeParameters().get(0);
        Type valueType = type.getTypeParameters().get(1);
        Map<Object, Object> newMap = new HashMap<>();
        for (Entry<?, ?> entry : ((Map<?, ?>) value).entrySet()) {
            newMap.put(preprocessWriteValueHive(keyType, entry.getKey()), preprocessWriteValueHive(valueType, entry.getValue()));
        }
        return newMap;
    }
    if (type.getTypeSignature().getBase().equals(StandardTypes.ROW)) {
        List<?> fieldValues = (List<?>) value;
        List<Type> fieldTypes = type.getTypeParameters();
        List<Object> newStruct = new ArrayList<>();
        for (int fieldId = 0; fieldId < fieldValues.size(); fieldId++) {
            newStruct.add(preprocessWriteValueHive(fieldTypes.get(fieldId), fieldValues.get(fieldId)));
        }
        return newStruct;
    }
    throw new IllegalArgumentException("unsupported type: " + type);
}

Example 27 with SqlVarbinary

use of io.prestosql.spi.type.SqlVarbinary in project hetu-core by openlookeng.

the class AbstractTestHive method assertGetRecords.

protected void assertGetRecords(HiveStorageFormat hiveStorageFormat, ConnectorTableMetadata tableMetadata, HiveSplit hiveSplit, ConnectorPageSource pageSource, List<? extends ColumnHandle> columnHandles) throws IOException {
    try {
        MaterializedResult result = materializeSourceDataStream(newSession(), pageSource, getTypes(columnHandles));
        assertPageSourceType(pageSource, hiveStorageFormat);
        ImmutableMap<String, Integer> columnIndex = indexColumns(tableMetadata);
        long rowNumber = 0;
        long completedBytes = 0;
        for (MaterializedRow row : result) {
            try {
                assertValueTypes(row, tableMetadata.getColumns());
            } catch (RuntimeException e) {
                throw new RuntimeException("row " + rowNumber, e);
            }
            rowNumber++;
            Integer index;
            Object value;
            // STRING
            index = columnIndex.get("t_string");
            value = row.getField(index);
            if (rowNumber % 19 == 0) {
                assertNull(value);
            } else if (rowNumber % 19 == 1) {
                assertEquals(value, "");
            } else {
                assertEquals(value, "test");
            }
            // NUMBERS
            assertEquals(row.getField(columnIndex.get("t_tinyint")), (byte) (1 + rowNumber));
            assertEquals(row.getField(columnIndex.get("t_smallint")), (short) (2 + rowNumber));
            assertEquals(row.getField(columnIndex.get("t_int")), (int) (3 + rowNumber));
            index = columnIndex.get("t_bigint");
            if ((rowNumber % 13) == 0) {
                assertNull(row.getField(index));
            } else {
                assertEquals(row.getField(index), 4 + rowNumber);
            }
            assertEquals((Float) row.getField(columnIndex.get("t_float")), 5.1f + rowNumber, 0.001);
            assertEquals(row.getField(columnIndex.get("t_double")), 6.2 + rowNumber);
            // BOOLEAN
            index = columnIndex.get("t_boolean");
            if ((rowNumber % 3) == 2) {
                assertNull(row.getField(index));
            } else {
                assertEquals(row.getField(index), (rowNumber % 3) != 0);
            }
            // TIMESTAMP
            index = columnIndex.get("t_timestamp");
            if (index != null) {
                if ((rowNumber % 17) == 0) {
                    assertNull(row.getField(index));
                } else {
                    SqlTimestamp expected = sqlTimestampOf(2011, 5, 6, 7, 8, 9, 123);
                    assertEquals(row.getField(index), expected);
                }
            }
            // BINARY
            index = columnIndex.get("t_binary");
            if (index != null) {
                if ((rowNumber % 23) == 0) {
                    assertNull(row.getField(index));
                } else {
                    assertEquals(row.getField(index), new SqlVarbinary("test binary".getBytes(UTF_8)));
                }
            }
            // DATE
            index = columnIndex.get("t_date");
            if (index != null) {
                if ((rowNumber % 37) == 0) {
                    assertNull(row.getField(index));
                } else {
                    SqlDate expected = new SqlDate(toIntExact(MILLISECONDS.toDays(new DateTime(2013, 8, 9, 0, 0, 0, UTC).getMillis())));
                    assertEquals(row.getField(index), expected);
                }
            }
            // VARCHAR(50)
            index = columnIndex.get("t_varchar");
            if (index != null) {
                value = row.getField(index);
                if (rowNumber % 39 == 0) {
                    assertNull(value);
                } else if (rowNumber % 39 == 1) {
                    // RCBINARY reads empty VARCHAR as null
                    if (hiveStorageFormat == RCBINARY) {
                        assertNull(value);
                    } else {
                        assertEquals(value, "");
                    }
                } else {
                    assertEquals(value, "test varchar");
                }
            }
            // CHAR(25)
            index = columnIndex.get("t_char");
            if (index != null) {
                value = row.getField(index);
                if ((rowNumber % 41) == 0) {
                    assertNull(value);
                } else {
                    assertEquals(value, (rowNumber % 41) == 1 ? "                         " : "test char                ");
                }
            }
            // MAP<STRING, STRING>
            index = columnIndex.get("t_map");
            if (index != null) {
                if ((rowNumber % 27) == 0) {
                    assertNull(row.getField(index));
                } else {
                    assertEquals(row.getField(index), ImmutableMap.of("test key", "test value"));
                }
            }
            // ARRAY<STRING>
            index = columnIndex.get("t_array_string");
            if (index != null) {
                if ((rowNumber % 29) == 0) {
                    assertNull(row.getField(index));
                } else {
                    assertEquals(row.getField(index), ImmutableList.of("abc", "xyz", "data"));
                }
            }
            // ARRAY<TIMESTAMP>
            index = columnIndex.get("t_array_timestamp");
            if (index != null) {
                if ((rowNumber % 43) == 0) {
                    assertNull(row.getField(index));
                } else {
                    SqlTimestamp expected = sqlTimestampOf(LocalDateTime.of(2011, 5, 6, 7, 8, 9, 123_000_000));
                    assertEquals(row.getField(index), ImmutableList.of(expected));
                }
            }
            // ARRAY<STRUCT<s_string: STRING, s_double:DOUBLE>>
            index = columnIndex.get("t_array_struct");
            if (index != null) {
                if ((rowNumber % 31) == 0) {
                    assertNull(row.getField(index));
                } else {
                    List<Object> expected1 = ImmutableList.of("test abc", 0.1);
                    List<Object> expected2 = ImmutableList.of("test xyz", 0.2);
                    assertEquals(row.getField(index), ImmutableList.of(expected1, expected2));
                }
            }
            // STRUCT<s_string: STRING, s_double:DOUBLE>
            index = columnIndex.get("t_struct");
            if (index != null) {
                if ((rowNumber % 31) == 0) {
                    assertNull(row.getField(index));
                } else {
                    assertTrue(row.getField(index) instanceof List);
                    List<?> values = (List<?>) row.getField(index);
                    assertEquals(values.size(), 2);
                    assertEquals(values.get(0), "test abc");
                    assertEquals(values.get(1), 0.1);
                }
            }
            // MAP<INT, ARRAY<STRUCT<s_string: STRING, s_double:DOUBLE>>>
            index = columnIndex.get("t_complex");
            if (index != null) {
                if ((rowNumber % 33) == 0) {
                    assertNull(row.getField(index));
                } else {
                    List<Object> expected1 = ImmutableList.of("test abc", 0.1);
                    List<Object> expected2 = ImmutableList.of("test xyz", 0.2);
                    assertEquals(row.getField(index), ImmutableMap.of(1, ImmutableList.of(expected1, expected2)));
                }
            }
            // NEW COLUMN
            assertNull(row.getField(columnIndex.get("new_column")));
            long newCompletedBytes = pageSource.getCompletedBytes();
            assertTrue(newCompletedBytes >= completedBytes);
            assertTrue(newCompletedBytes <= hiveSplit.getLength());
            completedBytes = newCompletedBytes;
        }
        assertTrue(completedBytes <= hiveSplit.getLength());
        assertEquals(rowNumber, 100);
    } finally {
        pageSource.close();
    }
}

Example 28 with SqlVarbinary

use of io.prestosql.spi.type.SqlVarbinary in project boostkit-bigdata by kunpengcompute.

the class AbstractTestHiveFileFormats method checkPageSource.

protected void checkPageSource(ConnectorPageSource pageSource, List<TestColumn> testColumns, List<Type> types, int rowCount) throws IOException {
    try {
        MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, types);
        assertEquals(result.getMaterializedRows().size(), rowCount);
        for (MaterializedRow row : result) {
            for (int i = 0, testColumnsSize = testColumns.size(); i < testColumnsSize; i++) {
                TestColumn testColumn = testColumns.get(i);
                Type type = types.get(i);
                Object actualValue = row.getField(i);
                Object expectedValue = testColumn.getExpectedValue();
                if (expectedValue instanceof Slice) {
                    expectedValue = ((Slice) expectedValue).toStringUtf8();
                }
                if (actualValue == null || expectedValue == null) {
                    assertEquals(actualValue, expectedValue, "Wrong value for column " + testColumn.getName());
                } else if (testColumn.getObjectInspector().getTypeName().equals("float")) {
                    assertEquals((float) actualValue, (float) expectedValue, EPSILON, "Wrong value for column " + testColumn.getName());
                } else if (testColumn.getObjectInspector().getTypeName().equals("double")) {
                    assertEquals((double) actualValue, (double) expectedValue, EPSILON, "Wrong value for column " + testColumn.getName());
                } else if (testColumn.getObjectInspector().getTypeName().equals("date")) {
                    SqlDate expectedDate = new SqlDate(((Long) expectedValue).intValue());
                    assertEquals(actualValue, expectedDate, "Wrong value for column " + testColumn.getName());
                } else if (testColumn.getObjectInspector().getTypeName().equals("int") || testColumn.getObjectInspector().getTypeName().equals("smallint") || testColumn.getObjectInspector().getTypeName().equals("tinyint")) {
                    assertEquals(actualValue, expectedValue);
                } else if (testColumn.getObjectInspector().getTypeName().equals("timestamp")) {
                    SqlTimestamp expectedTimestamp = sqlTimestampOf((Long) expectedValue);
                    assertEquals(actualValue, expectedTimestamp, "Wrong value for column " + testColumn.getName());
                } else if (testColumn.getObjectInspector().getTypeName().startsWith("char")) {
                    assertEquals(actualValue, padEnd((String) expectedValue, ((CharType) type).getLength(), ' '), "Wrong value for column " + testColumn.getName());
                } else if (testColumn.getObjectInspector().getCategory() == Category.PRIMITIVE) {
                    if (expectedValue instanceof Slice) {
                        expectedValue = ((Slice) expectedValue).toStringUtf8();
                    }
                    if (actualValue instanceof Slice) {
                        actualValue = ((Slice) actualValue).toStringUtf8();
                    }
                    if (actualValue instanceof SqlVarbinary) {
                        actualValue = new String(((SqlVarbinary) actualValue).getBytes(), UTF_8);
                    }
                    if (actualValue instanceof SqlDecimal) {
                        actualValue = new BigDecimal(actualValue.toString());
                    }
                    assertEquals(actualValue, expectedValue, "Wrong value for column " + testColumn.getName());
                } else {
                    BlockBuilder builder = type.createBlockBuilder(null, 1);
                    type.writeObject(builder, expectedValue);
                    expectedValue = type.getObjectValue(SESSION, builder.build(), 0);
                    assertEquals(actualValue, expectedValue, "Wrong value for column " + testColumn.getName());
                }
            }
        }
    } finally {
        pageSource.close();
    }
}