Examples with SqlVarbinary io.trino.spi.type.SqlVarbinary used on opensource projects

Example 1 with SqlVarbinary

use of io.trino.spi.type.SqlVarbinary in project trino by trinodb.

the class AbstractTestHive method assertGetRecords.

protected void assertGetRecords(HiveStorageFormat hiveStorageFormat, ConnectorTableMetadata tableMetadata, HiveSplit hiveSplit, ConnectorPageSource pageSource, List<? extends ColumnHandle> columnHandles) throws IOException {
    try {
        MaterializedResult result = materializeSourceDataStream(newSession(), pageSource, getTypes(columnHandles));
        assertPageSourceType(pageSource, hiveStorageFormat);
        ImmutableMap<String, Integer> columnIndex = indexColumns(tableMetadata);
        long rowNumber = 0;
        long completedBytes = 0;
        for (MaterializedRow row : result) {
            try {
                assertValueTypes(row, tableMetadata.getColumns());
            } catch (RuntimeException e) {
                throw new RuntimeException("row " + rowNumber, e);
            }
            rowNumber++;
            Integer index;
            Object value;
            // STRING
            index = columnIndex.get("t_string");
            value = row.getField(index);
            if (rowNumber % 19 == 0) {
                assertNull(value);
            } else if (rowNumber % 19 == 1) {
                assertEquals(value, "");
            } else {
                assertEquals(value, "test");
            }
            // NUMBERS
            assertEquals(row.getField(columnIndex.get("t_tinyint")), (byte) (1 + rowNumber));
            assertEquals(row.getField(columnIndex.get("t_smallint")), (short) (2 + rowNumber));
            assertEquals(row.getField(columnIndex.get("t_int")), (int) (3 + rowNumber));
            index = columnIndex.get("t_bigint");
            if ((rowNumber % 13) == 0) {
                assertNull(row.getField(index));
            } else {
                assertEquals(row.getField(index), 4 + rowNumber);
            }
            assertEquals((Float) row.getField(columnIndex.get("t_float")), 5.1f + rowNumber, 0.001);
            assertEquals(row.getField(columnIndex.get("t_double")), 6.2 + rowNumber);
            // BOOLEAN
            index = columnIndex.get("t_boolean");
            if ((rowNumber % 3) == 2) {
                assertNull(row.getField(index));
            } else {
                assertEquals(row.getField(index), (rowNumber % 3) != 0);
            }
            // TIMESTAMP
            index = columnIndex.get("t_timestamp");
            if (index != null) {
                if ((rowNumber % 17) == 0) {
                    assertNull(row.getField(index));
                } else {
                    SqlTimestamp expected = sqlTimestampOf(3, 2011, 5, 6, 7, 8, 9, 123);
                    assertEquals(row.getField(index), expected);
                }
            }
            // BINARY
            index = columnIndex.get("t_binary");
            if (index != null) {
                if ((rowNumber % 23) == 0) {
                    assertNull(row.getField(index));
                } else {
                    assertEquals(row.getField(index), new SqlVarbinary("test binary".getBytes(UTF_8)));
                }
            }
            // DATE
            index = columnIndex.get("t_date");
            if (index != null) {
                if ((rowNumber % 37) == 0) {
                    assertNull(row.getField(index));
                } else {
                    SqlDate expected = new SqlDate(toIntExact(MILLISECONDS.toDays(new DateTime(2013, 8, 9, 0, 0, 0, UTC).getMillis())));
                    assertEquals(row.getField(index), expected);
                }
            }
            // VARCHAR(50)
            index = columnIndex.get("t_varchar");
            if (index != null) {
                value = row.getField(index);
                if (rowNumber % 39 == 0) {
                    assertNull(value);
                } else if (rowNumber % 39 == 1) {
                    // RCBINARY reads empty VARCHAR as null
                    if (hiveStorageFormat == RCBINARY) {
                        assertNull(value);
                    } else {
                        assertEquals(value, "");
                    }
                } else {
                    assertEquals(value, "test varchar");
                }
            }
            // CHAR(25)
            index = columnIndex.get("t_char");
            if (index != null) {
                value = row.getField(index);
                if ((rowNumber % 41) == 0) {
                    assertNull(value);
                } else {
                    assertEquals(value, (rowNumber % 41) == 1 ? "                         " : "test char                ");
                }
            }
            // MAP<STRING, STRING>
            index = columnIndex.get("t_map");
            if (index != null) {
                if ((rowNumber % 27) == 0) {
                    assertNull(row.getField(index));
                } else {
                    assertEquals(row.getField(index), ImmutableMap.of("test key", "test value"));
                }
            }
            // ARRAY<STRING>
            index = columnIndex.get("t_array_string");
            if (index != null) {
                if ((rowNumber % 29) == 0) {
                    assertNull(row.getField(index));
                } else {
                    assertEquals(row.getField(index), ImmutableList.of("abc", "xyz", "data"));
                }
            }
            // ARRAY<TIMESTAMP>
            index = columnIndex.get("t_array_timestamp");
            if (index != null) {
                if ((rowNumber % 43) == 0) {
                    assertNull(row.getField(index));
                } else {
                    SqlTimestamp expected = sqlTimestampOf(3, LocalDateTime.of(2011, 5, 6, 7, 8, 9, 123_000_000));
                    assertEquals(row.getField(index), ImmutableList.of(expected));
                }
            }
            // ARRAY<STRUCT<s_string: STRING, s_double:DOUBLE>>
            index = columnIndex.get("t_array_struct");
            if (index != null) {
                if ((rowNumber % 31) == 0) {
                    assertNull(row.getField(index));
                } else {
                    List<Object> expected1 = ImmutableList.of("test abc", 0.1);
                    List<Object> expected2 = ImmutableList.of("test xyz", 0.2);
                    assertEquals(row.getField(index), ImmutableList.of(expected1, expected2));
                }
            }
            // STRUCT<s_string: STRING, s_double:DOUBLE>
            index = columnIndex.get("t_struct");
            if (index != null) {
                if ((rowNumber % 31) == 0) {
                    assertNull(row.getField(index));
                } else {
                    assertTrue(row.getField(index) instanceof List);
                    List<?> values = (List<?>) row.getField(index);
                    assertEquals(values.size(), 2);
                    assertEquals(values.get(0), "test abc");
                    assertEquals(values.get(1), 0.1);
                }
            }
            // MAP<INT, ARRAY<STRUCT<s_string: STRING, s_double:DOUBLE>>>
            index = columnIndex.get("t_complex");
            if (index != null) {
                if ((rowNumber % 33) == 0) {
                    assertNull(row.getField(index));
                } else {
                    List<Object> expected1 = ImmutableList.of("test abc", 0.1);
                    List<Object> expected2 = ImmutableList.of("test xyz", 0.2);
                    assertEquals(row.getField(index), ImmutableMap.of(1, ImmutableList.of(expected1, expected2)));
                }
            }
            // NEW COLUMN
            assertNull(row.getField(columnIndex.get("new_column")));
            long newCompletedBytes = pageSource.getCompletedBytes();
            assertTrue(newCompletedBytes >= completedBytes);
            // some formats (e.g., parquet) over read the data by a bit
            assertLessThanOrEqual(newCompletedBytes, hiveSplit.getLength() + (100 * 1024));
            completedBytes = newCompletedBytes;
        }
        assertLessThanOrEqual(completedBytes, hiveSplit.getLength() + (100 * 1024));
        assertEquals(rowNumber, 100);
    } finally {
        pageSource.close();
    }
}

Example 2 with SqlVarbinary

use of io.trino.spi.type.SqlVarbinary in project trino by trinodb.

the class ParquetTester method writeValue.

private static void writeValue(Type type, BlockBuilder blockBuilder, Object value) {
    if (value == null) {
        blockBuilder.appendNull();
    } else {
        if (BOOLEAN.equals(type)) {
            type.writeBoolean(blockBuilder, (Boolean) value);
        } else if (TINYINT.equals(type) || SMALLINT.equals(type) || INTEGER.equals(type) || BIGINT.equals(type)) {
            type.writeLong(blockBuilder, ((Number) value).longValue());
        } else if (Decimals.isShortDecimal(type)) {
            type.writeLong(blockBuilder, ((SqlDecimal) value).getUnscaledValue().longValue());
        } else if (Decimals.isLongDecimal(type)) {
            if (Decimals.overflows(((SqlDecimal) value).getUnscaledValue(), MAX_PRECISION_INT64)) {
                type.writeObject(blockBuilder, Int128.valueOf(((SqlDecimal) value).toBigDecimal().unscaledValue()));
            } else {
                type.writeObject(blockBuilder, Int128.valueOf(((SqlDecimal) value).getUnscaledValue().longValue()));
            }
        } else if (DOUBLE.equals(type)) {
            type.writeDouble(blockBuilder, ((Number) value).doubleValue());
        } else if (REAL.equals(type)) {
            float floatValue = ((Number) value).floatValue();
            type.writeLong(blockBuilder, Float.floatToIntBits(floatValue));
        } else if (type instanceof VarcharType) {
            Slice slice = truncateToLength(utf8Slice((String) value), type);
            type.writeSlice(blockBuilder, slice);
        } else if (type instanceof CharType) {
            Slice slice = truncateToLengthAndTrimSpaces(utf8Slice((String) value), type);
            type.writeSlice(blockBuilder, slice);
        } else if (VARBINARY.equals(type)) {
            type.writeSlice(blockBuilder, Slices.wrappedBuffer(((SqlVarbinary) value).getBytes()));
        } else if (DATE.equals(type)) {
            long days = ((SqlDate) value).getDays();
            type.writeLong(blockBuilder, days);
        } else if (TIMESTAMP_MILLIS.equals(type)) {
            type.writeLong(blockBuilder, ((SqlTimestamp) value).getEpochMicros());
        } else {
            if (type instanceof ArrayType) {
                List<?> array = (List<?>) value;
                Type elementType = type.getTypeParameters().get(0);
                BlockBuilder arrayBlockBuilder = blockBuilder.beginBlockEntry();
                for (Object elementValue : array) {
                    writeValue(elementType, arrayBlockBuilder, elementValue);
                }
                blockBuilder.closeEntry();
            } else if (type instanceof MapType) {
                Map<?, ?> map = (Map<?, ?>) value;
                Type keyType = type.getTypeParameters().get(0);
                Type valueType = type.getTypeParameters().get(1);
                BlockBuilder mapBlockBuilder = blockBuilder.beginBlockEntry();
                for (Map.Entry<?, ?> entry : map.entrySet()) {
                    writeValue(keyType, mapBlockBuilder, entry.getKey());
                    writeValue(valueType, mapBlockBuilder, entry.getValue());
                }
                blockBuilder.closeEntry();
            } else if (type instanceof RowType) {
                List<?> array = (List<?>) value;
                List<Type> fieldTypes = type.getTypeParameters();
                BlockBuilder rowBlockBuilder = blockBuilder.beginBlockEntry();
                for (int fieldId = 0; fieldId < fieldTypes.size(); fieldId++) {
                    Type fieldType = fieldTypes.get(fieldId);
                    writeValue(fieldType, rowBlockBuilder, array.get(fieldId));
                }
                blockBuilder.closeEntry();
            } else {
                throw new IllegalArgumentException("Unsupported type " + type);
            }
        }
    }
}

Example 3 with SqlVarbinary

use of io.trino.spi.type.SqlVarbinary in project trino by trinodb.

the class TestQuantileDigestAggregationFunction method assertPercentilesWithinError.

private void assertPercentilesWithinError(String type, SqlVarbinary binary, double error, List<? extends Number> rows, double[] percentiles) {
    List<Double> boxedPercentiles = Arrays.stream(percentiles).sorted().boxed().collect(toImmutableList());
    List<Number> lowerBounds = boxedPercentiles.stream().map(percentile -> getLowerBound(error, rows, percentile)).collect(toImmutableList());
    List<Number> upperBounds = boxedPercentiles.stream().map(percentile -> getUpperBound(error, rows, percentile)).collect(toImmutableList());
    // Ensure that the lower bound of each item in the distribution is not greater than the chosen quantiles
    functionAssertions.assertFunction(format("zip_with(values_at_quantiles(CAST(X'%s' AS qdigest(%s)), ARRAY[%s]), ARRAY[%s], (value, lowerbound) -> value >= lowerbound)", binary.toString().replaceAll("\\s+", " "), type, ARRAY_JOINER.join(boxedPercentiles), ARRAY_JOINER.join(lowerBounds)), new ArrayType(BOOLEAN), Collections.nCopies(percentiles.length, true));
    // Ensure that the upper bound of each item in the distribution is not less than the chosen quantiles
    functionAssertions.assertFunction(format("zip_with(values_at_quantiles(CAST(X'%s' AS qdigest(%s)), ARRAY[%s]), ARRAY[%s], (value, upperbound) -> value <= upperbound)", binary.toString().replaceAll("\\s+", " "), type, ARRAY_JOINER.join(boxedPercentiles), ARRAY_JOINER.join(upperBounds)), new ArrayType(BOOLEAN), Collections.nCopies(percentiles.length, true));
}

Example 4 with SqlVarbinary

use of io.trino.spi.type.SqlVarbinary in project trino by trinodb.

the class TestQuantileDigestAggregationFunction method testAggregationDoubles.

private void testAggregationDoubles(List<TypeSignatureProvider> parameterTypes, Page page, double maxError, double... inputs) {
    assertAggregation(FUNCTION_RESOLUTION, NAME, parameterTypes, QDIGEST_EQUALITY, "test multiple positions", page, getExpectedValueDoubles(maxError, inputs));
    // test scalars
    List<Double> rows = Arrays.stream(inputs).sorted().boxed().collect(Collectors.toList());
    SqlVarbinary returned = (SqlVarbinary) AggregationTestUtils.aggregation(FUNCTION_RESOLUTION.getAggregateFunction(NAME, parameterTypes), page);
    assertPercentileWithinError(StandardTypes.DOUBLE, returned, maxError, rows, 0.1, 0.5, 0.9, 0.99);
}

Example 5 with SqlVarbinary

use of io.trino.spi.type.SqlVarbinary in project trino by trinodb.

the class TestQuantileDigestAggregationFunction method testAggregationReal.

private void testAggregationReal(List<TypeSignatureProvider> parameterTypes, Page page, double maxError, float... inputs) {
    assertAggregation(FUNCTION_RESOLUTION, NAME, parameterTypes, QDIGEST_EQUALITY, "test multiple positions", page, getExpectedValuesFloats(maxError, inputs));
    // test scalars
    List<Double> rows = Floats.asList(inputs).stream().sorted().map(Float::doubleValue).collect(Collectors.toList());
    SqlVarbinary returned = (SqlVarbinary) AggregationTestUtils.aggregation(FUNCTION_RESOLUTION.getAggregateFunction(NAME, parameterTypes), page);
    assertPercentileWithinError(StandardTypes.REAL, returned, maxError, rows, 0.1, 0.5, 0.9, 0.99);
}