Examples with VARBINARY com.facebook.presto.common.type.VarbinaryType.VARBINARY used on opensource projects

Example 1 with VARBINARY

use of com.facebook.presto.common.type.VarbinaryType.VARBINARY in project presto by prestodb.

the class TestSelectiveOrcReader method testVarBinaries.

@Test
public void testVarBinaries() throws Exception {
    tester.testRoundTrip(VARBINARY, createList(NUM_ROWS, i -> new SqlVarbinary(String.valueOf(i).getBytes(UTF_8))), stringIn(false, "10", "11"));
    tester.testRoundTripTypes(ImmutableList.of(VARBINARY, VARBINARY), ImmutableList.of(createList(NUM_ROWS, i -> new SqlVarbinary(Ints.toByteArray(i))), Streams.stream(limit(cycle(ImmutableList.of("A", "B", "C")), NUM_ROWS)).map(String::getBytes).map(SqlVarbinary::new).collect(toImmutableList())), toSubfieldFilters(ImmutableMap.of(0, stringBetween(true, "10", "16"), 1, stringBetween(false, "A", "B"))));
    tester.testRoundTrip(VARBINARY, createList(NUM_ROWS, i -> new SqlVarbinary(String.valueOf(i).getBytes(UTF_8))), bytesBetween(false, new byte[] { 8 }, new byte[] { 9 }));
    tester.testRoundTrip(VARBINARY, nCopies(NUM_ROWS, new SqlVarbinary(new byte[0])), bytesBetween(false, new byte[0], new byte[] { 1 }));
    tester.testRoundTrip(VARBINARY, ImmutableList.copyOf(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), NUM_ROWS)).stream().map(String::valueOf).map(string -> string.getBytes(UTF_8)).map(SqlVarbinary::new).collect(toImmutableList()), bytesBetween(false, new byte[] { 1 }, new byte[] { 12 }));
}

Example 2 with VARBINARY

use of com.facebook.presto.common.type.VarbinaryType.VARBINARY in project presto by prestodb.

the class TestMapFlatSelectiveStreamReader method runTest.

private <K, V> void runTest(String testOrcFileName, Type keyType, Type valueType, ExpectedValuesBuilder<K, V> expectedValuesBuilder) throws Exception {
    List<Map<K, V>> expectedValues = expectedValuesBuilder.build();
    Type mapType = mapType(keyType, valueType);
    OrcPredicate orcPredicate = createOrcPredicate(0, mapType, expectedValues, OrcTester.Format.DWRF, true);
    runTest(testOrcFileName, mapType, expectedValues, orcPredicate, Optional.empty(), ImmutableList.of());
    runTest(testOrcFileName, mapType, expectedValues.stream().filter(Objects::isNull).collect(toList()), orcPredicate, Optional.of(IS_NULL), ImmutableList.of());
    runTest(testOrcFileName, mapType, expectedValues.stream().filter(Objects::nonNull).collect(toList()), orcPredicate, Optional.of(IS_NOT_NULL), ImmutableList.of());
    if (keyType != VARBINARY) {
        // read only some keys
        List<K> keys = expectedValues.stream().filter(Objects::nonNull).flatMap(v -> v.keySet().stream()).distinct().collect(toImmutableList());
        if (!keys.isEmpty()) {
            List<K> requiredKeys = ImmutableList.of(keys.get(0));
            runTest(testOrcFileName, mapType, pruneMaps(expectedValues, requiredKeys), orcPredicate, Optional.empty(), toSubfields(keyType, requiredKeys));
            List<Integer> keyIndices = ImmutableList.of(1, 3, 7, 11);
            requiredKeys = keyIndices.stream().filter(k -> k < keys.size()).map(keys::get).collect(toList());
            runTest(testOrcFileName, mapType, pruneMaps(expectedValues, requiredKeys), orcPredicate, Optional.empty(), toSubfields(keyType, requiredKeys));
        }
    }
    // read only some rows
    List<Integer> ids = IntStream.range(0, expectedValues.size()).map(i -> i % 10).boxed().collect(toImmutableList());
    ImmutableList<Type> types = ImmutableList.of(mapType, INTEGER);
    Map<Integer, Map<Subfield, TupleDomainFilter>> filters = ImmutableMap.of(1, ImmutableMap.of(new Subfield("c"), toBigintValues(new long[] { 1, 5, 6 }, true)));
    assertFileContentsPresto(types, new File(getResource(testOrcFileName).getFile()), filterRows(types, ImmutableList.of(expectedValues, ids), filters), OrcEncoding.DWRF, OrcPredicate.TRUE, Optional.of(filters), ImmutableList.of(), ImmutableMap.of(), ImmutableMap.of());
    TestingFilterFunction filterFunction = new TestingFilterFunction(mapType);
    assertFileContentsPresto(types, new File(getResource(testOrcFileName).getFile()), filterFunction.filterRows(ImmutableList.of(expectedValues, ids)), OrcEncoding.DWRF, OrcPredicate.TRUE, Optional.empty(), ImmutableList.of(filterFunction), ImmutableMap.of(0, 0), ImmutableMap.of());
}

Example 3 with VARBINARY

use of com.facebook.presto.common.type.VarbinaryType.VARBINARY in project presto by prestodb.

the class TestCassandraIntegrationSmokeTest method assertSelect.

private void assertSelect(String tableName, boolean createdByPresto) {
    Type uuidType = createdByPresto ? createUnboundedVarcharType() : createVarcharType(36);
    Type inetType = createdByPresto ? createUnboundedVarcharType() : createVarcharType(45);
    String sql = "SELECT " + " key, " + " typeuuid, " + " typeinteger, " + " typelong, " + " typebytes, " + " typetimestamp, " + " typeansi, " + " typeboolean, " + " typedecimal, " + " typedouble, " + " typefloat, " + " typeinet, " + " typevarchar, " + " typevarint, " + " typetimeuuid, " + " typelist, " + " typemap, " + " typeset " + " FROM " + tableName;
    MaterializedResult result = execute(sql);
    int rowCount = result.getRowCount();
    assertEquals(rowCount, 9);
    assertEquals(result.getTypes(), ImmutableList.of(createUnboundedVarcharType(), uuidType, INTEGER, BIGINT, VARBINARY, TIMESTAMP, createUnboundedVarcharType(), BOOLEAN, DOUBLE, DOUBLE, REAL, inetType, createUnboundedVarcharType(), createUnboundedVarcharType(), uuidType, createUnboundedVarcharType(), createUnboundedVarcharType(), createUnboundedVarcharType()));
    List<MaterializedRow> sortedRows = result.getMaterializedRows().stream().sorted((o1, o2) -> o1.getField(1).toString().compareTo(o2.getField(1).toString())).collect(toList());
    for (int rowNumber = 1; rowNumber <= rowCount; rowNumber++) {
        assertEquals(sortedRows.get(rowNumber - 1), new MaterializedRow(DEFAULT_PRECISION, "key " + rowNumber, String.format("00000000-0000-0000-0000-%012d", rowNumber), rowNumber, rowNumber + 1000L, ByteBuffer.wrap(toByteArray(rowNumber)), TIMESTAMP_LOCAL, "ansi " + rowNumber, rowNumber % 2 == 0, Math.pow(2, rowNumber), Math.pow(4, rowNumber), (float) Math.pow(8, rowNumber), "127.0.0.1", "varchar " + rowNumber, BigInteger.TEN.pow(rowNumber).toString(), String.format("d2177dd0-eaa2-11de-a572-001b779c76e%d", rowNumber), String.format("[\"list-value-1%1$d\",\"list-value-2%1$d\"]", rowNumber), String.format("{%d:%d,%d:%d}", rowNumber, rowNumber + 1L, rowNumber + 2, rowNumber + 3L), "[false,true]"));
    }
}

Example 4 with VARBINARY

use of com.facebook.presto.common.type.VarbinaryType.VARBINARY in project presto by prestodb.

the class LogicalPlanner method createTableWriterPlan.

private RelationPlan createTableWriterPlan(Analysis analysis, RelationPlan plan, WriterTarget target, List<String> columnNames, List<ColumnMetadata> columnMetadataList, Optional<NewTableLayout> writeTableLayout, Optional<NewTableLayout> preferredShuffleLayout, TableStatisticsMetadata statisticsMetadata) {
    verify(!(writeTableLayout.isPresent() && preferredShuffleLayout.isPresent()), "writeTableLayout and preferredShuffleLayout cannot both exist");
    PlanNode source = plan.getRoot();
    if (!analysis.isCreateTableAsSelectWithData()) {
        source = new LimitNode(source.getSourceLocation(), idAllocator.getNextId(), source, 0L, FINAL);
    }
    List<VariableReferenceExpression> variables = plan.getFieldMappings();
    Optional<PartitioningScheme> tablePartitioningScheme = getPartitioningSchemeForTableWrite(writeTableLayout, columnNames, variables);
    Optional<PartitioningScheme> preferredShufflePartitioningScheme = getPartitioningSchemeForTableWrite(preferredShuffleLayout, columnNames, variables);
    verify(columnNames.size() == variables.size(), "columnNames.size() != variables.size(): %s and %s", columnNames, variables);
    Map<String, VariableReferenceExpression> columnToVariableMap = zip(columnNames.stream(), plan.getFieldMappings().stream(), SimpleImmutableEntry::new).collect(toImmutableMap(Entry::getKey, Entry::getValue));
    Set<VariableReferenceExpression> notNullColumnVariables = columnMetadataList.stream().filter(column -> !column.isNullable()).map(ColumnMetadata::getName).map(columnToVariableMap::get).collect(toImmutableSet());
    if (!statisticsMetadata.isEmpty()) {
        TableStatisticAggregation result = statisticsAggregationPlanner.createStatisticsAggregation(statisticsMetadata, columnToVariableMap, true);
        StatisticAggregations.Parts aggregations = result.getAggregations().splitIntoPartialAndFinal(variableAllocator, metadata.getFunctionAndTypeManager());
        TableFinishNode commitNode = new TableFinishNode(source.getSourceLocation(), idAllocator.getNextId(), new TableWriterNode(source.getSourceLocation(), idAllocator.getNextId(), source, Optional.of(target), variableAllocator.newVariable("rows", BIGINT), variableAllocator.newVariable("fragments", VARBINARY), variableAllocator.newVariable("commitcontext", VARBINARY), plan.getFieldMappings(), columnNames, notNullColumnVariables, tablePartitioningScheme, preferredShufflePartitioningScheme, // the data consumed by the TableWriteOperator
        Optional.of(aggregations.getPartialAggregation())), Optional.of(target), variableAllocator.newVariable("rows", BIGINT), // by the partial aggregation from all of the writer nodes
        Optional.of(aggregations.getFinalAggregation()), Optional.of(result.getDescriptor()));
        return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputVariables());
    }
    TableFinishNode commitNode = new TableFinishNode(source.getSourceLocation(), idAllocator.getNextId(), new TableWriterNode(source.getSourceLocation(), idAllocator.getNextId(), source, Optional.of(target), variableAllocator.newVariable("rows", BIGINT), variableAllocator.newVariable("fragments", VARBINARY), variableAllocator.newVariable("commitcontext", VARBINARY), plan.getFieldMappings(), columnNames, notNullColumnVariables, tablePartitioningScheme, preferredShufflePartitioningScheme, Optional.empty()), Optional.of(target), variableAllocator.newVariable("rows", BIGINT), Optional.empty(), Optional.empty());
    return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputVariables());
}

Example 5 with VARBINARY

use of com.facebook.presto.common.type.VarbinaryType.VARBINARY in project presto by prestodb.

the class RcFileTester method preprocessWriteValueOld.

private static Object preprocessWriteValueOld(Type type, Object value) {
    if (value == null) {
        return null;
    }
    if (type.equals(BOOLEAN)) {
        return value;
    } else if (type.equals(TINYINT)) {
        return ((Number) value).byteValue();
    } else if (type.equals(SMALLINT)) {
        return ((Number) value).shortValue();
    } else if (type.equals(INTEGER)) {
        return ((Number) value).intValue();
    } else if (type.equals(BIGINT)) {
        return ((Number) value).longValue();
    } else if (type.equals(REAL)) {
        return ((Number) value).floatValue();
    } else if (type.equals(DOUBLE)) {
        return ((Number) value).doubleValue();
    } else if (type instanceof VarcharType) {
        return value;
    } else if (type.equals(VARBINARY)) {
        return ((SqlVarbinary) value).getBytes();
    } else if (type.equals(DATE)) {
        int days = ((SqlDate) value).getDays();
        LocalDate localDate = LocalDate.ofEpochDay(days);
        ZonedDateTime zonedDateTime = localDate.atStartOfDay(ZoneId.systemDefault());
        long millis = zonedDateTime.toEpochSecond() * 1000;
        Date date = new Date(0);
        // mills must be set separately to avoid masking
        date.setTime(millis);
        return date;
    } else if (type.equals(TIMESTAMP)) {
        long millisUtc = (int) ((SqlTimestamp) value).getMillisUtc();
        return new Timestamp(millisUtc);
    } else if (type instanceof DecimalType) {
        return HiveDecimal.create(((SqlDecimal) value).toBigDecimal());
    } else if (type.getTypeSignature().getBase().equals(ARRAY)) {
        Type elementType = type.getTypeParameters().get(0);
        return ((List<?>) value).stream().map(element -> preprocessWriteValueOld(elementType, element)).collect(toList());
    } else if (type.getTypeSignature().getBase().equals(MAP)) {
        Type keyType = type.getTypeParameters().get(0);
        Type valueType = type.getTypeParameters().get(1);
        Map<Object, Object> newMap = new HashMap<>();
        for (Entry<?, ?> entry : ((Map<?, ?>) value).entrySet()) {
            newMap.put(preprocessWriteValueOld(keyType, entry.getKey()), preprocessWriteValueOld(valueType, entry.getValue()));
        }
        return newMap;
    } else if (type.getTypeSignature().getBase().equals(ROW)) {
        List<?> fieldValues = (List<?>) value;
        List<Type> fieldTypes = type.getTypeParameters();
        List<Object> newStruct = new ArrayList<>();
        for (int fieldId = 0; fieldId < fieldValues.size(); fieldId++) {
            newStruct.add(preprocessWriteValueOld(fieldTypes.get(fieldId), fieldValues.get(fieldId)));
        }
        return newStruct;
    }
    throw new IllegalArgumentException("unsupported type: " + type);
}