Examples with INTEGER com.facebook.presto.common.type.IntegerType.INTEGER used on opensource projects

Example 6 with INTEGER

use of com.facebook.presto.common.type.IntegerType.INTEGER in project presto by prestodb.

the class AbstractTestParquetReader method testCaching.

@Test
public void testCaching() throws Exception {
    Cache<ParquetDataSourceId, ParquetFileMetadata> parquetFileMetadataCache = CacheBuilder.newBuilder().maximumWeight(new DataSize(1, MEGABYTE).toBytes()).weigher((id, metadata) -> ((ParquetFileMetadata) metadata).getMetadataSize()).expireAfterAccess(new Duration(10, MINUTES).toMillis(), MILLISECONDS).recordStats().build();
    ParquetMetadataSource parquetMetadataSource = new CachingParquetMetadataSource(parquetFileMetadataCache, new MetadataReader());
    try (ParquetTester.TempFile tempFile = new ParquetTester.TempFile("test", "parquet")) {
        Iterable<Integer> values = intsBetween(0, 10);
        Iterator<?>[] readValues = stream(new Iterable<?>[] { values }).map(Iterable::iterator).toArray(size -> new Iterator<?>[size]);
        List<String> columnNames = singletonList("column1");
        List<Type> columnTypes = singletonList(INTEGER);
        writeParquetFileFromPresto(tempFile.getFile(), columnTypes, columnNames, readValues, 10, CompressionCodecName.GZIP);
        testSingleRead(new Iterable<?>[] { values }, columnNames, columnTypes, parquetMetadataSource, tempFile.getFile());
        assertEquals(parquetFileMetadataCache.stats().missCount(), 1);
        assertEquals(parquetFileMetadataCache.stats().hitCount(), 0);
        testSingleRead(new Iterable<?>[] { values }, columnNames, columnTypes, parquetMetadataSource, tempFile.getFile());
        assertEquals(parquetFileMetadataCache.stats().missCount(), 1);
        assertEquals(parquetFileMetadataCache.stats().hitCount(), 1);
        testSingleRead(new Iterable<?>[] { values }, columnNames, columnTypes, parquetMetadataSource, tempFile.getFile());
        assertEquals(parquetFileMetadataCache.stats().missCount(), 1);
        assertEquals(parquetFileMetadataCache.stats().hitCount(), 2);
        parquetFileMetadataCache.invalidateAll();
        testSingleRead(new Iterable<?>[] { values }, columnNames, columnTypes, parquetMetadataSource, tempFile.getFile());
        assertEquals(parquetFileMetadataCache.stats().missCount(), 2);
        assertEquals(parquetFileMetadataCache.stats().hitCount(), 2);
        testSingleRead(new Iterable<?>[] { values }, columnNames, columnTypes, parquetMetadataSource, tempFile.getFile());
        assertEquals(parquetFileMetadataCache.stats().missCount(), 2);
        assertEquals(parquetFileMetadataCache.stats().hitCount(), 3);
    }
}

Example 7 with INTEGER

use of com.facebook.presto.common.type.IntegerType.INTEGER in project presto by prestodb.

the class TestMergingPageIterator method testMerging.

@Test
public void testMerging() {
    List<Type> types = ImmutableList.of(INTEGER, INTEGER);
    List<Integer> sortIndexes = ImmutableList.of(1);
    List<SortOrder> sortOrders = ImmutableList.of(SortOrder.ASC_NULLS_FIRST);
    List<List<Page>> pageLists = new ArrayList<>();
    PageBuilder pageBuilder = new PageBuilder(types);
    for (int i = 0; i < 10; i++) {
        Iterator<Integer> values = IntStream.range(0, 1000).map(ignored -> ThreadLocalRandom.current().nextInt(100_000)).mapToObj(n -> ((n % 100) == 0) ? null : n).sorted(nullsFirst(naturalOrder())).iterator();
        List<Page> pages = new ArrayList<>();
        for (int j = 0; j < 10; j++) {
            for (int k = 0; k < 100; k++) {
                Integer n = values.next();
                pageBuilder.declarePosition();
                if (n == null) {
                    pageBuilder.getBlockBuilder(0).appendNull();
                    pageBuilder.getBlockBuilder(1).appendNull();
                } else {
                    INTEGER.writeLong(pageBuilder.getBlockBuilder(0), n);
                    INTEGER.writeLong(pageBuilder.getBlockBuilder(1), n * 22L);
                }
            }
            pages.add(pageBuilder.build());
            pageBuilder.reset();
        }
        pageLists.add(pages);
        assertFalse(values.hasNext());
    }
    List<Iterator<Page>> pages = pageLists.stream().map(List::iterator).collect(toList());
    Iterator<Page> iterator = new MergingPageIterator(pages, types, sortIndexes, sortOrders);
    List<Long> values = new ArrayList<>();
    while (iterator.hasNext()) {
        Page page = iterator.next();
        for (int i = 0; i < page.getPositionCount(); i++) {
            if (page.getBlock(0).isNull(i)) {
                assertTrue(page.getBlock(1).isNull(i));
                values.add(null);
            } else {
                long x = INTEGER.getLong(page.getBlock(0), i);
                long y = INTEGER.getLong(page.getBlock(1), i);
                assertEquals(y, x * 22);
                values.add(x);
            }
        }
    }
    assertThat(values).isSortedAccordingTo(nullsFirst(naturalOrder()));
}

Example 8 with INTEGER

use of com.facebook.presto.common.type.IntegerType.INTEGER in project presto by prestodb.

the class TestSignatureBinder method testFunction.

@Test
public void testFunction() {
    Signature simple = functionSignature().returnType(parseTypeSignature("boolean")).argumentTypes(parseTypeSignature("function(integer,integer)")).build();
    assertThat(simple).boundTo("integer").fails();
    assertThat(simple).boundTo("function(integer,integer)").succeeds();
    // TODO: Support coercion of return type of lambda
    assertThat(simple).boundTo("function(integer,smallint)").withCoercion().fails();
    assertThat(simple).boundTo("function(integer,bigint)").withCoercion().fails();
    Signature applyTwice = functionSignature().returnType(parseTypeSignature("V")).argumentTypes(parseTypeSignature("T"), parseTypeSignature("function(T,U)"), parseTypeSignature("function(U,V)")).typeVariableConstraints(typeVariable("T"), typeVariable("U"), typeVariable("V")).build();
    assertThat(applyTwice).boundTo("integer", "integer", "integer").fails();
    assertThat(applyTwice).boundTo("integer", "function(integer,varchar)", "function(varchar,double)").produces(BoundVariables.builder().setTypeVariable("T", INTEGER).setTypeVariable("U", VARCHAR).setTypeVariable("V", DOUBLE).build());
    assertThat(applyTwice).boundTo("integer", new TypeSignatureProvider(functionArgumentTypes -> TypeSignature.parseTypeSignature("function(integer,varchar)")), new TypeSignatureProvider(functionArgumentTypes -> TypeSignature.parseTypeSignature("function(varchar,double)"))).produces(BoundVariables.builder().setTypeVariable("T", INTEGER).setTypeVariable("U", VARCHAR).setTypeVariable("V", DOUBLE).build());
    assertThat(applyTwice).boundTo(// pass function argument to non-function position of a function
    new TypeSignatureProvider(functionArgumentTypes -> TypeSignature.parseTypeSignature("function(integer,varchar)")), new TypeSignatureProvider(functionArgumentTypes -> TypeSignature.parseTypeSignature("function(integer,varchar)")), new TypeSignatureProvider(functionArgumentTypes -> TypeSignature.parseTypeSignature("function(varchar,double)"))).fails();
    assertThat(applyTwice).boundTo(new TypeSignatureProvider(functionArgumentTypes -> TypeSignature.parseTypeSignature("function(integer,varchar)")), // pass non-function argument to function position of a function
    "integer", new TypeSignatureProvider(functionArgumentTypes -> TypeSignature.parseTypeSignature("function(varchar,double)"))).fails();
    Signature flatMap = functionSignature().returnType(parseTypeSignature("array(T)")).argumentTypes(parseTypeSignature("array(T)"), parseTypeSignature("function(T, array(T))")).typeVariableConstraints(typeVariable("T")).build();
    assertThat(flatMap).boundTo("array(integer)", "function(integer, array(integer))").produces(BoundVariables.builder().setTypeVariable("T", INTEGER).build());
    Signature varargApply = functionSignature().returnType(parseTypeSignature("T")).argumentTypes(parseTypeSignature("T"), parseTypeSignature("function(T, T)")).typeVariableConstraints(typeVariable("T")).setVariableArity(true).build();
    assertThat(varargApply).boundTo("integer", "function(integer, integer)", "function(integer, integer)", "function(integer, integer)").produces(BoundVariables.builder().setTypeVariable("T", INTEGER).build());
    assertThat(varargApply).boundTo("integer", "function(integer, integer)", "function(integer, double)", "function(double, double)").fails();
    Signature loop = functionSignature().returnType(parseTypeSignature("T")).argumentTypes(parseTypeSignature("T"), parseTypeSignature("function(T, T)")).typeVariableConstraints(typeVariable("T")).build();
    assertThat(loop).boundTo("integer", new TypeSignatureProvider(paramTypes -> new FunctionType(paramTypes, BIGINT).getTypeSignature())).fails();
    assertThat(loop).boundTo("integer", new TypeSignatureProvider(paramTypes -> new FunctionType(paramTypes, BIGINT).getTypeSignature())).withCoercion().produces(BoundVariables.builder().setTypeVariable("T", BIGINT).build());
    // TODO: Support coercion of return type of lambda
    assertThat(loop).withCoercion().boundTo("integer", new TypeSignatureProvider(paramTypes -> new FunctionType(paramTypes, SMALLINT).getTypeSignature())).fails();
    // TODO: Support coercion of return type of lambda
    // Without coercion support for return type of lambda, the return type of lambda must be `varchar(x)` to avoid need for coercions.
    Signature varcharApply = functionSignature().returnType(parseTypeSignature("varchar")).argumentTypes(parseTypeSignature("varchar"), parseTypeSignature("function(varchar, varchar(x))", ImmutableSet.of("x"))).build();
    assertThat(varcharApply).withCoercion().boundTo("varchar(10)", new TypeSignatureProvider(paramTypes -> new FunctionType(paramTypes, createVarcharType(1)).getTypeSignature())).succeeds();
    Signature sortByKey = functionSignature().returnType(parseTypeSignature("array(T)")).argumentTypes(parseTypeSignature("array(T)"), parseTypeSignature("function(T,E)")).typeVariableConstraints(typeVariable("T"), orderableTypeParameter("E")).build();
    assertThat(sortByKey).boundTo("array(integer)", new TypeSignatureProvider(paramTypes -> new FunctionType(paramTypes, VARCHAR).getTypeSignature())).produces(BoundVariables.builder().setTypeVariable("T", INTEGER).setTypeVariable("E", VARCHAR).build());
}

Example 9 with INTEGER

use of com.facebook.presto.common.type.IntegerType.INTEGER in project presto by prestodb.

the class TestRaptorIntegrationSmokeTest method testBucketNumberHiddenColumn.

@Test
public void testBucketNumberHiddenColumn() {
    assertUpdate("" + "CREATE TABLE test_bucket_number " + "WITH (bucket_count = 50, bucketed_on = ARRAY ['orderkey']) " + "AS SELECT * FROM orders", "SELECT count(*) FROM orders");
    MaterializedResult actualResults = computeActual("SELECT DISTINCT \"$bucket_number\" FROM test_bucket_number");
    assertEquals(actualResults.getTypes(), ImmutableList.of(INTEGER));
    Set<Object> actual = actualResults.getMaterializedRows().stream().map(row -> row.getField(0)).collect(toSet());
    assertEquals(actual, IntStream.range(0, 50).boxed().collect(toSet()));
}

Example 10 with INTEGER

use of com.facebook.presto.common.type.IntegerType.INTEGER in project presto by prestodb.

the class OrcSelectiveRecordReader method createStreamReaders.

private static SelectiveStreamReader[] createStreamReaders(OrcDataSource orcDataSource, List<OrcType> types, DateTimeZone hiveStorageTimeZone, OrcRecordReaderOptions options, boolean legacyMapSubscript, Map<Integer, Type> includedColumns, List<Integer> outputColumns, Map<Integer, Map<Subfield, TupleDomainFilter>> filters, List<FilterFunction> filterFunctions, Map<Integer, Integer> filterFunctionInputMapping, Map<Integer, List<Subfield>> requiredSubfields, OrcAggregatedMemoryContext systemMemoryContext) {
    List<StreamDescriptor> streamDescriptors = createStreamDescriptor("", "", 0, types, orcDataSource).getNestedStreams();
    requireNonNull(filterFunctions, "filterFunctions is null");
    requireNonNull(filterFunctionInputMapping, "filterFunctionInputMapping is null");
    Set<Integer> filterFunctionInputColumns = filterFunctions.stream().flatMapToInt(function -> Arrays.stream(function.getInputChannels())).boxed().map(filterFunctionInputMapping::get).collect(toImmutableSet());
    OrcType rowType = types.get(0);
    SelectiveStreamReader[] streamReaders = new SelectiveStreamReader[rowType.getFieldCount()];
    for (int columnId = 0; columnId < rowType.getFieldCount(); columnId++) {
        if (includedColumns.containsKey(columnId)) {
            StreamDescriptor streamDescriptor = streamDescriptors.get(columnId);
            boolean outputRequired = outputColumns.contains(columnId) || filterFunctionInputColumns.contains(columnId);
            streamReaders[columnId] = createStreamReader(streamDescriptor, Optional.ofNullable(filters.get(columnId)).orElse(ImmutableMap.of()), outputRequired ? Optional.of(includedColumns.get(columnId)) : Optional.empty(), Optional.ofNullable(requiredSubfields.get(columnId)).orElse(ImmutableList.of()), hiveStorageTimeZone, options, legacyMapSubscript, systemMemoryContext);
        }
    }
    return streamReaders;
}