Examples with BIGINT com.facebook.presto.common.type.BigintType.BIGINT used on opensource projects

Example 1 with BIGINT

use of com.facebook.presto.common.type.BigintType.BIGINT in project presto by prestodb.

the class TestDatabaseShardManager method testReplaceShardUuidsFunction.

@Test
public void testReplaceShardUuidsFunction() throws SQLException {
    // node1 shard1 shard4
    // node2 shard2
    // node3 shard3
    // goal: replace shard1 and shard4 with newUuid5
    // Initial data
    long tableId = createTable("test");
    List<ColumnInfo> columns = ImmutableList.of(new ColumnInfo(1, BIGINT));
    UUID uuid1 = UUID.randomUUID();
    UUID uuid2 = UUID.randomUUID();
    UUID uuid3 = UUID.randomUUID();
    UUID uuid4 = UUID.randomUUID();
    ShardInfo shardInfo1 = new ShardInfo(uuid1, OptionalInt.empty(), ImmutableSet.of("node1"), ImmutableList.of(), 1, 1, 1, 1);
    ShardInfo shardInfo2 = new ShardInfo(uuid2, OptionalInt.empty(), ImmutableSet.of("node2"), ImmutableList.of(), 2, 2, 2, 2);
    ShardInfo shardInfo3 = new ShardInfo(uuid3, OptionalInt.empty(), ImmutableSet.of("node3"), ImmutableList.of(), 3, 3, 3, 3);
    ShardInfo shardInfo4 = new ShardInfo(uuid4, OptionalInt.empty(), ImmutableSet.of("node1"), ImmutableList.of(), 4, 4, 4, 4);
    shardManager.createTable(tableId, columns, false, OptionalLong.empty(), true);
    long transactionId = shardManager.beginTransaction();
    shardManager.commitShards(transactionId, tableId, columns, ImmutableList.of(shardInfo1, shardInfo2, shardInfo3, shardInfo4), Optional.empty(), 0);
    // New data
    UUID newUuid5 = UUID.randomUUID();
    ShardInfo newShardInfo4 = new ShardInfo(newUuid5, OptionalInt.empty(), ImmutableSet.of("node1"), ImmutableList.of(), 5, 5, 5, 5);
    // toReplace
    Set<ShardMetadata> shardMetadata = shardManager.getNodeShardsAndDeltas("node1");
    Set<UUID> replacedUuids = shardMetadata.stream().map(ShardMetadata::getShardUuid).collect(toSet());
    Map<UUID, Optional<UUID>> replaceUuidMap = replacedUuids.stream().collect(Collectors.toMap(uuid -> uuid, uuid -> Optional.empty()));
    transactionId = shardManager.beginTransaction();
    shardManager.replaceShardUuids(transactionId, tableId, columns, replaceUuidMap, ImmutableList.of(newShardInfo4), OptionalLong.of(0), true);
    // check shards on this node1 are correct
    shardMetadata = shardManager.getNodeShardsAndDeltas("node1");
    assertEquals(shardMetadata.size(), 1);
    for (ShardMetadata actual : shardMetadata) {
        assertEquals(actual.getShardUuid(), newUuid5);
        assertEquals(actual.getDeltaUuid(), Optional.empty());
        assertEquals(actual.getRowCount(), 5);
        assertEquals(actual.getCompressedSize(), 5);
        assertEquals(actual.getUncompressedSize(), 5);
    }
    // check that shards are replaced in index table as well
    Set<BucketShards> shardNodes = ImmutableSet.copyOf(shardManager.getShardNodes(tableId, TupleDomain.all(), true));
    Set<UUID> actualAllUuids = shardNodes.stream().map(BucketShards::getShards).flatMap(Collection::stream).map(ShardNodes::getShardUuid).collect(toSet());
    Set<UUID> expectedAllUuids = ImmutableSet.of(uuid2, uuid3, newUuid5);
    assertEquals(actualAllUuids, expectedAllUuids);
    // Verify statistics
    Statement statement = dummyHandle.getConnection().createStatement();
    ResultSet resultSet = statement.executeQuery("SELECT * FROM tables where table_id = " + tableId);
    resultSet.next();
    assertEquals(resultSet.getLong("shard_count"), 3);
    assertEquals(resultSet.getLong("delta_count"), 0);
    assertEquals(resultSet.getLong("row_count"), 10);
    assertEquals(resultSet.getLong("compressed_size"), 10);
    assertEquals(resultSet.getLong("uncompressed_size"), 10);
    resultSet.close();
    statement.close();
}

Example 2 with BIGINT

use of com.facebook.presto.common.type.BigintType.BIGINT in project presto by prestodb.

the class TestHashJoinOperator method testYield.

@Test
public void testYield() {
    // create a filter function that yields for every probe match
    // verify we will yield #match times totally
    TaskContext taskContext = createTaskContext();
    DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
    // force a yield for every match
    AtomicInteger filterFunctionCalls = new AtomicInteger();
    InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction(((leftPosition, leftPage, rightPosition, rightPage) -> {
        filterFunctionCalls.incrementAndGet();
        driverContext.getYieldSignal().forceYieldForTesting();
        return true;
    }));
    // build with 40 entries
    int entries = 40;
    RowPagesBuilder buildPages = rowPagesBuilder(true, Ints.asList(0), ImmutableList.of(BIGINT)).addSequencePage(entries, 42);
    BuildSideSetup buildSideSetup = setupBuildSide(true, taskContext, Ints.asList(0), buildPages, Optional.of(filterFunction), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
    // probe matching the above 40 entries
    RowPagesBuilder probePages = rowPagesBuilder(false, Ints.asList(0), ImmutableList.of(BIGINT));
    List<Page> probeInput = probePages.addSequencePage(100, 0).build();
    OperatorFactory joinOperatorFactory = LOOKUP_JOIN_OPERATORS.innerJoin(0, new PlanNodeId("test"), lookupSourceFactory, probePages.getTypes(), Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), OptionalInt.of(1), PARTITIONING_SPILLER_FACTORY);
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(buildSideSetup);
    Operator operator = joinOperatorFactory.createOperator(driverContext);
    assertTrue(operator.needsInput());
    operator.addInput(probeInput.get(0));
    operator.finish();
    // we will yield 40 times due to filterFunction
    for (int i = 0; i < entries; i++) {
        driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor());
        filterFunctionCalls.set(0);
        assertNull(operator.getOutput());
        assertEquals(filterFunctionCalls.get(), 1, "Expected join to stop processing (yield) after calling filter function once");
        driverContext.getYieldSignal().reset();
    }
    // delayed yield is not going to prevent operator from producing a page now (yield won't be forced because filter function won't be called anymore)
    driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor());
    // expect output page to be produced within few calls to getOutput(), e.g. to facilitate spill
    Page output = null;
    for (int i = 0; output == null && i < 5; i++) {
        output = operator.getOutput();
    }
    assertNotNull(output);
    driverContext.getYieldSignal().reset();
    // make sure we have all 4 entries
    assertEquals(output.getPositionCount(), entries);
}

Example 3 with BIGINT

use of com.facebook.presto.common.type.BigintType.BIGINT in project presto by prestodb.

the class TestSignatureBinder method testFunction.

@Test
public void testFunction() {
    Signature simple = functionSignature().returnType(parseTypeSignature("boolean")).argumentTypes(parseTypeSignature("function(integer,integer)")).build();
    assertThat(simple).boundTo("integer").fails();
    assertThat(simple).boundTo("function(integer,integer)").succeeds();
    // TODO: Support coercion of return type of lambda
    assertThat(simple).boundTo("function(integer,smallint)").withCoercion().fails();
    assertThat(simple).boundTo("function(integer,bigint)").withCoercion().fails();
    Signature applyTwice = functionSignature().returnType(parseTypeSignature("V")).argumentTypes(parseTypeSignature("T"), parseTypeSignature("function(T,U)"), parseTypeSignature("function(U,V)")).typeVariableConstraints(typeVariable("T"), typeVariable("U"), typeVariable("V")).build();
    assertThat(applyTwice).boundTo("integer", "integer", "integer").fails();
    assertThat(applyTwice).boundTo("integer", "function(integer,varchar)", "function(varchar,double)").produces(BoundVariables.builder().setTypeVariable("T", INTEGER).setTypeVariable("U", VARCHAR).setTypeVariable("V", DOUBLE).build());
    assertThat(applyTwice).boundTo("integer", new TypeSignatureProvider(functionArgumentTypes -> TypeSignature.parseTypeSignature("function(integer,varchar)")), new TypeSignatureProvider(functionArgumentTypes -> TypeSignature.parseTypeSignature("function(varchar,double)"))).produces(BoundVariables.builder().setTypeVariable("T", INTEGER).setTypeVariable("U", VARCHAR).setTypeVariable("V", DOUBLE).build());
    assertThat(applyTwice).boundTo(// pass function argument to non-function position of a function
    new TypeSignatureProvider(functionArgumentTypes -> TypeSignature.parseTypeSignature("function(integer,varchar)")), new TypeSignatureProvider(functionArgumentTypes -> TypeSignature.parseTypeSignature("function(integer,varchar)")), new TypeSignatureProvider(functionArgumentTypes -> TypeSignature.parseTypeSignature("function(varchar,double)"))).fails();
    assertThat(applyTwice).boundTo(new TypeSignatureProvider(functionArgumentTypes -> TypeSignature.parseTypeSignature("function(integer,varchar)")), // pass non-function argument to function position of a function
    "integer", new TypeSignatureProvider(functionArgumentTypes -> TypeSignature.parseTypeSignature("function(varchar,double)"))).fails();
    Signature flatMap = functionSignature().returnType(parseTypeSignature("array(T)")).argumentTypes(parseTypeSignature("array(T)"), parseTypeSignature("function(T, array(T))")).typeVariableConstraints(typeVariable("T")).build();
    assertThat(flatMap).boundTo("array(integer)", "function(integer, array(integer))").produces(BoundVariables.builder().setTypeVariable("T", INTEGER).build());
    Signature varargApply = functionSignature().returnType(parseTypeSignature("T")).argumentTypes(parseTypeSignature("T"), parseTypeSignature("function(T, T)")).typeVariableConstraints(typeVariable("T")).setVariableArity(true).build();
    assertThat(varargApply).boundTo("integer", "function(integer, integer)", "function(integer, integer)", "function(integer, integer)").produces(BoundVariables.builder().setTypeVariable("T", INTEGER).build());
    assertThat(varargApply).boundTo("integer", "function(integer, integer)", "function(integer, double)", "function(double, double)").fails();
    Signature loop = functionSignature().returnType(parseTypeSignature("T")).argumentTypes(parseTypeSignature("T"), parseTypeSignature("function(T, T)")).typeVariableConstraints(typeVariable("T")).build();
    assertThat(loop).boundTo("integer", new TypeSignatureProvider(paramTypes -> new FunctionType(paramTypes, BIGINT).getTypeSignature())).fails();
    assertThat(loop).boundTo("integer", new TypeSignatureProvider(paramTypes -> new FunctionType(paramTypes, BIGINT).getTypeSignature())).withCoercion().produces(BoundVariables.builder().setTypeVariable("T", BIGINT).build());
    // TODO: Support coercion of return type of lambda
    assertThat(loop).withCoercion().boundTo("integer", new TypeSignatureProvider(paramTypes -> new FunctionType(paramTypes, SMALLINT).getTypeSignature())).fails();
    // TODO: Support coercion of return type of lambda
    // Without coercion support for return type of lambda, the return type of lambda must be `varchar(x)` to avoid need for coercions.
    Signature varcharApply = functionSignature().returnType(parseTypeSignature("varchar")).argumentTypes(parseTypeSignature("varchar"), parseTypeSignature("function(varchar, varchar(x))", ImmutableSet.of("x"))).build();
    assertThat(varcharApply).withCoercion().boundTo("varchar(10)", new TypeSignatureProvider(paramTypes -> new FunctionType(paramTypes, createVarcharType(1)).getTypeSignature())).succeeds();
    Signature sortByKey = functionSignature().returnType(parseTypeSignature("array(T)")).argumentTypes(parseTypeSignature("array(T)"), parseTypeSignature("function(T,E)")).typeVariableConstraints(typeVariable("T"), orderableTypeParameter("E")).build();
    assertThat(sortByKey).boundTo("array(integer)", new TypeSignatureProvider(paramTypes -> new FunctionType(paramTypes, VARCHAR).getTypeSignature())).produces(BoundVariables.builder().setTypeVariable("T", INTEGER).setTypeVariable("E", VARCHAR).build());
}

Example 4 with BIGINT

use of com.facebook.presto.common.type.BigintType.BIGINT in project presto by prestodb.

the class AbstractTestOrcReader method testCaching.

@Test
public void testCaching() throws Exception {
    Cache<OrcDataSourceId, OrcFileTail> orcFileTailCache = CacheBuilder.newBuilder().maximumWeight(new DataSize(1, MEGABYTE).toBytes()).weigher((id, tail) -> ((OrcFileTail) tail).getFooterSize() + ((OrcFileTail) tail).getMetadataSize()).expireAfterAccess(new Duration(10, MINUTES).toMillis(), MILLISECONDS).recordStats().build();
    OrcFileTailSource orcFileTailSource = new CachingOrcFileTailSource(new StorageOrcFileTailSource(), orcFileTailCache);
    Cache<StripeId, Slice> stripeFootercache = CacheBuilder.newBuilder().maximumWeight(new DataSize(1, MEGABYTE).toBytes()).weigher((id, footer) -> ((Slice) footer).length()).expireAfterAccess(new Duration(10, MINUTES).toMillis(), MILLISECONDS).recordStats().build();
    Cache<StripeStreamId, Slice> stripeStreamCache = CacheBuilder.newBuilder().maximumWeight(new DataSize(1, MEGABYTE).toBytes()).weigher((id, stream) -> ((Slice) stream).length()).expireAfterAccess(new Duration(10, MINUTES).toMillis(), MILLISECONDS).recordStats().build();
    Optional<Cache<StripeStreamId, List<RowGroupIndex>>> rowGroupIndexCache = Optional.of(CacheBuilder.newBuilder().maximumWeight(new DataSize(1, MEGABYTE).toBytes()).weigher((id, rowGroupIndices) -> toIntExact(((List<RowGroupIndex>) rowGroupIndices).stream().mapToLong(RowGroupIndex::getRetainedSizeInBytes).sum())).expireAfterAccess(new Duration(10, MINUTES).toMillis(), MILLISECONDS).recordStats().build());
    StripeMetadataSource stripeMetadataSource = new CachingStripeMetadataSource(new StorageStripeMetadataSource(), stripeFootercache, stripeStreamCache, rowGroupIndexCache);
    try (TempFile tempFile = createTempFile(10001)) {
        OrcBatchRecordReader storageReader = createCustomOrcRecordReader(tempFile, ORC, OrcPredicate.TRUE, ImmutableList.of(BIGINT), INITIAL_BATCH_SIZE, orcFileTailSource, stripeMetadataSource, true, ImmutableMap.of(), false);
        assertEquals(orcFileTailCache.stats().missCount(), 1);
        assertEquals(orcFileTailCache.stats().hitCount(), 0);
        OrcBatchRecordReader cacheReader = createCustomOrcRecordReader(tempFile, ORC, OrcPredicate.TRUE, ImmutableList.of(BIGINT), INITIAL_BATCH_SIZE, orcFileTailSource, stripeMetadataSource, true, ImmutableMap.of(), false);
        assertEquals(orcFileTailCache.stats().missCount(), 1);
        assertEquals(orcFileTailCache.stats().hitCount(), 1);
        assertEquals(storageReader.getRetainedSizeInBytes(), cacheReader.getRetainedSizeInBytes());
        assertEquals(storageReader.getFileRowCount(), cacheReader.getFileRowCount());
        assertEquals(storageReader.getSplitLength(), cacheReader.getSplitLength());
        storageReader.nextBatch();
        assertEquals(stripeFootercache.stats().missCount(), 1);
        assertEquals(stripeFootercache.stats().hitCount(), 0);
        assertEquals(stripeStreamCache.stats().missCount(), 2);
        assertEquals(stripeStreamCache.stats().hitCount(), 0);
        assertEquals(rowGroupIndexCache.get().stats().missCount(), 1);
        assertEquals(rowGroupIndexCache.get().stats().hitCount(), 0);
        cacheReader.nextBatch();
        assertEquals(stripeFootercache.stats().missCount(), 1);
        assertEquals(stripeFootercache.stats().hitCount(), 1);
        assertEquals(stripeStreamCache.stats().missCount(), 2);
        assertEquals(stripeStreamCache.stats().hitCount(), 2);
        assertEquals(rowGroupIndexCache.get().stats().missCount(), 1);
        assertEquals(rowGroupIndexCache.get().stats().hitCount(), 1);
        assertEquals(storageReader.readBlock(0).getInt(0), cacheReader.readBlock(0).getInt(0));
    }
}

Example 5 with BIGINT

use of com.facebook.presto.common.type.BigintType.BIGINT in project presto by prestodb.

the class TestCassandraIntegrationSmokeTest method assertSelect.

private void assertSelect(String tableName, boolean createdByPresto) {
    Type uuidType = createdByPresto ? createUnboundedVarcharType() : createVarcharType(36);
    Type inetType = createdByPresto ? createUnboundedVarcharType() : createVarcharType(45);
    String sql = "SELECT " + " key, " + " typeuuid, " + " typeinteger, " + " typelong, " + " typebytes, " + " typetimestamp, " + " typeansi, " + " typeboolean, " + " typedecimal, " + " typedouble, " + " typefloat, " + " typeinet, " + " typevarchar, " + " typevarint, " + " typetimeuuid, " + " typelist, " + " typemap, " + " typeset " + " FROM " + tableName;
    MaterializedResult result = execute(sql);
    int rowCount = result.getRowCount();
    assertEquals(rowCount, 9);
    assertEquals(result.getTypes(), ImmutableList.of(createUnboundedVarcharType(), uuidType, INTEGER, BIGINT, VARBINARY, TIMESTAMP, createUnboundedVarcharType(), BOOLEAN, DOUBLE, DOUBLE, REAL, inetType, createUnboundedVarcharType(), createUnboundedVarcharType(), uuidType, createUnboundedVarcharType(), createUnboundedVarcharType(), createUnboundedVarcharType()));
    List<MaterializedRow> sortedRows = result.getMaterializedRows().stream().sorted((o1, o2) -> o1.getField(1).toString().compareTo(o2.getField(1).toString())).collect(toList());
    for (int rowNumber = 1; rowNumber <= rowCount; rowNumber++) {
        assertEquals(sortedRows.get(rowNumber - 1), new MaterializedRow(DEFAULT_PRECISION, "key " + rowNumber, String.format("00000000-0000-0000-0000-%012d", rowNumber), rowNumber, rowNumber + 1000L, ByteBuffer.wrap(toByteArray(rowNumber)), TIMESTAMP_LOCAL, "ansi " + rowNumber, rowNumber % 2 == 0, Math.pow(2, rowNumber), Math.pow(4, rowNumber), (float) Math.pow(8, rowNumber), "127.0.0.1", "varchar " + rowNumber, BigInteger.TEN.pow(rowNumber).toString(), String.format("d2177dd0-eaa2-11de-a572-001b779c76e%d", rowNumber), String.format("[\"list-value-1%1$d\",\"list-value-2%1$d\"]", rowNumber), String.format("{%d:%d,%d:%d}", rowNumber, rowNumber + 1L, rowNumber + 2, rowNumber + 3L), "[false,true]"));
    }
}