Examples with BIGINT io.prestosql.spi.type.BigintType.BIGINT used on opensource projects

Example 21 with BIGINT

use of io.prestosql.spi.type.BigintType.BIGINT in project hetu-core by openlookeng.

the class TestBaseBaseJdbcQueryGenerator method testCountStar.

@Test
public void testCountStar() {
    BiConsumer<PlanBuilder, PlanBuilder.AggregationBuilder> aggregationFunctionBuilder = ((planBuilder, aggregationBuilder) -> aggregationBuilder.addAggregation(planBuilder.symbol("agg"), getRowExpression("count(*)", defaultSessionHolder)));
    PlanNode justScan = buildPlan(planBuilder -> tableScan(planBuilder, jdbcTable, regionId, city, fare, amount));
    testJQL(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(justScan).globalGrouping())), "SELECT CAST(count(*) AS bigint) AS agg FROM (SELECT regionid, city, fare, amount FROM 'table') hetu_table_1");
}

Example 22 with BIGINT

use of io.prestosql.spi.type.BigintType.BIGINT in project hetu-core by openlookeng.

the class TestHashJoinOperator method testOuterJoinSnapshot.

@Test(dataProvider = "hashJoinTestValues")
public void testOuterJoinSnapshot(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws ExecutionException, InterruptedException {
    TaskContext taskContext = createSnapshotTaskContext();
    // build factory
    List<Type> buildTypes = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
    RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT, BIGINT)).addSequencePage(10, 20, 30, 40);
    BuildSideSetup buildSideSetup = setupBuildSide(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.empty(), false, SINGLE_STREAM_SPILLER_FACTORY, true);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    List<Type> probeTypes = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
    List<Page> probeInput = probePages.addSequencePage(10, 15, 1020, 2020).build();
    // Add markers between pages
    List<Page> withMarkers = new ArrayList<>();
    long snapshotId = 1;
    for (Page page : probeInput) {
        withMarkers.add(page);
        withMarkers.add(MarkerPage.snapshotPage(snapshotId++));
    }
    probeInput = withMarkers;
    OperatorFactory joinOperatorFactory = new LookupJoinOperators().fullOuterJoin(0, new PlanNodeId("test"), lookupSourceFactory, probePages.getTypes(), Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), OptionalInt.of(1), PARTITIONING_SPILLER_FACTORY);
    // build drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(buildSideSetup);
    // Construct lookup-outer operator
    PipelineContext buildPipeline = taskContext.addPipelineContext(2, false, true, false);
    DriverContext outerDriverContext = buildPipeline.addDriverContext(Lifespan.taskWide(), 0);
    Operator lookupOuter = ((LookupJoinOperatorFactory) joinOperatorFactory).createOuterOperatorFactory().get().getOuterOperatorFactory().createOperator(outerDriverContext);
    assertFalse(lookupOuter.isBlocked().isDone());
    // expected
    MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).row("15", 1020L, 2020L, null, null, null).row("16", 1021L, 2021L, null, null, null).row("17", 1022L, 2022L, null, null, null).row("18", 1023L, 2023L, null, null, null).row("19", 1024L, 2024L, null, null, null).row("20", 1025L, 2025L, "20", 30L, 40L).row("21", 1026L, 2026L, "21", 31L, 41L).row("22", 1027L, 2027L, "22", 32L, 42L).row("23", 1028L, 2028L, "23", 33L, 43L).row("24", 1029L, 2029L, "24", 34L, 44L).build();
    List<Integer> hashChannels = getHashChannels(probePages, buildPages);
    assertOperatorEquals(joinOperatorFactory, taskContext.addPipelineContext(0, true, true, false).addDriverContext(), probeInput, expected, true, hashChannels);
    assertTrue(lookupOuter.isBlocked().isDone());
    assertTrue(lookupOuter.getOutput() instanceof MarkerPage);
    Object state = lookupOuter.capture(null);
    lookupOuter.isBlocked().get();
    Page page = lookupOuter.getOutput();
    page = dropChannel(ImmutableList.of(page), hashChannels).get(0);
    MaterializedResult outerExpected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).row(null, null, null, "25", 35L, 45L).row(null, null, null, "26", 36L, 46L).row(null, null, null, "27", 37L, 47L).row(null, null, null, "28", 38L, 48L).row(null, null, null, "29", 39L, 49L).build();
    List<MaterializedRow> rows = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).page(page).build().getMaterializedRows();
    rows = new ArrayList<>(rows);
    rows.sort(Comparator.comparing(a -> (String) a.getField(3)));
    MaterializedResult outerResult = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).rows(rows).build();
    assertEquals(outerResult, outerExpected);
    long matched;
    if (state instanceof boolean[]) {
        boolean[] positions = (boolean[]) state;
        matched = Booleans.asList(positions).stream().filter(e -> e).count();
    } else {
        ByteBuffer bb = ByteBuffer.wrap((byte[]) state);
        List<RoaringBitmap> visitedPositions = new ArrayList<>();
        for (int i = 0; i < (parallelBuild ? PARTITION_COUNT : 1); i++) {
            ImmutableRoaringBitmap bm = new ImmutableRoaringBitmap(bb);
            visitedPositions.add(new RoaringBitmap(bm));
            bb.position(bb.position() + visitedPositions.get(i).serializedSizeInBytes());
        }
        matched = visitedPositions.stream().mapToLong(rr -> rr.getCardinality()).sum();
    }
    assertEquals(matched, 5);
}

Example 23 with BIGINT

use of io.prestosql.spi.type.BigintType.BIGINT in project hetu-core by openlookeng.

the class ElasticsearchMetadata method toPrestoType.

private Type toPrestoType(IndexMetadata.Field metaDataField, boolean isArray) {
    IndexMetadata.Type type = metaDataField.getType();
    if (isArray) {
        Type elementType = toPrestoType(metaDataField, false);
        return new ArrayType(elementType);
    }
    if (type instanceof PrimitiveType) {
        switch(((PrimitiveType) type).getName()) {
            case "float":
                return REAL;
            case "double":
                return DOUBLE;
            case "byte":
                return TINYINT;
            case "short":
                return SMALLINT;
            case "integer":
                return INTEGER;
            case "long":
                return BIGINT;
            case "string":
            case "text":
            case "keyword":
                return VARCHAR;
            case "ip":
                return ipAddressType;
            case "boolean":
                return BOOLEAN;
            case "binary":
                return VARBINARY;
            default:
                break;
        }
    } else if (type instanceof DateTimeType) {
        if (((DateTimeType) type).getFormats().isEmpty()) {
            return TIMESTAMP;
        }
    // otherwise, skip -- we don't support custom formats, yet
    } else if (type instanceof ObjectType) {
        ObjectType objectType = (ObjectType) type;
        List<RowType.Field> fields = objectType.getFields().stream().map(field -> RowType.field(field.getName(), toPrestoType(field))).collect(toImmutableList());
        return RowType.from(fields);
    }
    return null;
}

Example 24 with BIGINT

use of io.prestosql.spi.type.BigintType.BIGINT in project hetu-core by openlookeng.

the class QueryPlanner method aggregate.

private PlanBuilder aggregate(PlanBuilder inputSubPlan, QuerySpecification node) {
    PlanBuilder subPlan = inputSubPlan;
    if (!analysis.isAggregation(node)) {
        return subPlan;
    }
    // 1. Pre-project all scalar inputs (arguments and non-trivial group by expressions)
    Set<Expression> groupByExpressions = ImmutableSet.copyOf(analysis.getGroupByExpressions(node));
    ImmutableList.Builder<Expression> arguments = ImmutableList.builder();
    analysis.getAggregates(node).stream().map(FunctionCall::getArguments).flatMap(List::stream).filter(// lambda expression is generated at execution time
    exp -> !(exp instanceof LambdaExpression)).forEach(arguments::add);
    analysis.getAggregates(node).stream().map(FunctionCall::getOrderBy).filter(Optional::isPresent).map(Optional::get).map(OrderBy::getSortItems).flatMap(List::stream).map(SortItem::getSortKey).forEach(arguments::add);
    // filter expressions need to be projected first
    analysis.getAggregates(node).stream().map(FunctionCall::getFilter).filter(Optional::isPresent).map(Optional::get).forEach(arguments::add);
    Iterable<Expression> inputs = Iterables.concat(groupByExpressions, arguments.build());
    subPlan = handleSubqueries(subPlan, node, inputs);
    if (!Iterables.isEmpty(inputs)) {
        // avoid an empty projection if the only aggregation is COUNT (which has no arguments)
        subPlan = project(subPlan, inputs);
    }
    // 2. Aggregate
    // 2.a. Rewrite aggregate arguments
    TranslationMap argumentTranslations = new TranslationMap(subPlan.getRelationPlan(), analysis, lambdaDeclarationToSymbolMap);
    ImmutableList.Builder<Symbol> aggregationArgumentsBuilder = ImmutableList.builder();
    for (Expression argument : arguments.build()) {
        Symbol symbol = subPlan.translate(argument);
        argumentTranslations.put(argument, symbol);
        aggregationArgumentsBuilder.add(symbol);
    }
    List<Symbol> aggregationArguments = aggregationArgumentsBuilder.build();
    // 2.b. Rewrite grouping columns
    TranslationMap groupingTranslations = new TranslationMap(subPlan.getRelationPlan(), analysis, lambdaDeclarationToSymbolMap);
    Map<Symbol, Symbol> groupingSetMappings = new LinkedHashMap<>();
    for (Expression expression : groupByExpressions) {
        Symbol input = subPlan.translate(expression);
        Symbol output = planSymbolAllocator.newSymbol(expression, analysis.getTypeWithCoercions(expression), "gid");
        groupingTranslations.put(expression, output);
        groupingSetMappings.put(output, input);
    }
    // This tracks the grouping sets before complex expressions are considered (see comments below)
    // It's also used to compute the descriptors needed to implement grouping()
    List<Set<FieldId>> columnOnlyGroupingSets = ImmutableList.of(ImmutableSet.of());
    List<List<Symbol>> groupingSets = ImmutableList.of(ImmutableList.of());
    if (node.getGroupBy().isPresent()) {
        // For the purpose of "distinct", we need to canonicalize column references that may have varying
        // syntactic forms (e.g., "t.a" vs "a"). Thus we need to enumerate grouping sets based on the underlying
        // fieldId associated with each column reference expression.
        // The catch is that simple group-by expressions can be arbitrary expressions (this is a departure from the SQL specification).
        // But, they don't affect the number of grouping sets or the behavior of "distinct" . We can compute all the candidate
        // grouping sets in terms of fieldId, dedup as appropriate and then cross-join them with the complex expressions.
        Analysis.GroupingSetAnalysis groupingSetAnalysis = analysis.getGroupingSets(node);
        columnOnlyGroupingSets = enumerateGroupingSets(groupingSetAnalysis);
        if (node.getGroupBy().get().isDistinct()) {
            columnOnlyGroupingSets = columnOnlyGroupingSets.stream().distinct().collect(toImmutableList());
        }
        // add in the complex expressions an turn materialize the grouping sets in terms of plan columns
        ImmutableList.Builder<List<Symbol>> groupingSetBuilder = ImmutableList.builder();
        for (Set<FieldId> groupingSet : columnOnlyGroupingSets) {
            ImmutableList.Builder<Symbol> columns = ImmutableList.builder();
            groupingSetAnalysis.getComplexExpressions().stream().map(groupingTranslations::get).forEach(columns::add);
            groupingSet.stream().map(field -> groupingTranslations.get(new FieldReference(field.getFieldIndex()))).forEach(columns::add);
            groupingSetBuilder.add(columns.build());
        }
        groupingSets = groupingSetBuilder.build();
    }
    // 2.c. Generate GroupIdNode (multiple grouping sets) or ProjectNode (single grouping set)
    Optional<Symbol> groupIdSymbol = Optional.empty();
    if (groupingSets.size() > 1) {
        groupIdSymbol = Optional.of(planSymbolAllocator.newSymbol("groupId", BIGINT));
        GroupIdNode groupId = new GroupIdNode(idAllocator.getNextId(), subPlan.getRoot(), groupingSets, groupingSetMappings, aggregationArguments, groupIdSymbol.get());
        subPlan = new PlanBuilder(groupingTranslations, groupId);
    } else {
        Assignments.Builder assignments = Assignments.builder();
        aggregationArguments.forEach(symbol -> assignments.put(symbol, castToRowExpression(toSymbolReference(symbol))));
        groupingSetMappings.forEach((key, value) -> assignments.put(key, castToRowExpression(toSymbolReference(value))));
        ProjectNode project = new ProjectNode(idAllocator.getNextId(), subPlan.getRoot(), assignments.build());
        subPlan = new PlanBuilder(groupingTranslations, project);
    }
    TranslationMap aggregationTranslations = new TranslationMap(subPlan.getRelationPlan(), analysis, lambdaDeclarationToSymbolMap);
    aggregationTranslations.copyMappingsFrom(groupingTranslations);
    // 2.d. Rewrite aggregates
    ImmutableMap.Builder<Symbol, Aggregation> aggregationsBuilder = ImmutableMap.builder();
    boolean needPostProjectionCoercion = false;
    for (FunctionCall aggregate : analysis.getAggregates(node)) {
        Expression rewritten = argumentTranslations.rewrite(aggregate);
        Symbol newSymbol = planSymbolAllocator.newSymbol(rewritten, analysis.getType(aggregate));
        // Therefore we can end up with this implicit cast, and have to move it into a post-projection
        if (rewritten instanceof Cast) {
            rewritten = ((Cast) rewritten).getExpression();
            needPostProjectionCoercion = true;
        }
        aggregationTranslations.put(aggregate, newSymbol);
        FunctionCall functionCall = (FunctionCall) rewritten;
        aggregationsBuilder.put(newSymbol, new Aggregation(call(aggregate.getName().getSuffix(), analysis.getFunctionHandle(aggregate), analysis.getType(aggregate), functionCall.getArguments().stream().map(OriginalExpressionUtils::castToRowExpression).collect(toImmutableList())), functionCall.getArguments().stream().map(OriginalExpressionUtils::castToRowExpression).collect(toImmutableList()), functionCall.isDistinct(), functionCall.getFilter().map(SymbolUtils::from), functionCall.getOrderBy().map(OrderingSchemeUtils::fromOrderBy), Optional.empty()));
    }
    Map<Symbol, Aggregation> aggregations = aggregationsBuilder.build();
    ImmutableSet.Builder<Integer> globalGroupingSets = ImmutableSet.builder();
    for (int i = 0; i < groupingSets.size(); i++) {
        if (groupingSets.get(i).isEmpty()) {
            globalGroupingSets.add(i);
        }
    }
    ImmutableList.Builder<Symbol> groupingKeys = ImmutableList.builder();
    groupingSets.stream().flatMap(List::stream).distinct().forEach(groupingKeys::add);
    groupIdSymbol.ifPresent(groupingKeys::add);
    AggregationNode aggregationNode = new AggregationNode(idAllocator.getNextId(), subPlan.getRoot(), aggregations, groupingSets(groupingKeys.build(), groupingSets.size(), globalGroupingSets.build()), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), groupIdSymbol, AggregationNode.AggregationType.HASH, Optional.empty());
    subPlan = new PlanBuilder(aggregationTranslations, aggregationNode);
    // TODO: this is a hack, we should change type coercions to coerce the inputs to functions/operators instead of coercing the output
    if (needPostProjectionCoercion) {
        ImmutableList.Builder<Expression> alreadyCoerced = ImmutableList.builder();
        alreadyCoerced.addAll(groupByExpressions);
        groupIdSymbol.map(SymbolUtils::toSymbolReference).ifPresent(alreadyCoerced::add);
        subPlan = explicitCoercionFields(subPlan, alreadyCoerced.build(), analysis.getAggregates(node));
    }
    // 4. Project and re-write all grouping functions
    return handleGroupingOperations(subPlan, node, groupIdSymbol, columnOnlyGroupingSets);
}

Example 25 with BIGINT

use of io.prestosql.spi.type.BigintType.BIGINT in project hetu-core by openlookeng.

the class LogicalPlanner method createInsertCubePlan.

private RelationPlan createInsertCubePlan(Analysis analysis, InsertCube insertCubeStatement) {
    Analysis.CubeInsert insert = analysis.getCubeInsert().get();
    TableMetadata tableMetadata = metadata.getTableMetadata(session, insert.getTarget());
    List<ColumnMetadata> visibleTableColumns = tableMetadata.getColumns().stream().filter(column -> !column.isHidden()).collect(toImmutableList());
    List<String> visibleTableColumnNames = visibleTableColumns.stream().map(ColumnMetadata::getName).collect(toImmutableList());
    RelationPlan plan = createRelationPlan(analysis, insertCubeStatement.getQuery());
    Map<String, ColumnHandle> columns = metadata.getColumnHandles(session, insert.getTarget());
    Assignments.Builder assignments = Assignments.builder();
    for (ColumnMetadata column : tableMetadata.getColumns()) {
        if (column.isHidden()) {
            continue;
        }
        Symbol output = planSymbolAllocator.newSymbol(column.getName(), column.getType());
        int index = insert.getColumns().indexOf(columns.get(column.getName()));
        if (index < 0) {
            Expression cast = new Cast(new NullLiteral(), column.getType().getTypeSignature().toString());
            assignments.put(output, castToRowExpression(cast));
        } else {
            Symbol input = plan.getSymbol(index);
            Type tableType = column.getType();
            Type queryType = planSymbolAllocator.getTypes().get(input);
            if (queryType.equals(tableType) || typeCoercion.isTypeOnlyCoercion(queryType, tableType)) {
                assignments.put(output, castToRowExpression(toSymbolReference(input)));
            } else {
                Expression cast = noTruncationCast(toSymbolReference(input), queryType, tableType);
                assignments.put(output, castToRowExpression(cast));
            }
        }
    }
    ProjectNode projectNode = new ProjectNode(idAllocator.getNextId(), plan.getRoot(), assignments.build());
    List<Field> fields = visibleTableColumns.stream().map(column -> Field.newUnqualified(column.getName(), column.getType())).collect(toImmutableList());
    Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields)).build();
    plan = new RelationPlan(projectNode, scope, projectNode.getOutputSymbols());
    Optional<NewTableLayout> newTableLayout = metadata.getInsertLayout(session, insert.getTarget());
    String catalogName = insert.getTarget().getCatalogName().getCatalogName();
    TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogName, tableMetadata.getMetadata());
    RelationPlan tableWriterPlan = createTableWriterPlan(analysis, plan, new InsertReference(insert.getTarget(), analysis.isCubeOverwrite()), visibleTableColumnNames, newTableLayout, statisticsMetadata);
    Expression rewritten = null;
    Set<Identifier> predicateColumns = new HashSet<>();
    if (insertCubeStatement.getWhere().isPresent()) {
        rewritten = new QueryPlanner(analysis, planSymbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator), metadata, session, namedSubPlan, uniqueIdAllocator).rewriteExpression(tableWriterPlan, insertCubeStatement.getWhere().get(), analysis, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator));
        predicateColumns.addAll(ExpressionUtils.getIdentifiers(rewritten));
    }
    CubeMetadata cubeMetadata = insert.getMetadata();
    if (!insertCubeStatement.isOverwrite() && !insertCubeStatement.getWhere().isPresent() && cubeMetadata.getCubeStatus() != CubeStatus.INACTIVE) {
        // Means data some data was inserted before, but trying to insert entire dataset
        throw new PrestoException(QUERY_REJECTED, "Cannot allow insert. Inserting entire dataset but cube already has partial data");
    } else if (insertCubeStatement.getWhere().isPresent()) {
        if (!canSupportPredicate(rewritten)) {
            throw new PrestoException(QUERY_REJECTED, String.format("Cannot support predicate '%s'", ExpressionFormatter.formatExpression(rewritten, Optional.empty())));
        }
        if (!insertCubeStatement.isOverwrite() && arePredicatesOverlapping(rewritten, cubeMetadata)) {
            throw new PrestoException(QUERY_REJECTED, String.format("Cannot allow insert. Cube already contains data for the given predicate '%s'", ExpressionFormatter.formatExpression(insertCubeStatement.getWhere().get(), Optional.empty())));
        }
    }
    TableHandle sourceTableHandle = insert.getSourceTable();
    // At this point it has been verified that source table has not been updated
    // so insert into cube should be allowed
    LongSupplier tableLastModifiedTimeSupplier = metadata.getTableLastModifiedTimeSupplier(session, sourceTableHandle);
    checkState(tableLastModifiedTimeSupplier != null, "Table last modified time is null");
    Map<Symbol, Type> predicateColumnsType = predicateColumns.stream().map(identifier -> new Symbol(identifier.getValue())).collect(Collectors.toMap(Function.identity(), symbol -> planSymbolAllocator.getTypes().get(symbol), (key1, ignored) -> key1));
    CubeFinishNode cubeFinishNode = new CubeFinishNode(idAllocator.getNextId(), tableWriterPlan.getRoot(), planSymbolAllocator.newSymbol("rows", BIGINT), new CubeUpdateMetadata(tableMetadata.getQualifiedName().toString(), tableLastModifiedTimeSupplier.getAsLong(), rewritten != null ? ExpressionFormatter.formatExpression(rewritten, Optional.empty()) : null, insertCubeStatement.isOverwrite()), predicateColumnsType);
    return new RelationPlan(cubeFinishNode, analysis.getScope(insertCubeStatement), cubeFinishNode.getOutputSymbols());
}