Examples with Lists2.mapTail io.crate.common.collections.Lists2.mapTail used on opensource projects

Example 1 with Lists2.mapTail

use of io.crate.common.collections.Lists2.mapTail in project crate by crate.

the class RoutedCollectPhase method normalize.

/**
 * normalizes the symbols of this node with the given normalizer
 *
 * @return a normalized node, if no changes occurred returns this
 */
public RoutedCollectPhase normalize(EvaluatingNormalizer normalizer, @Nonnull TransactionContext txnCtx) {
    RoutedCollectPhase result = this;
    Function<Symbol, Symbol> normalize = s -> normalizer.normalize(s, txnCtx);
    List<Symbol> newToCollect = Lists2.map(toCollect, normalize);
    boolean changed = !newToCollect.equals(toCollect);
    Symbol newWhereClause = normalizer.normalize(where, txnCtx);
    OrderBy orderBy = this.orderBy;
    if (orderBy != null) {
        orderBy = orderBy.map(normalize);
    }
    changed = changed || newWhereClause != where || orderBy != this.orderBy;
    if (changed) {
        result = new RoutedCollectPhase(jobId(), phaseId(), name(), routing, maxRowGranularity, newToCollect, projections, newWhereClause, distributionInfo);
        result.nodePageSizeHint(nodePageSizeHint);
        result.orderBy(orderBy);
    }
    return result;
}

Example 2 with Lists2.mapTail

use of io.crate.common.collections.Lists2.mapTail in project crate by crate.

the class JoinPlanBuilder method buildJoinTree.

static LogicalPlan buildJoinTree(List<AnalyzedRelation> from, Symbol whereClause, List<JoinPair> joinPairs, Function<AnalyzedRelation, LogicalPlan> plan, boolean hashJoinEnabled) {
    if (from.size() == 1) {
        return Filter.create(plan.apply(from.get(0)), whereClause);
    }
    Map<Set<RelationName>, Symbol> queryParts = QuerySplitter.split(whereClause);
    List<JoinPair> allJoinPairs = JoinOperations.convertImplicitJoinConditionsToJoinPairs(joinPairs, queryParts);
    boolean optimizeOrder = true;
    for (var joinPair : allJoinPairs) {
        if (hasAdditionalDependencies(joinPair)) {
            optimizeOrder = false;
            break;
        }
    }
    LinkedHashMap<Set<RelationName>, JoinPair> joinPairsByRelations = JoinOperations.buildRelationsToJoinPairsMap(allJoinPairs);
    Iterator<RelationName> it;
    if (optimizeOrder) {
        Collection<RelationName> orderedRelationNames = JoinOrdering.getOrderedRelationNames(Lists2.map(from, AnalyzedRelation::relationName), joinPairsByRelations.keySet(), queryParts.keySet());
        it = orderedRelationNames.iterator();
    } else {
        it = Lists2.map(from, AnalyzedRelation::relationName).iterator();
    }
    final RelationName lhsName = it.next();
    final RelationName rhsName = it.next();
    Set<RelationName> joinNames = new HashSet<>();
    joinNames.add(lhsName);
    joinNames.add(rhsName);
    JoinPair joinLhsRhs = joinPairsByRelations.remove(joinNames);
    final JoinType joinType;
    final Symbol joinCondition;
    if (joinLhsRhs == null) {
        joinType = JoinType.CROSS;
        joinCondition = null;
    } else {
        joinType = maybeInvertPair(rhsName, joinLhsRhs);
        joinCondition = joinLhsRhs.condition();
    }
    Map<RelationName, AnalyzedRelation> sources = from.stream().collect(Collectors.toMap(AnalyzedRelation::relationName, rel -> rel));
    AnalyzedRelation lhs = sources.get(lhsName);
    AnalyzedRelation rhs = sources.get(rhsName);
    LogicalPlan lhsPlan = plan.apply(lhs);
    LogicalPlan rhsPlan = plan.apply(rhs);
    Symbol query = removeParts(queryParts, lhsName, rhsName);
    LogicalPlan joinPlan = createJoinPlan(lhsPlan, rhsPlan, joinType, joinCondition, lhs, rhs, query, hashJoinEnabled);
    joinPlan = Filter.create(joinPlan, query);
    while (it.hasNext()) {
        AnalyzedRelation nextRel = sources.get(it.next());
        joinPlan = joinWithNext(plan, joinPlan, nextRel, joinNames, joinPairsByRelations, queryParts, lhs, hashJoinEnabled);
        joinNames.add(nextRel.relationName());
    }
    if (!queryParts.isEmpty()) {
        joinPlan = Filter.create(joinPlan, AndOperator.join(queryParts.values()));
        queryParts.clear();
    }
    assert joinPairsByRelations.isEmpty() : "Must've applied all joinPairs";
    return joinPlan;
}

Example 3 with Lists2.mapTail

use of io.crate.common.collections.Lists2.mapTail in project crate by crate.

the class RefreshTablePlan method executeOrFail.

@Override
public void executeOrFail(DependencyCarrier dependencies, PlannerContext plannerContext, RowConsumer consumer, Row parameters, SubQueryResults subQueryResults) {
    if (analysis.tables().isEmpty()) {
        consumer.accept(InMemoryBatchIterator.empty(SENTINEL), null);
        return;
    }
    Function<? super Symbol, Object> eval = x -> SymbolEvaluator.evaluate(plannerContext.transactionContext(), plannerContext.nodeContext(), x, parameters, subQueryResults);
    ArrayList<String> toRefresh = new ArrayList<>();
    for (Map.Entry<Table<Symbol>, DocTableInfo> table : analysis.tables().entrySet()) {
        var tableInfo = table.getValue();
        var tableSymbol = table.getKey();
        if (tableSymbol.partitionProperties().isEmpty()) {
            toRefresh.addAll(Arrays.asList(tableInfo.concreteOpenIndices()));
        } else {
            var partitionName = toPartitionName(tableInfo, Lists2.map(tableSymbol.partitionProperties(), p -> p.map(eval)));
            if (!tableInfo.partitions().contains(partitionName)) {
                throw new PartitionUnknownException(partitionName);
            }
            toRefresh.add(partitionName.asIndexName());
        }
    }
    RefreshRequest request = new RefreshRequest(toRefresh.toArray(String[]::new));
    request.indicesOptions(IndicesOptions.lenientExpandOpen());
    var transportRefreshAction = dependencies.transportActionProvider().transportRefreshAction();
    transportRefreshAction.execute(request, new OneRowActionListener<>(consumer, response -> new Row1(toRefresh.isEmpty() ? -1L : (long) toRefresh.size())));
}

Example 4 with Lists2.mapTail

use of io.crate.common.collections.Lists2.mapTail in project crate by crate.

the class Get method build.

@Override
public ExecutionPlan build(PlannerContext plannerContext, Set<PlanHint> hints, ProjectionBuilder projectionBuilder, int limitHint, int offsetHint, @Nullable OrderBy order, @Nullable Integer pageSizeHint, Row params, SubQueryResults subQueryResults) {
    HashMap<String, Map<ShardId, List<PKAndVersion>>> idsByShardByNode = new HashMap<>();
    DocTableInfo docTableInfo = tableRelation.tableInfo();
    for (DocKeys.DocKey docKey : docKeys) {
        String id = docKey.getId(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults);
        if (id == null) {
            continue;
        }
        List<String> partitionValues = docKey.getPartitionValues(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults);
        String indexName = indexName(docTableInfo, partitionValues);
        String routing = docKey.getRouting(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults);
        ShardRouting shardRouting;
        try {
            shardRouting = plannerContext.resolveShard(indexName, id, routing);
        } catch (IndexNotFoundException e) {
            if (docTableInfo.isPartitioned()) {
                continue;
            }
            throw e;
        }
        String currentNodeId = shardRouting.currentNodeId();
        if (currentNodeId == null) {
            // If relocating is fast enough this will work, otherwise it will result in a shard failure which
            // will cause a statement retry
            currentNodeId = shardRouting.relocatingNodeId();
            if (currentNodeId == null) {
                throw new ShardNotFoundException(shardRouting.shardId());
            }
        }
        Map<ShardId, List<PKAndVersion>> idsByShard = idsByShardByNode.get(currentNodeId);
        if (idsByShard == null) {
            idsByShard = new HashMap<>();
            idsByShardByNode.put(currentNodeId, idsByShard);
        }
        List<PKAndVersion> pkAndVersions = idsByShard.get(shardRouting.shardId());
        if (pkAndVersions == null) {
            pkAndVersions = new ArrayList<>();
            idsByShard.put(shardRouting.shardId(), pkAndVersions);
        }
        long version = docKey.version(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults).orElse(Versions.MATCH_ANY);
        long sequenceNumber = docKey.sequenceNo(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults).orElse(SequenceNumbers.UNASSIGNED_SEQ_NO);
        long primaryTerm = docKey.primaryTerm(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults).orElse(SequenceNumbers.UNASSIGNED_PRIMARY_TERM);
        pkAndVersions.add(new PKAndVersion(id, version, sequenceNumber, primaryTerm));
    }
    var docKeyColumns = new ArrayList<>(docTableInfo.primaryKey());
    docKeyColumns.addAll(docTableInfo.partitionedBy());
    docKeyColumns.add(docTableInfo.clusteredBy());
    docKeyColumns.add(DocSysColumns.VERSION);
    docKeyColumns.add(DocSysColumns.SEQ_NO);
    docKeyColumns.add(DocSysColumns.PRIMARY_TERM);
    var binder = new SubQueryAndParamBinder(params, subQueryResults);
    List<Symbol> boundOutputs = Lists2.map(outputs, binder);
    var boundQuery = binder.apply(query);
    // Collect all columns which are used inside the query
    // If the query contains only DocKeys, no filter is needed as all DocKeys are handled by the PKLookupOperation
    AtomicBoolean requiresAdditionalFilteringOnNonDocKeyColumns = new AtomicBoolean(false);
    var toCollectSet = new LinkedHashSet<>(boundOutputs);
    Consumer<Reference> addRefIfMatch = ref -> {
        toCollectSet.add(ref);
        if (docKeyColumns.contains(ref.column()) == false) {
            requiresAdditionalFilteringOnNonDocKeyColumns.set(true);
        }
    };
    RefVisitor.visitRefs(boundQuery, addRefIfMatch);
    var toCollect = boundOutputs;
    ArrayList<Projection> projections = new ArrayList<>();
    if (requiresAdditionalFilteringOnNonDocKeyColumns.get()) {
        toCollect = List.copyOf(toCollectSet);
        var filterProjection = ProjectionBuilder.filterProjection(toCollect, boundQuery);
        filterProjection.requiredGranularity(RowGranularity.SHARD);
        projections.add(filterProjection);
        // reduce outputs which have been added for the filter projection
        var evalProjection = new EvalProjection(InputColumn.mapToInputColumns(boundOutputs), RowGranularity.SHARD);
        projections.add(evalProjection);
    }
    var collect = new Collect(new PKLookupPhase(plannerContext.jobId(), plannerContext.nextExecutionPhaseId(), docTableInfo.partitionedBy(), toCollect, idsByShardByNode), TopN.NO_LIMIT, 0, toCollect.size(), docKeys.size(), null);
    for (var projection : projections) {
        collect.addProjection(projection);
    }
    return collect;
}

Example 5 with Lists2.mapTail

use of io.crate.common.collections.Lists2.mapTail in project crate by crate.

the class CopyToPlan method bind.

@VisibleForTesting
public static BoundCopyTo bind(AnalyzedCopyTo copyTo, CoordinatorTxnCtx txnCtx, NodeContext nodeCtx, Row parameters, SubQueryResults subQueryResults) {
    Function<? super Symbol, Object> eval = x -> SymbolEvaluator.evaluate(txnCtx, nodeCtx, x, parameters, subQueryResults);
    DocTableInfo table = (DocTableInfo) copyTo.tableInfo();
    List<String> partitions = resolvePartitions(Lists2.map(copyTo.table().partitionProperties(), x -> x.map(eval)), table);
    List<Symbol> outputs = new ArrayList<>();
    Map<ColumnIdent, Symbol> overwrites = null;
    boolean columnsDefined = false;
    final List<String> outputNames = new ArrayList<>(copyTo.columns().size());
    if (!copyTo.columns().isEmpty()) {
        // TODO: remove outputNames?
        for (Symbol symbol : copyTo.columns()) {
            assert symbol instanceof Reference : "Only references are expected here";
            RefVisitor.visitRefs(symbol, r -> outputNames.add(r.column().sqlFqn()));
            outputs.add(DocReferences.toSourceLookup(symbol));
        }
        columnsDefined = true;
    } else {
        Reference sourceRef;
        if (table.isPartitioned() && partitions.isEmpty()) {
            // table is partitioned, insert partitioned columns into the output
            overwrites = new HashMap<>();
            for (Reference reference : table.partitionedByColumns()) {
                if (!(reference instanceof GeneratedReference)) {
                    overwrites.put(reference.column(), reference);
                }
            }
            if (overwrites.size() > 0) {
                sourceRef = table.getReference(DocSysColumns.DOC);
            } else {
                sourceRef = table.getReference(DocSysColumns.RAW);
            }
        } else {
            sourceRef = table.getReference(DocSysColumns.RAW);
        }
        outputs = List.of(sourceRef);
    }
    Settings settings = genericPropertiesToSettings(copyTo.properties().map(eval));
    WriterProjection.CompressionType compressionType = settingAsEnum(WriterProjection.CompressionType.class, COMPRESSION_SETTING.get(settings));
    WriterProjection.OutputFormat outputFormat = settingAsEnum(WriterProjection.OutputFormat.class, OUTPUT_FORMAT_SETTING.get(settings));
    if (!columnsDefined && outputFormat == WriterProjection.OutputFormat.JSON_ARRAY) {
        throw new UnsupportedFeatureException("Output format not supported without specifying columns.");
    }
    WhereClause whereClause = new WhereClause(copyTo.whereClause(), partitions, Collections.emptySet());
    return new BoundCopyTo(outputs, table, whereClause, Literal.of(DataTypes.STRING.sanitizeValue(eval.apply(copyTo.uri()))), compressionType, outputFormat, outputNames.isEmpty() ? null : outputNames, columnsDefined, overwrites, settings);
}