Examples with DocKeys io.crate.analyze.where.DocKeys used on opensource projects

Example 1 with DocKeys

use of io.crate.analyze.where.DocKeys in project crate by crate.

the class Get method build.

@Override
public ExecutionPlan build(PlannerContext plannerContext, Set<PlanHint> hints, ProjectionBuilder projectionBuilder, int limitHint, int offsetHint, @Nullable OrderBy order, @Nullable Integer pageSizeHint, Row params, SubQueryResults subQueryResults) {
    HashMap<String, Map<ShardId, List<PKAndVersion>>> idsByShardByNode = new HashMap<>();
    DocTableInfo docTableInfo = tableRelation.tableInfo();
    for (DocKeys.DocKey docKey : docKeys) {
        String id = docKey.getId(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults);
        if (id == null) {
            continue;
        }
        List<String> partitionValues = docKey.getPartitionValues(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults);
        String indexName = indexName(docTableInfo, partitionValues);
        String routing = docKey.getRouting(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults);
        ShardRouting shardRouting;
        try {
            shardRouting = plannerContext.resolveShard(indexName, id, routing);
        } catch (IndexNotFoundException e) {
            if (docTableInfo.isPartitioned()) {
                continue;
            }
            throw e;
        }
        String currentNodeId = shardRouting.currentNodeId();
        if (currentNodeId == null) {
            // If relocating is fast enough this will work, otherwise it will result in a shard failure which
            // will cause a statement retry
            currentNodeId = shardRouting.relocatingNodeId();
            if (currentNodeId == null) {
                throw new ShardNotFoundException(shardRouting.shardId());
            }
        }
        Map<ShardId, List<PKAndVersion>> idsByShard = idsByShardByNode.get(currentNodeId);
        if (idsByShard == null) {
            idsByShard = new HashMap<>();
            idsByShardByNode.put(currentNodeId, idsByShard);
        }
        List<PKAndVersion> pkAndVersions = idsByShard.get(shardRouting.shardId());
        if (pkAndVersions == null) {
            pkAndVersions = new ArrayList<>();
            idsByShard.put(shardRouting.shardId(), pkAndVersions);
        }
        long version = docKey.version(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults).orElse(Versions.MATCH_ANY);
        long sequenceNumber = docKey.sequenceNo(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults).orElse(SequenceNumbers.UNASSIGNED_SEQ_NO);
        long primaryTerm = docKey.primaryTerm(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults).orElse(SequenceNumbers.UNASSIGNED_PRIMARY_TERM);
        pkAndVersions.add(new PKAndVersion(id, version, sequenceNumber, primaryTerm));
    }
    var docKeyColumns = new ArrayList<>(docTableInfo.primaryKey());
    docKeyColumns.addAll(docTableInfo.partitionedBy());
    docKeyColumns.add(docTableInfo.clusteredBy());
    docKeyColumns.add(DocSysColumns.VERSION);
    docKeyColumns.add(DocSysColumns.SEQ_NO);
    docKeyColumns.add(DocSysColumns.PRIMARY_TERM);
    var binder = new SubQueryAndParamBinder(params, subQueryResults);
    List<Symbol> boundOutputs = Lists2.map(outputs, binder);
    var boundQuery = binder.apply(query);
    // Collect all columns which are used inside the query
    // If the query contains only DocKeys, no filter is needed as all DocKeys are handled by the PKLookupOperation
    AtomicBoolean requiresAdditionalFilteringOnNonDocKeyColumns = new AtomicBoolean(false);
    var toCollectSet = new LinkedHashSet<>(boundOutputs);
    Consumer<Reference> addRefIfMatch = ref -> {
        toCollectSet.add(ref);
        if (docKeyColumns.contains(ref.column()) == false) {
            requiresAdditionalFilteringOnNonDocKeyColumns.set(true);
        }
    };
    RefVisitor.visitRefs(boundQuery, addRefIfMatch);
    var toCollect = boundOutputs;
    ArrayList<Projection> projections = new ArrayList<>();
    if (requiresAdditionalFilteringOnNonDocKeyColumns.get()) {
        toCollect = List.copyOf(toCollectSet);
        var filterProjection = ProjectionBuilder.filterProjection(toCollect, boundQuery);
        filterProjection.requiredGranularity(RowGranularity.SHARD);
        projections.add(filterProjection);
        // reduce outputs which have been added for the filter projection
        var evalProjection = new EvalProjection(InputColumn.mapToInputColumns(boundOutputs), RowGranularity.SHARD);
        projections.add(evalProjection);
    }
    var collect = new Collect(new PKLookupPhase(plannerContext.jobId(), plannerContext.nextExecutionPhaseId(), docTableInfo.partitionedBy(), toCollect, idsByShardByNode), TopN.NO_LIMIT, 0, toCollect.size(), docKeys.size(), null);
    for (var projection : projections) {
        collect.addProjection(projection);
    }
    return collect;
}

Example 2 with DocKeys

use of io.crate.analyze.where.DocKeys in project crate by crate.

the class ShardRequestExecutor method addRequests.

private int addRequests(int location, Row parameters, Map<ShardId, Req> requests, SubQueryResults subQueryResults) {
    for (DocKeys.DocKey docKey : docKeys) {
        String id = docKey.getId(txnCtx, nodeCtx, parameters, subQueryResults);
        if (id == null) {
            continue;
        }
        String routing = docKey.getRouting(txnCtx, nodeCtx, parameters, subQueryResults);
        List<String> partitionValues = docKey.getPartitionValues(txnCtx, nodeCtx, parameters, subQueryResults);
        final String indexName;
        if (partitionValues == null) {
            indexName = table.ident().indexNameOrAlias();
        } else {
            indexName = IndexParts.toIndexName(table.ident(), PartitionName.encodeIdent(partitionValues));
        }
        final ShardId shardId;
        try {
            shardId = getShardId(clusterService, indexName, id, routing);
        } catch (IndexNotFoundException e) {
            if (table.isPartitioned()) {
                continue;
            }
            throw e;
        }
        Req request = requests.get(shardId);
        if (request == null) {
            request = grouper.newRequest(shardId);
            requests.put(shardId, request);
        }
        Long version = docKey.version(txnCtx, nodeCtx, parameters, subQueryResults).orElse(null);
        Long seqNo = docKey.sequenceNo(txnCtx, nodeCtx, parameters, subQueryResults).orElse(null);
        Long primaryTerm = docKey.primaryTerm(txnCtx, nodeCtx, parameters, subQueryResults).orElse(null);
        grouper.addItem(request, location, id, version, seqNo, primaryTerm);
        location++;
    }
    return location;
}

Example 3 with DocKeys

use of io.crate.analyze.where.DocKeys in project crate by crate.

the class WhereClauseOptimizer method optimize.

public static DetailedQuery optimize(EvaluatingNormalizer normalizer, Symbol query, DocTableInfo table, TransactionContext txnCtx, NodeContext nodeCtx) {
    Symbol queryGenColsProcessed = GeneratedColumnExpander.maybeExpand(query, table.generatedColumns(), Lists2.concat(table.partitionedByColumns(), Lists2.map(table.primaryKey(), table::getReference)), nodeCtx);
    if (!query.equals(queryGenColsProcessed)) {
        query = normalizer.normalize(queryGenColsProcessed, txnCtx);
    }
    WhereClause.validateVersioningColumnsUsage(query);
    boolean versionInQuery = Symbols.containsColumn(query, DocSysColumns.VERSION);
    boolean sequenceVersioningInQuery = Symbols.containsColumn(query, DocSysColumns.SEQ_NO) && Symbols.containsColumn(query, DocSysColumns.PRIMARY_TERM);
    List<ColumnIdent> pkCols = pkColsInclVersioning(table, versionInQuery, sequenceVersioningInQuery);
    EqualityExtractor eqExtractor = new EqualityExtractor(normalizer);
    List<List<Symbol>> pkValues = eqExtractor.extractExactMatches(pkCols, query, txnCtx);
    int clusterIdxWithinPK = table.primaryKey().indexOf(table.clusteredBy());
    final DocKeys docKeys;
    if (pkValues == null) {
        docKeys = null;
    } else {
        List<Integer> partitionIndicesWithinPks = null;
        if (table.isPartitioned()) {
            partitionIndicesWithinPks = getPartitionIndices(table.primaryKey(), table.partitionedBy());
        }
        docKeys = new DocKeys(pkValues, versionInQuery, sequenceVersioningInQuery, clusterIdxWithinPK, partitionIndicesWithinPks);
    }
    List<List<Symbol>> partitionValues = null;
    if (table.isPartitioned()) {
        partitionValues = eqExtractor.extractExactMatches(table.partitionedBy(), query, txnCtx);
    }
    Set<Symbol> clusteredBy = Collections.emptySet();
    if (table.clusteredBy() != null) {
        List<List<Symbol>> clusteredByValues = eqExtractor.extractParentMatches(Collections.singletonList(table.clusteredBy()), query, txnCtx);
        if (clusteredByValues != null) {
            clusteredBy = new HashSet<>(clusteredByValues.size());
            for (List<Symbol> s : clusteredByValues) {
                clusteredBy.add(s.get(0));
            }
        }
    }
    WhereClauseValidator.validate(query);
    return new DetailedQuery(query, docKeys, partitionValues, clusteredBy);
}

Example 4 with DocKeys

use of io.crate.analyze.where.DocKeys in project crate by crate.

the class ESGetStatementPlanner method convert.

public static Plan convert(QueriedDocTable table, Planner.Context context) {
    QuerySpec querySpec = table.querySpec();
    Optional<DocKeys> optKeys = querySpec.where().docKeys();
    assert !querySpec.hasAggregates() : "Can't create ESGet plan for queries with aggregates";
    assert !querySpec.groupBy().isPresent() : "Can't create ESGet plan for queries with group by";
    assert optKeys.isPresent() : "Can't create ESGet without docKeys";
    DocTableInfo tableInfo = table.tableRelation().tableInfo();
    DocKeys docKeys = optKeys.get();
    if (docKeys.withVersions()) {
        throw new VersionInvalidException();
    }
    Limits limits = context.getLimits(querySpec);
    if (limits.hasLimit() && limits.finalLimit() == 0) {
        return new NoopPlan(context.jobId());
    }
    table.tableRelation().validateOrderBy(querySpec.orderBy());
    return new ESGet(context.nextExecutionPhaseId(), tableInfo, querySpec.outputs(), optKeys.get(), querySpec.orderBy(), limits.finalLimit(), limits.offset(), context.jobId());
}

Example 5 with DocKeys

use of io.crate.analyze.where.DocKeys in project crate by crate.

the class DeleteStatementPlanner method planDelete.

public static Plan planDelete(DeleteAnalyzedStatement analyzedStatement, Planner.Context context) {
    DocTableRelation tableRelation = analyzedStatement.analyzedRelation();
    List<WhereClause> whereClauses = new ArrayList<>(analyzedStatement.whereClauses().size());
    List<DocKeys.DocKey> docKeys = new ArrayList<>(analyzedStatement.whereClauses().size());
    Map<Integer, Integer> itemToBulkIdx = new HashMap<>();
    int bulkIdx = -1;
    int itemIdx = 0;
    for (WhereClause whereClause : analyzedStatement.whereClauses()) {
        bulkIdx++;
        if (whereClause.noMatch()) {
            continue;
        }
        if (whereClause.docKeys().isPresent() && whereClause.docKeys().get().size() == 1) {
            DocKeys.DocKey docKey = whereClause.docKeys().get().getOnlyKey();
            if (docKey.id() != null) {
                docKeys.add(docKey);
                itemToBulkIdx.put(itemIdx, bulkIdx);
                itemIdx++;
            }
        } else if (!whereClause.noMatch()) {
            whereClauses.add(whereClause);
        }
    }
    if (!docKeys.isEmpty()) {
        return new ESDelete(context.jobId(), context.nextExecutionPhaseId(), tableRelation.tableInfo(), docKeys, itemToBulkIdx, analyzedStatement.whereClauses().size());
    } else if (!whereClauses.isEmpty()) {
        return deleteByQuery(tableRelation.tableInfo(), whereClauses, context);
    }
    return new NoopPlan(context.jobId());
}