Examples with ShardNotFoundException org.elasticsearch.index.shard.ShardNotFoundException used on opensource projects

Example 11 with ShardNotFoundException

use of org.elasticsearch.index.shard.ShardNotFoundException in project crate by crate.

the class Get method build.

@Override
public ExecutionPlan build(PlannerContext plannerContext, Set<PlanHint> hints, ProjectionBuilder projectionBuilder, int limitHint, int offsetHint, @Nullable OrderBy order, @Nullable Integer pageSizeHint, Row params, SubQueryResults subQueryResults) {
    HashMap<String, Map<ShardId, List<PKAndVersion>>> idsByShardByNode = new HashMap<>();
    DocTableInfo docTableInfo = tableRelation.tableInfo();
    for (DocKeys.DocKey docKey : docKeys) {
        String id = docKey.getId(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults);
        if (id == null) {
            continue;
        }
        List<String> partitionValues = docKey.getPartitionValues(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults);
        String indexName = indexName(docTableInfo, partitionValues);
        String routing = docKey.getRouting(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults);
        ShardRouting shardRouting;
        try {
            shardRouting = plannerContext.resolveShard(indexName, id, routing);
        } catch (IndexNotFoundException e) {
            if (docTableInfo.isPartitioned()) {
                continue;
            }
            throw e;
        }
        String currentNodeId = shardRouting.currentNodeId();
        if (currentNodeId == null) {
            // If relocating is fast enough this will work, otherwise it will result in a shard failure which
            // will cause a statement retry
            currentNodeId = shardRouting.relocatingNodeId();
            if (currentNodeId == null) {
                throw new ShardNotFoundException(shardRouting.shardId());
            }
        }
        Map<ShardId, List<PKAndVersion>> idsByShard = idsByShardByNode.get(currentNodeId);
        if (idsByShard == null) {
            idsByShard = new HashMap<>();
            idsByShardByNode.put(currentNodeId, idsByShard);
        }
        List<PKAndVersion> pkAndVersions = idsByShard.get(shardRouting.shardId());
        if (pkAndVersions == null) {
            pkAndVersions = new ArrayList<>();
            idsByShard.put(shardRouting.shardId(), pkAndVersions);
        }
        long version = docKey.version(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults).orElse(Versions.MATCH_ANY);
        long sequenceNumber = docKey.sequenceNo(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults).orElse(SequenceNumbers.UNASSIGNED_SEQ_NO);
        long primaryTerm = docKey.primaryTerm(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults).orElse(SequenceNumbers.UNASSIGNED_PRIMARY_TERM);
        pkAndVersions.add(new PKAndVersion(id, version, sequenceNumber, primaryTerm));
    }
    var docKeyColumns = new ArrayList<>(docTableInfo.primaryKey());
    docKeyColumns.addAll(docTableInfo.partitionedBy());
    docKeyColumns.add(docTableInfo.clusteredBy());
    docKeyColumns.add(DocSysColumns.VERSION);
    docKeyColumns.add(DocSysColumns.SEQ_NO);
    docKeyColumns.add(DocSysColumns.PRIMARY_TERM);
    var binder = new SubQueryAndParamBinder(params, subQueryResults);
    List<Symbol> boundOutputs = Lists2.map(outputs, binder);
    var boundQuery = binder.apply(query);
    // Collect all columns which are used inside the query
    // If the query contains only DocKeys, no filter is needed as all DocKeys are handled by the PKLookupOperation
    AtomicBoolean requiresAdditionalFilteringOnNonDocKeyColumns = new AtomicBoolean(false);
    var toCollectSet = new LinkedHashSet<>(boundOutputs);
    Consumer<Reference> addRefIfMatch = ref -> {
        toCollectSet.add(ref);
        if (docKeyColumns.contains(ref.column()) == false) {
            requiresAdditionalFilteringOnNonDocKeyColumns.set(true);
        }
    };
    RefVisitor.visitRefs(boundQuery, addRefIfMatch);
    var toCollect = boundOutputs;
    ArrayList<Projection> projections = new ArrayList<>();
    if (requiresAdditionalFilteringOnNonDocKeyColumns.get()) {
        toCollect = List.copyOf(toCollectSet);
        var filterProjection = ProjectionBuilder.filterProjection(toCollect, boundQuery);
        filterProjection.requiredGranularity(RowGranularity.SHARD);
        projections.add(filterProjection);
        // reduce outputs which have been added for the filter projection
        var evalProjection = new EvalProjection(InputColumn.mapToInputColumns(boundOutputs), RowGranularity.SHARD);
        projections.add(evalProjection);
    }
    var collect = new Collect(new PKLookupPhase(plannerContext.jobId(), plannerContext.nextExecutionPhaseId(), docTableInfo.partitionedBy(), toCollect, idsByShardByNode), TopN.NO_LIMIT, 0, toCollect.size(), docKeys.size(), null);
    for (var projection : projections) {
        collect.addProjection(projection);
    }
    return collect;
}

Example 12 with ShardNotFoundException

use of org.elasticsearch.index.shard.ShardNotFoundException in project crate by crate.

the class ShardCollectSource method createMultiShardScoreDocCollector.

private CompletableFuture<BatchIterator<Row>> createMultiShardScoreDocCollector(RoutedCollectPhase collectPhase, boolean supportMoveToStart, CollectTask collectTask, String localNodeId) {
    Map<String, Map<String, IntIndexedContainer>> locations = collectPhase.routing().locations();
    SharedShardContexts sharedShardContexts = collectTask.sharedShardContexts();
    Map<String, IntIndexedContainer> indexShards = locations.get(localNodeId);
    List<CompletableFuture<OrderedDocCollector>> orderedDocCollectors = new ArrayList<>();
    Metadata metadata = clusterService.state().metadata();
    for (Map.Entry<String, IntIndexedContainer> entry : indexShards.entrySet()) {
        String indexName = entry.getKey();
        Index index = metadata.index(indexName).getIndex();
        for (IntCursor shard : entry.getValue()) {
            ShardId shardId = new ShardId(index, shard.value);
            try {
                SharedShardContext context = sharedShardContexts.getOrCreateContext(shardId);
                ShardCollectorProvider shardCollectorProvider = getCollectorProviderSafe(shardId);
                orderedDocCollectors.add(shardCollectorProvider.getFutureOrderedCollector(collectPhase, context, collectTask, supportMoveToStart));
            } catch (ShardNotFoundException | IllegalIndexShardStateException e) {
                throw e;
            } catch (IndexNotFoundException e) {
                if (IndexParts.isPartitioned(indexName)) {
                    break;
                }
                throw e;
            }
        }
    }
    List<DataType<?>> columnTypes = Symbols.typeView(collectPhase.toCollect());
    OrderBy orderBy = collectPhase.orderBy();
    assert orderBy != null : "orderBy must not be null";
    return CompletableFutures.allAsList(orderedDocCollectors).thenApply(collectors -> OrderedLuceneBatchIteratorFactory.newInstance(collectors, OrderingByPosition.rowOrdering(OrderByPositionVisitor.orderByPositions(orderBy.orderBySymbols(), collectPhase.toCollect()), orderBy.reverseFlags(), orderBy.nullsFirst()), new RowAccountingWithEstimators(columnTypes, collectTask.getRamAccounting()), executor, availableThreads, supportMoveToStart));
}

Example 13 with ShardNotFoundException

use of org.elasticsearch.index.shard.ShardNotFoundException in project crate by crate.

the class ShardCollectSource method getShardsIterator.

private Iterable<Row> getShardsIterator(TransactionContext txnCtx, RoutedCollectPhase collectPhase, String localNodeId) {
    Map<String, Map<String, IntIndexedContainer>> locations = collectPhase.routing().locations();
    List<UnassignedShard> unassignedShards = new ArrayList<>();
    List<ShardRowContext> shardRowContexts = new ArrayList<>();
    Map<String, IntIndexedContainer> indexShardsMap = locations.get(localNodeId);
    Metadata metadata = clusterService.state().metadata();
    for (Map.Entry<String, IntIndexedContainer> indexShards : indexShardsMap.entrySet()) {
        String indexName = indexShards.getKey();
        IndexMetadata indexMetadata = metadata.index(indexName);
        if (indexMetadata == null) {
            continue;
        }
        Index index = indexMetadata.getIndex();
        IntIndexedContainer shards = indexShards.getValue();
        IndexService indexService = indicesService.indexService(index);
        if (indexService == null) {
            for (IntCursor shard : shards) {
                unassignedShards.add(toUnassignedShard(index.getName(), UnassignedShard.markAssigned(shard.value)));
            }
            continue;
        }
        for (IntCursor shard : shards) {
            if (UnassignedShard.isUnassigned(shard.value)) {
                unassignedShards.add(toUnassignedShard(index.getName(), UnassignedShard.markAssigned(shard.value)));
                continue;
            }
            ShardId shardId = new ShardId(index, shard.value);
            try {
                ShardCollectorProvider shardCollectorProvider = getCollectorProviderSafe(shardId);
                shardRowContexts.add(shardCollectorProvider.shardRowContext());
            } catch (ShardNotFoundException | IllegalIndexShardStateException e) {
                unassignedShards.add(toUnassignedShard(index.getName(), shard.value));
            }
        }
    }
    Iterable<Row> assignedShardRows = RowsTransformer.toRowsIterable(txnCtx, inputFactory, shardReferenceResolver, collectPhase, shardRowContexts, false);
    Iterable<Row> rows;
    if (unassignedShards.size() > 0) {
        Iterable<Row> unassignedShardRows = RowsTransformer.toRowsIterable(txnCtx, inputFactory, unassignedShardReferenceResolver, collectPhase, unassignedShards, false);
        rows = Iterables.concat(assignedShardRows, unassignedShardRows);
    } else {
        rows = assignedShardRows;
    }
    if (collectPhase.orderBy() != null) {
        return RowsTransformer.sortRows(Iterables.transform(rows, Row::materialize), collectPhase);
    }
    return rows;
}

Example 14 with ShardNotFoundException

use of org.elasticsearch.index.shard.ShardNotFoundException in project crate by crate.

the class PKLookupOperation method lookup.

public BatchIterator<Row> lookup(UUID jobId, TransactionContext txnCtx, Supplier<RamAccounting> ramAccountingSupplier, Supplier<MemoryManager> memoryManagerSupplier, boolean ignoreMissing, Map<ShardId, List<PKAndVersion>> idsByShard, Collection<? extends Projection> projections, boolean requiresScroll, Function<Doc, Row> resultToRow) {
    ArrayList<BatchIterator<Row>> iterators = new ArrayList<>(idsByShard.size());
    for (Map.Entry<ShardId, List<PKAndVersion>> idsByShardEntry : idsByShard.entrySet()) {
        ShardId shardId = idsByShardEntry.getKey();
        IndexService indexService = indicesService.indexService(shardId.getIndex());
        if (indexService == null) {
            if (ignoreMissing) {
                continue;
            }
            throw new IndexNotFoundException(shardId.getIndex());
        }
        IndexShard shard = indexService.getShardOrNull(shardId.id());
        if (shard == null) {
            if (ignoreMissing) {
                continue;
            }
            throw new ShardNotFoundException(shardId);
        }
        Stream<Row> rowStream = idsByShardEntry.getValue().stream().map(pkAndVersion -> lookupDoc(shard, pkAndVersion.id(), pkAndVersion.version(), VersionType.EXTERNAL, pkAndVersion.seqNo(), pkAndVersion.primaryTerm())).filter(Objects::nonNull).map(resultToRow);
        if (projections.isEmpty()) {
            final Iterable<Row> rowIterable = requiresScroll ? rowStream.map(row -> new RowN(row.materialize())).collect(Collectors.toList()) : rowStream::iterator;
            iterators.add(InMemoryBatchIterator.of(rowIterable, SentinelRow.SENTINEL, true));
        } else {
            ProjectorFactory projectorFactory;
            try {
                projectorFactory = shardCollectSource.getProjectorFactory(shardId);
            } catch (ShardNotFoundException e) {
                if (ignoreMissing) {
                    continue;
                }
                throw e;
            }
            Projectors projectors = new Projectors(projections, jobId, txnCtx, ramAccountingSupplier.get(), memoryManagerSupplier.get(), projectorFactory);
            final Iterable<Row> rowIterable = requiresScroll && !projectors.providesIndependentScroll() ? rowStream.map(row -> new RowN(row.materialize())).collect(Collectors.toList()) : rowStream::iterator;
            iterators.add(projectors.wrap(InMemoryBatchIterator.of(rowIterable, SentinelRow.SENTINEL, true)));
        }
    }
    // noinspection unchecked
    return CompositeBatchIterator.seqComposite(iterators.toArray(new BatchIterator[0]));
}

Example 15 with ShardNotFoundException

use of org.elasticsearch.index.shard.ShardNotFoundException in project crate by crate.

the class AllocateReplicaAllocationCommand method execute.

@Override
public RerouteExplanation execute(RoutingAllocation allocation, boolean explain) {
    final DiscoveryNode discoNode;
    try {
        discoNode = allocation.nodes().resolveNode(node);
    } catch (IllegalArgumentException e) {
        return explainOrThrowRejectedCommand(explain, allocation, e);
    }
    final RoutingNodes routingNodes = allocation.routingNodes();
    RoutingNode routingNode = routingNodes.node(discoNode.getId());
    if (routingNode == null) {
        return explainOrThrowMissingRoutingNode(allocation, explain, discoNode);
    }
    try {
        allocation.routingTable().shardRoutingTable(index, shardId).primaryShard();
    } catch (IndexNotFoundException | ShardNotFoundException e) {
        return explainOrThrowRejectedCommand(explain, allocation, e);
    }
    ShardRouting primaryShardRouting = null;
    for (RoutingNode node : allocation.routingNodes()) {
        for (ShardRouting shard : node) {
            if (shard.getIndexName().equals(index) && shard.getId() == shardId && shard.primary()) {
                primaryShardRouting = shard;
                break;
            }
        }
    }
    if (primaryShardRouting == null) {
        return explainOrThrowRejectedCommand(explain, allocation, "trying to allocate a replica shard [" + index + "][" + shardId + "], while corresponding primary shard is still unassigned");
    }
    List<ShardRouting> replicaShardRoutings = new ArrayList<>();
    for (ShardRouting shard : allocation.routingNodes().unassigned()) {
        if (shard.getIndexName().equals(index) && shard.getId() == shardId && shard.primary() == false) {
            replicaShardRoutings.add(shard);
        }
    }
    ShardRouting shardRouting;
    if (replicaShardRoutings.isEmpty()) {
        return explainOrThrowRejectedCommand(explain, allocation, "all copies of [" + index + "][" + shardId + "] are already assigned. Use the move allocation command instead");
    } else {
        shardRouting = replicaShardRoutings.get(0);
    }
    Decision decision = allocation.deciders().canAllocate(shardRouting, routingNode, allocation);
    if (decision.type() == Decision.Type.NO) {
        // don't use explainOrThrowRejectedCommand to keep the original "NO" decision
        if (explain) {
            return new RerouteExplanation(this, decision);
        }
        throw new IllegalArgumentException("[" + name() + "] allocation of [" + index + "][" + shardId + "] on node " + discoNode + " is not allowed, reason: " + decision);
    }
    initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting);
    return new RerouteExplanation(this, decision);
}