Examples with TotalHitCountCollector org.apache.lucene.search.TotalHitCountCollector used on opensource projects

Example 26 with TotalHitCountCollector

use of org.apache.lucene.search.TotalHitCountCollector in project neo4j by neo4j.

the class SimpleValueIndexReader method countIndexedEntities.

@Override
public long countIndexedEntities(long entityId, CursorContext cursorContext, int[] propertyKeyIds, Value... propertyValues) {
    Query entityIdQuery = new TermQuery(LuceneDocumentStructure.newTermForChangeOrRemove(entityId));
    Query valueQuery = LuceneDocumentStructure.newSeekQuery(propertyValues);
    BooleanQuery.Builder entityIdAndValueQuery = new BooleanQuery.Builder();
    entityIdAndValueQuery.add(entityIdQuery, BooleanClause.Occur.MUST);
    entityIdAndValueQuery.add(valueQuery, BooleanClause.Occur.MUST);
    try {
        TotalHitCountCollector collector = new TotalHitCountCollector();
        getIndexSearcher().search(entityIdAndValueQuery.build(), collector);
        // A <label,propertyKeyId,nodeId> tuple should only match at most a single propertyValue
        return collector.getTotalHits();
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 27 with TotalHitCountCollector

use of org.apache.lucene.search.TotalHitCountCollector in project crate by crate.

the class EngineTestCase method assertVisibleCount.

protected static void assertVisibleCount(InternalEngine engine, int numDocs, boolean refresh) throws IOException {
    if (refresh) {
        engine.refresh("test");
    }
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        final TotalHitCountCollector collector = new TotalHitCountCollector();
        searcher.search(new MatchAllDocsQuery(), collector);
        assertThat(collector.getTotalHits(), equalTo(numDocs));
    }
}

Example 28 with TotalHitCountCollector

use of org.apache.lucene.search.TotalHitCountCollector in project crate by crate.

the class EngineTestCase method assertOpsOnReplica.

public static void assertOpsOnReplica(final List<Engine.Operation> ops, final InternalEngine replicaEngine, boolean shuffleOps, final Logger logger) throws IOException {
    final Engine.Operation lastOp = ops.get(ops.size() - 1);
    final String lastFieldValue;
    if (lastOp instanceof Engine.Index) {
        Engine.Index index = (Engine.Index) lastOp;
        lastFieldValue = index.docs().get(0).get("value");
    } else {
        // delete
        lastFieldValue = null;
    }
    if (shuffleOps) {
        int firstOpWithSeqNo = 0;
        while (firstOpWithSeqNo < ops.size() && ops.get(firstOpWithSeqNo).seqNo() < 0) {
            firstOpWithSeqNo++;
        }
        // shuffle ops but make sure legacy ops are first
        shuffle(ops.subList(0, firstOpWithSeqNo), random());
        shuffle(ops.subList(firstOpWithSeqNo, ops.size()), random());
    }
    boolean firstOp = true;
    for (Engine.Operation op : ops) {
        logger.info("performing [{}], v [{}], seq# [{}], term [{}]", op.operationType().name().charAt(0), op.version(), op.seqNo(), op.primaryTerm());
        if (op instanceof Engine.Index) {
            Engine.IndexResult result = replicaEngine.index((Engine.Index) op);
            // replicas don't really care to about creation status of documents
            // this allows to ignore the case where a document was found in the live version maps in
            // a delete state and return false for the created flag in favor of code simplicity
            // as deleted or not. This check is just signal regression so a decision can be made if it's
            // intentional
            assertThat(result.isCreated(), equalTo(firstOp));
            assertThat(result.getVersion(), equalTo(op.version()));
            assertThat(result.getResultType(), equalTo(Engine.Result.Type.SUCCESS));
        } else {
            Engine.DeleteResult result = replicaEngine.delete((Engine.Delete) op);
            // Replicas don't really care to about found status of documents
            // this allows to ignore the case where a document was found in the live version maps in
            // a delete state and return true for the found flag in favor of code simplicity
            // his check is just signal regression so a decision can be made if it's
            // intentional
            assertThat(result.isFound(), equalTo(firstOp == false));
            assertThat(result.getVersion(), equalTo(op.version()));
            assertThat(result.getResultType(), equalTo(Engine.Result.Type.SUCCESS));
        }
        if (randomBoolean()) {
            replicaEngine.refresh("test");
        }
        if (randomBoolean()) {
            replicaEngine.flush();
            replicaEngine.refresh("test");
        }
        firstOp = false;
    }
    assertVisibleCount(replicaEngine, lastFieldValue == null ? 0 : 1);
    if (lastFieldValue != null) {
        try (Engine.Searcher searcher = replicaEngine.acquireSearcher("test")) {
            final TotalHitCountCollector collector = new TotalHitCountCollector();
            searcher.search(new TermQuery(new Term("value", lastFieldValue)), collector);
            assertThat(collector.getTotalHits(), equalTo(1));
        }
    }
}

Example 29 with TotalHitCountCollector

use of org.apache.lucene.search.TotalHitCountCollector in project crate by crate.

the class InternalEngineTests method assertOpsOnPrimary.

private int assertOpsOnPrimary(List<Engine.Operation> ops, long currentOpVersion, boolean docDeleted, InternalEngine engine) throws IOException {
    String lastFieldValue = null;
    int opsPerformed = 0;
    long lastOpVersion = currentOpVersion;
    long lastOpSeqNo = UNASSIGNED_SEQ_NO;
    long lastOpTerm = UNASSIGNED_PRIMARY_TERM;
    PrimaryTermSupplier currentTerm = (PrimaryTermSupplier) engine.engineConfig.getPrimaryTermSupplier();
    BiFunction<Long, Engine.Index, Engine.Index> indexWithVersion = (version, index) -> new Engine.Index(index.uid(), index.parsedDoc(), UNASSIGNED_SEQ_NO, currentTerm.get(), version, index.versionType(), index.origin(), index.startTime(), index.getAutoGeneratedIdTimestamp(), index.isRetry(), UNASSIGNED_SEQ_NO, 0);
    BiFunction<Long, Engine.Delete, Engine.Delete> delWithVersion = (version, delete) -> new Engine.Delete(delete.id(), delete.uid(), UNASSIGNED_SEQ_NO, currentTerm.get(), version, delete.versionType(), delete.origin(), delete.startTime(), UNASSIGNED_SEQ_NO, 0);
    TriFunction<Long, Long, Engine.Index, Engine.Index> indexWithSeq = (seqNo, term, index) -> new Engine.Index(index.uid(), index.parsedDoc(), UNASSIGNED_SEQ_NO, currentTerm.get(), index.version(), index.versionType(), index.origin(), index.startTime(), index.getAutoGeneratedIdTimestamp(), index.isRetry(), seqNo, term);
    TriFunction<Long, Long, Engine.Delete, Engine.Delete> delWithSeq = (seqNo, term, delete) -> new Engine.Delete(delete.id(), delete.uid(), UNASSIGNED_SEQ_NO, currentTerm.get(), delete.version(), delete.versionType(), delete.origin(), delete.startTime(), seqNo, term);
    Function<Engine.Index, Engine.Index> indexWithCurrentTerm = index -> new Engine.Index(index.uid(), index.parsedDoc(), UNASSIGNED_SEQ_NO, currentTerm.get(), index.version(), index.versionType(), index.origin(), index.startTime(), index.getAutoGeneratedIdTimestamp(), index.isRetry(), index.getIfSeqNo(), index.getIfPrimaryTerm());
    Function<Engine.Delete, Engine.Delete> deleteWithCurrentTerm = delete -> new Engine.Delete(delete.id(), delete.uid(), UNASSIGNED_SEQ_NO, currentTerm.get(), delete.version(), delete.versionType(), delete.origin(), delete.startTime(), delete.getIfSeqNo(), delete.getIfPrimaryTerm());
    for (Engine.Operation op : ops) {
        final boolean versionConflict = rarely();
        final boolean versionedOp = versionConflict || randomBoolean();
        final long conflictingVersion = docDeleted || randomBoolean() ? lastOpVersion + (randomBoolean() ? 1 : -1) : Versions.MATCH_DELETED;
        final long conflictingSeqNo = lastOpSeqNo == UNASSIGNED_SEQ_NO || randomBoolean() ? // use 5 to go above 0 for magic numbers
        lastOpSeqNo + 5 : lastOpSeqNo;
        final long conflictingTerm = conflictingSeqNo == lastOpSeqNo || randomBoolean() ? lastOpTerm + 1 : lastOpTerm;
        if (rarely()) {
            currentTerm.set(currentTerm.get() + 1L);
            engine.rollTranslogGeneration();
        }
        final long correctVersion = docDeleted ? Versions.MATCH_DELETED : lastOpVersion;
        logger.info("performing [{}]{}{}", op.operationType().name().charAt(0), versionConflict ? " (conflict " + conflictingVersion + ")" : "", versionedOp ? " (versioned " + correctVersion + ", seqNo " + lastOpSeqNo + ", term " + lastOpTerm + " )" : "");
        if (op instanceof Engine.Index) {
            final Engine.Index index = (Engine.Index) op;
            if (versionConflict) {
                // generate a conflict
                final Engine.IndexResult result;
                if (randomBoolean()) {
                    result = engine.index(indexWithSeq.apply(conflictingSeqNo, conflictingTerm, index));
                } else {
                    result = engine.index(indexWithVersion.apply(conflictingVersion, index));
                }
                assertThat(result.isCreated(), equalTo(false));
                assertThat(result.getVersion(), equalTo(lastOpVersion));
                assertThat(result.getResultType(), equalTo(Engine.Result.Type.FAILURE));
                assertThat(result.getFailure(), instanceOf(VersionConflictEngineException.class));
            } else {
                final Engine.IndexResult result;
                if (versionedOp) {
                    // TODO: add support for non-existing docs
                    if (randomBoolean() && lastOpSeqNo != SequenceNumbers.UNASSIGNED_SEQ_NO && docDeleted == false) {
                        result = engine.index(indexWithSeq.apply(lastOpSeqNo, lastOpTerm, index));
                    } else {
                        result = engine.index(indexWithVersion.apply(correctVersion, index));
                    }
                } else {
                    result = engine.index(indexWithCurrentTerm.apply(index));
                }
                assertThat(result.isCreated(), equalTo(docDeleted));
                assertThat(result.getVersion(), equalTo(Math.max(lastOpVersion + 1, 1)));
                assertThat(result.getResultType(), equalTo(Engine.Result.Type.SUCCESS));
                assertThat(result.getFailure(), nullValue());
                lastFieldValue = index.docs().get(0).get("value");
                docDeleted = false;
                lastOpVersion = result.getVersion();
                lastOpSeqNo = result.getSeqNo();
                lastOpTerm = result.getTerm();
                opsPerformed++;
            }
        } else {
            final Engine.Delete delete = (Engine.Delete) op;
            if (versionConflict) {
                // generate a conflict
                Engine.DeleteResult result;
                if (randomBoolean()) {
                    result = engine.delete(delWithSeq.apply(conflictingSeqNo, conflictingTerm, delete));
                } else {
                    result = engine.delete(delWithVersion.apply(conflictingVersion, delete));
                }
                assertThat(result.isFound(), equalTo(docDeleted == false));
                assertThat(result.getVersion(), equalTo(lastOpVersion));
                assertThat(result.getResultType(), equalTo(Engine.Result.Type.FAILURE));
                assertThat(result.getFailure(), instanceOf(VersionConflictEngineException.class));
            } else {
                final Engine.DeleteResult result;
                long correctSeqNo = docDeleted ? UNASSIGNED_SEQ_NO : lastOpSeqNo;
                if (versionedOp && lastOpSeqNo != UNASSIGNED_SEQ_NO && randomBoolean()) {
                    result = engine.delete(delWithSeq.apply(correctSeqNo, lastOpTerm, delete));
                } else if (versionedOp) {
                    result = engine.delete(delWithVersion.apply(correctVersion, delete));
                } else {
                    result = engine.delete(deleteWithCurrentTerm.apply(delete));
                }
                assertThat(result.isFound(), equalTo(docDeleted == false));
                assertThat(result.getVersion(), equalTo(Math.max(lastOpVersion + 1, 1)));
                assertThat(result.getResultType(), equalTo(Engine.Result.Type.SUCCESS));
                assertThat(result.getFailure(), nullValue());
                docDeleted = true;
                lastOpVersion = result.getVersion();
                lastOpSeqNo = result.getSeqNo();
                lastOpTerm = result.getTerm();
                opsPerformed++;
            }
        }
        if (randomBoolean()) {
            // refresh and take the chance to check everything is ok so far
            assertVisibleCount(engine, docDeleted ? 0 : 1);
            // first op and it failed.
            if (docDeleted == false && lastFieldValue != null) {
                try (Searcher searcher = engine.acquireSearcher("test")) {
                    final TotalHitCountCollector collector = new TotalHitCountCollector();
                    searcher.search(new TermQuery(new Term("value", lastFieldValue)), collector);
                    assertThat(collector.getTotalHits(), equalTo(1));
                }
            }
        }
        if (randomBoolean()) {
            engine.flush();
            engine.refresh("test");
        }
        if (rarely()) {
            // simulate GC deletes
            engine.refresh("gc_simulation", Engine.SearcherScope.INTERNAL, true);
            engine.clearDeletedTombstones();
            if (docDeleted) {
                lastOpVersion = Versions.NOT_FOUND;
                lastOpSeqNo = UNASSIGNED_SEQ_NO;
                lastOpTerm = UNASSIGNED_PRIMARY_TERM;
            }
        }
    }
    assertVisibleCount(engine, docDeleted ? 0 : 1);
    if (docDeleted == false) {
        try (Searcher searcher = engine.acquireSearcher("test")) {
            final TotalHitCountCollector collector = new TotalHitCountCollector();
            searcher.search(new TermQuery(new Term("value", lastFieldValue)), collector);
            assertThat(collector.getTotalHits(), equalTo(1));
        }
    }
    return opsPerformed;
}

Example 30 with TotalHitCountCollector

use of org.apache.lucene.search.TotalHitCountCollector in project crate by crate.

the class InternalEngineTests method testVersioningPromotedReplica.

@Test
public void testVersioningPromotedReplica() throws IOException {
    final List<Engine.Operation> replicaOps = generateSingleDocHistory(true, VersionType.INTERNAL, false, 1, 2, 20, "1");
    List<Engine.Operation> primaryOps = generateSingleDocHistory(false, VersionType.INTERNAL, false, 2, 2, 20, "1");
    Engine.Operation lastReplicaOp = replicaOps.get(replicaOps.size() - 1);
    final boolean deletedOnReplica = lastReplicaOp instanceof Engine.Delete;
    final long finalReplicaVersion = lastReplicaOp.version();
    final long finalReplicaSeqNo = lastReplicaOp.seqNo();
    assertOpsOnReplica(replicaOps, replicaEngine, true, logger);
    final int opsOnPrimary = assertOpsOnPrimary(primaryOps, finalReplicaVersion, deletedOnReplica, replicaEngine);
    final long currentSeqNo = getSequenceID(replicaEngine, new Engine.Get(lastReplicaOp.uid().text(), lastReplicaOp.uid())).v1();
    try (Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
        final TotalHitCountCollector collector = new TotalHitCountCollector();
        searcher.search(new MatchAllDocsQuery(), collector);
        if (collector.getTotalHits() > 0) {
            // last op wasn't delete
            assertThat(currentSeqNo, equalTo(finalReplicaSeqNo + opsOnPrimary));
        }
    }
}