Examples with MatchAllDocsQuery org.apache.lucene.search.MatchAllDocsQuery used on opensource projects

Example 91 with MatchAllDocsQuery

use of org.apache.lucene.search.MatchAllDocsQuery in project lucene-solr by apache.

the class TestSimpleQueryParser method testNOT.

/** test negated term */
public void testNOT() throws Exception {
    BooleanQuery.Builder expected = new BooleanQuery.Builder();
    expected.add(new TermQuery(new Term("field", "foo")), Occur.MUST_NOT);
    expected.add(new MatchAllDocsQuery(), Occur.SHOULD);
    assertEquals(expected.build(), parse("-foo"));
    assertEquals(expected.build(), parse("-(foo)"));
    assertEquals(expected.build(), parse("---foo"));
}

Example 92 with MatchAllDocsQuery

use of org.apache.lucene.search.MatchAllDocsQuery in project lucene-solr by apache.

the class TestNearest method testNearestNeighborRandom.

public void testNearestNeighborRandom() throws Exception {
    int numPoints = atLeast(5000);
    Directory dir;
    if (numPoints > 100000) {
        dir = newFSDirectory(createTempDir(getClass().getSimpleName()));
    } else {
        dir = newDirectory();
    }
    double[] lats = new double[numPoints];
    double[] lons = new double[numPoints];
    IndexWriterConfig iwc = getIndexWriterConfig();
    iwc.setMergePolicy(newLogMergePolicy());
    iwc.setMergeScheduler(new SerialMergeScheduler());
    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
    for (int id = 0; id < numPoints; id++) {
        lats[id] = quantizeLat(GeoTestUtil.nextLatitude());
        lons[id] = quantizeLon(GeoTestUtil.nextLongitude());
        Document doc = new Document();
        doc.add(new LatLonPoint("point", lats[id], lons[id]));
        doc.add(new LatLonDocValuesField("point", lats[id], lons[id]));
        doc.add(new StoredField("id", id));
        w.addDocument(doc);
    }
    if (random().nextBoolean()) {
        w.forceMerge(1);
    }
    DirectoryReader r = w.getReader();
    if (VERBOSE) {
        System.out.println("TEST: reader=" + r);
    }
    // can't wrap because we require Lucene60PointsFormat directly but e.g. ParallelReader wraps with its own points impl:
    IndexSearcher s = newSearcher(r, false);
    int iters = atLeast(100);
    for (int iter = 0; iter < iters; iter++) {
        if (VERBOSE) {
            System.out.println("\nTEST: iter=" + iter);
        }
        double pointLat = GeoTestUtil.nextLatitude();
        double pointLon = GeoTestUtil.nextLongitude();
        // dumb brute force search to get the expected result:
        NearestHit[] expectedHits = new NearestHit[lats.length];
        for (int id = 0; id < lats.length; id++) {
            NearestHit hit = new NearestHit();
            hit.distanceMeters = SloppyMath.haversinMeters(pointLat, pointLon, lats[id], lons[id]);
            hit.docID = id;
            expectedHits[id] = hit;
        }
        Arrays.sort(expectedHits, new Comparator<NearestHit>() {

            @Override
            public int compare(NearestHit a, NearestHit b) {
                int cmp = Double.compare(a.distanceMeters, b.distanceMeters);
                if (cmp != 0) {
                    return cmp;
                }
                // tie break by smaller docID:
                return a.docID - b.docID;
            }
        });
        int topN = TestUtil.nextInt(random(), 1, lats.length);
        if (VERBOSE) {
            System.out.println("\nhits for pointLat=" + pointLat + " pointLon=" + pointLon);
        }
        // Also test with MatchAllDocsQuery, sorting by distance:
        TopFieldDocs fieldDocs = s.search(new MatchAllDocsQuery(), topN, new Sort(LatLonDocValuesField.newDistanceSort("point", pointLat, pointLon)));
        ScoreDoc[] hits = LatLonPoint.nearest(s, "point", pointLat, pointLon, topN).scoreDocs;
        for (int i = 0; i < topN; i++) {
            NearestHit expected = expectedHits[i];
            FieldDoc expected2 = (FieldDoc) fieldDocs.scoreDocs[i];
            FieldDoc actual = (FieldDoc) hits[i];
            Document actualDoc = r.document(actual.doc);
            if (VERBOSE) {
                System.out.println("hit " + i);
                System.out.println("  expected id=" + expected.docID + " lat=" + lats[expected.docID] + " lon=" + lons[expected.docID] + " distance=" + expected.distanceMeters + " meters");
                System.out.println("  actual id=" + actualDoc.getField("id") + " distance=" + actual.fields[0] + " meters");
            }
            assertEquals(expected.docID, actual.doc);
            assertEquals(expected.distanceMeters, ((Double) actual.fields[0]).doubleValue(), 0.0);
            assertEquals(expected.docID, expected.docID);
            assertEquals(((Double) expected2.fields[0]).doubleValue(), expected.distanceMeters, 0.0);
        }
    }
    r.close();
    w.close();
    dir.close();
}

Example 93 with MatchAllDocsQuery

use of org.apache.lucene.search.MatchAllDocsQuery in project lucene-solr by apache.

the class TestLatLonPointDistanceSort method doRandomTest.

private void doRandomTest(int numDocs, int numQueries) throws IOException {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig();
    // else seeds may not to reproduce:
    iwc.setMergeScheduler(new SerialMergeScheduler());
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
    for (int i = 0; i < numDocs; i++) {
        Document doc = new Document();
        doc.add(new StoredField("id", i));
        doc.add(new NumericDocValuesField("id", i));
        if (random().nextInt(10) > 7) {
            double latRaw = GeoTestUtil.nextLatitude();
            double lonRaw = GeoTestUtil.nextLongitude();
            // pre-normalize up front, so we can just use quantized value for testing and do simple exact comparisons
            double lat = decodeLatitude(encodeLatitude(latRaw));
            double lon = decodeLongitude(encodeLongitude(lonRaw));
            doc.add(new LatLonDocValuesField("field", lat, lon));
            doc.add(new StoredField("lat", lat));
            doc.add(new StoredField("lon", lon));
        }
        // otherwise "missing"
        writer.addDocument(doc);
    }
    IndexReader reader = writer.getReader();
    IndexSearcher searcher = newSearcher(reader);
    for (int i = 0; i < numQueries; i++) {
        double lat = GeoTestUtil.nextLatitude();
        double lon = GeoTestUtil.nextLongitude();
        double missingValue = Double.POSITIVE_INFINITY;
        Result[] expected = new Result[reader.maxDoc()];
        for (int doc = 0; doc < reader.maxDoc(); doc++) {
            Document targetDoc = reader.document(doc);
            final double distance;
            if (targetDoc.getField("lat") == null) {
                // missing
                distance = missingValue;
            } else {
                double docLatitude = targetDoc.getField("lat").numericValue().doubleValue();
                double docLongitude = targetDoc.getField("lon").numericValue().doubleValue();
                distance = SloppyMath.haversinMeters(lat, lon, docLatitude, docLongitude);
            }
            int id = targetDoc.getField("id").numericValue().intValue();
            expected[doc] = new Result(id, distance);
        }
        Arrays.sort(expected);
        // randomize the topN a bit
        int topN = TestUtil.nextInt(random(), 1, reader.maxDoc());
        // sort by distance, then ID
        SortField distanceSort = LatLonDocValuesField.newDistanceSort("field", lat, lon);
        distanceSort.setMissingValue(missingValue);
        Sort sort = new Sort(distanceSort, new SortField("id", SortField.Type.INT));
        TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), topN, sort);
        for (int resultNumber = 0; resultNumber < topN; resultNumber++) {
            FieldDoc fieldDoc = (FieldDoc) topDocs.scoreDocs[resultNumber];
            Result actual = new Result((Integer) fieldDoc.fields[1], (Double) fieldDoc.fields[0]);
            assertEquals(expected[resultNumber], actual);
        }
        // get page2 with searchAfter()
        if (topN < reader.maxDoc()) {
            int page2 = TestUtil.nextInt(random(), 1, reader.maxDoc() - topN);
            TopDocs topDocs2 = searcher.searchAfter(topDocs.scoreDocs[topN - 1], new MatchAllDocsQuery(), page2, sort);
            for (int resultNumber = 0; resultNumber < page2; resultNumber++) {
                FieldDoc fieldDoc = (FieldDoc) topDocs2.scoreDocs[resultNumber];
                Result actual = new Result((Integer) fieldDoc.fields[1], (Double) fieldDoc.fields[0]);
                assertEquals(expected[topN + resultNumber], actual);
            }
        }
    }
    reader.close();
    writer.close();
    dir.close();
}

Example 94 with MatchAllDocsQuery

use of org.apache.lucene.search.MatchAllDocsQuery in project lucene-solr by apache.

the class RecoveryStrategy method cloudDebugLog.

private final void cloudDebugLog(SolrCore core, String op) {
    if (!LOG.isDebugEnabled()) {
        return;
    }
    try {
        RefCounted<SolrIndexSearcher> searchHolder = core.getNewestSearcher(false);
        SolrIndexSearcher searcher = searchHolder.get();
        try {
            final int totalHits = searcher.search(new MatchAllDocsQuery(), 1).totalHits;
            final String nodeName = core.getCoreContainer().getZkController().getNodeName();
            LOG.debug("[{}] {} [{} total hits]", nodeName, op, totalHits);
        } finally {
            searchHolder.decref();
        }
    } catch (Exception e) {
        LOG.debug("Error in solrcloud_debug block", e);
    }
}

Example 95 with MatchAllDocsQuery

use of org.apache.lucene.search.MatchAllDocsQuery in project lucene-solr by apache.

the class PrepRecoveryOp method execute.

@Override
public void execute(CallInfo it) throws Exception {
    assert TestInjection.injectPrepRecoveryOpPauseForever();
    final SolrParams params = it.req.getParams();
    String cname = params.get(CoreAdminParams.CORE);
    if (cname == null) {
        cname = "";
    }
    String nodeName = params.get("nodeName");
    String coreNodeName = params.get("coreNodeName");
    Replica.State waitForState = Replica.State.getState(params.get(ZkStateReader.STATE_PROP));
    Boolean checkLive = params.getBool("checkLive");
    Boolean onlyIfLeader = params.getBool("onlyIfLeader");
    Boolean onlyIfLeaderActive = params.getBool("onlyIfLeaderActive");
    CoreContainer coreContainer = it.handler.coreContainer;
    // wait long enough for the leader conflict to work itself out plus a little extra
    int conflictWaitMs = coreContainer.getZkController().getLeaderConflictResolveWait();
    int maxTries = (int) Math.round(conflictWaitMs / 1000) + 3;
    log.info("Going to wait for coreNodeName: {}, state: {}, checkLive: {}, onlyIfLeader: {}, onlyIfLeaderActive: {}, maxTime: {} s", coreNodeName, waitForState, checkLive, onlyIfLeader, onlyIfLeaderActive, maxTries);
    Replica.State state = null;
    boolean live = false;
    int retry = 0;
    while (true) {
        try (SolrCore core = coreContainer.getCore(cname)) {
            if (core == null && retry == Math.min(30, maxTries)) {
                throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "core not found:" + cname);
            }
            if (core != null) {
                if (onlyIfLeader != null && onlyIfLeader) {
                    if (!core.getCoreDescriptor().getCloudDescriptor().isLeader()) {
                        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "We are not the leader");
                    }
                }
                // wait until we are sure the recovering node is ready
                // to accept updates
                CloudDescriptor cloudDescriptor = core.getCoreDescriptor().getCloudDescriptor();
                String collectionName = cloudDescriptor.getCollectionName();
                if (retry % 15 == 0) {
                    if (retry > 0 && log.isInfoEnabled())
                        log.info("After " + retry + " seconds, core " + cname + " (" + cloudDescriptor.getShardId() + " of " + cloudDescriptor.getCollectionName() + ") still does not have state: " + waitForState + "; forcing ClusterState update from ZooKeeper");
                    // force a cluster state update
                    coreContainer.getZkController().getZkStateReader().forceUpdateCollection(collectionName);
                }
                ClusterState clusterState = coreContainer.getZkController().getClusterState();
                DocCollection collection = clusterState.getCollection(collectionName);
                Slice slice = collection.getSlice(cloudDescriptor.getShardId());
                if (slice != null) {
                    final Replica replica = slice.getReplicasMap().get(coreNodeName);
                    if (replica != null) {
                        state = replica.getState();
                        live = clusterState.liveNodesContain(nodeName);
                        final Replica.State localState = cloudDescriptor.getLastPublished();
                        // TODO: This is funky but I've seen this in testing where the replica asks the
                        // leader to be in recovery? Need to track down how that happens ... in the meantime,
                        // this is a safeguard
                        boolean leaderDoesNotNeedRecovery = (onlyIfLeader != null && onlyIfLeader && core.getName().equals(replica.getStr("core")) && waitForState == Replica.State.RECOVERING && localState == Replica.State.ACTIVE && state == Replica.State.ACTIVE);
                        if (leaderDoesNotNeedRecovery) {
                            log.warn("Leader " + core.getName() + " ignoring request to be in the recovering state because it is live and active.");
                        }
                        boolean onlyIfActiveCheckResult = onlyIfLeaderActive != null && onlyIfLeaderActive && localState != Replica.State.ACTIVE;
                        log.info("In WaitForState(" + waitForState + "): collection=" + collectionName + ", shard=" + slice.getName() + ", thisCore=" + core.getName() + ", leaderDoesNotNeedRecovery=" + leaderDoesNotNeedRecovery + ", isLeader? " + core.getCoreDescriptor().getCloudDescriptor().isLeader() + ", live=" + live + ", checkLive=" + checkLive + ", currentState=" + state.toString() + ", localState=" + localState + ", nodeName=" + nodeName + ", coreNodeName=" + coreNodeName + ", onlyIfActiveCheckResult=" + onlyIfActiveCheckResult + ", nodeProps: " + replica);
                        if (!onlyIfActiveCheckResult && replica != null && (state == waitForState || leaderDoesNotNeedRecovery)) {
                            if (checkLive == null) {
                                break;
                            } else if (checkLive && live) {
                                break;
                            } else if (!checkLive && !live) {
                                break;
                            }
                        }
                    }
                }
            }
            if (retry++ == maxTries) {
                String collection = null;
                String leaderInfo = null;
                String shardId = null;
                try {
                    CloudDescriptor cloudDescriptor = core.getCoreDescriptor().getCloudDescriptor();
                    collection = cloudDescriptor.getCollectionName();
                    shardId = cloudDescriptor.getShardId();
                    leaderInfo = coreContainer.getZkController().getZkStateReader().getLeaderUrl(collection, shardId, 5000);
                } catch (Exception exc) {
                    leaderInfo = "Not available due to: " + exc;
                }
                throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "I was asked to wait on state " + waitForState + " for " + shardId + " in " + collection + " on " + nodeName + " but I still do not see the requested state. I see state: " + Objects.toString(state) + " live:" + live + " leader from ZK: " + leaderInfo);
            }
            if (coreContainer.isShutDown()) {
                throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Solr is shutting down");
            }
            // solrcloud_debug
            if (log.isDebugEnabled() && core != null) {
                try {
                    LocalSolrQueryRequest r = new LocalSolrQueryRequest(core, new ModifiableSolrParams());
                    CommitUpdateCommand commitCmd = new CommitUpdateCommand(r, false);
                    commitCmd.softCommit = true;
                    core.getUpdateHandler().commit(commitCmd);
                    RefCounted<SolrIndexSearcher> searchHolder = core.getNewestSearcher(false);
                    SolrIndexSearcher searcher = searchHolder.get();
                    try {
                        log.debug(core.getCoreContainer().getZkController().getNodeName() + " to replicate " + searcher.search(new MatchAllDocsQuery(), 1).totalHits + " gen:" + core.getDeletionPolicy().getLatestCommit().getGeneration() + " data:" + core.getDataDir());
                    } finally {
                        searchHolder.decref();
                    }
                } catch (Exception e) {
                    log.debug("Error in solrcloud_debug block", e);
                }
            }
        }
        Thread.sleep(1000);
    }
    log.info("Waited coreNodeName: " + coreNodeName + ", state: " + waitForState + ", checkLive: " + checkLive + ", onlyIfLeader: " + onlyIfLeader + " for: " + retry + " seconds.");
}