Examples with DelegatingCollector org.apache.solr.search.DelegatingCollector used on opensource projects

Example 1 with DelegatingCollector

use of org.apache.solr.search.DelegatingCollector in project SearchServices by Alfresco.

the class MimetypeGroupingCollector method finish.

public void finish() throws IOException {
    NamedList<Object> analytics = new NamedList<>();
    rb.rsp.add("analytics", analytics);
    NamedList<Object> fieldCounts = new NamedList<>();
    analytics.add("mimetype()", fieldCounts);
    for (String key : counters.keySet()) {
        Counter counter = counters.get(key);
        fieldCounts.add(key, counter.get());
    }
    if (this.delegate instanceof DelegatingCollector) {
        ((DelegatingCollector) this.delegate).finish();
    }
}

Example 2 with DelegatingCollector

use of org.apache.solr.search.DelegatingCollector in project SearchServices by Alfresco.

the class SolrInformationServer method getCascades.

@Override
public List<Transaction> getCascades(int num) throws IOException {
    RefCounted<SolrIndexSearcher> refCounted = null;
    try {
        refCounted = this.core.getSearcher();
        SolrIndexSearcher searcher = refCounted.get();
        Collector collector = null;
        TopFieldCollector topFieldCollector = TopFieldCollector.create(new Sort(new SortField(FIELD_TXID, SortField.Type.LONG)), num, null, false, false, false);
        collector = topFieldCollector;
        LegacyNumericRangeQuery q = LegacyNumericRangeQuery.newIntRange(FIELD_CASCADE_FLAG, 1, 1, true, true);
        DelegatingCollector delegatingCollector = new TxnCacheFilter(cleanCascadeCache);
        delegatingCollector.setLastDelegate(collector);
        collector = delegatingCollector;
        searcher.search(q, collector);
        ScoreDoc[] scoreDocs = topFieldCollector.topDocs().scoreDocs;
        Set fields = new HashSet();
        fields.add(FIELD_S_TXID);
        fields.add(FIELD_S_TXCOMMITTIME);
        List<Transaction> transactions = new ArrayList(scoreDocs.length);
        for (ScoreDoc scoreDoc : scoreDocs) {
            Transaction transaction = new Transaction();
            Document doc = searcher.doc(scoreDoc.doc, fields);
            List<IndexableField> ifields = doc.getFields();
            IndexableField txID = doc.getField(FIELD_S_TXID);
            // System.out.println("############### Cascade Document:"+doc);
            long txnID = txID.numericValue().longValue();
            cleanCascadeCache.put(txnID, null);
            transaction.setId(txnID);
            IndexableField txnCommitTime = doc.getField(FIELD_S_TXCOMMITTIME);
            transaction.setCommitTimeMs(txnCommitTime.numericValue().longValue());
            transactions.add(transaction);
        }
        return transactions;
    } finally {
        refCounted.decref();
    }
}

Example 3 with DelegatingCollector

use of org.apache.solr.search.DelegatingCollector in project SearchServices by Alfresco.

the class SolrInformationServer method getDocsWithUncleanContent.

@Override
public List<TenantAclIdDbId> getDocsWithUncleanContent(int start, int rows) throws IOException {
    RefCounted<SolrIndexSearcher> refCounted = null;
    try {
        List<TenantAclIdDbId> docIds = new ArrayList<>();
        refCounted = this.core.getSearcher();
        SolrIndexSearcher searcher = refCounted.get();
        /*
            *  Below is the code for purging the cleanContentCache.
            *  The cleanContentCache is an in-memory LRU cache of the transactions that have already
            *  had their content fetched. This is needed because the ContentTracker does not have an up-to-date
            *  snapshot of the index to determine which nodes are marked as dirty/new. The cleanContentCache is used
            *  to filter out nodes that belong to transactions that have already been processed, which stops them from
            *  being re-processed.
            *
            *  The cleanContentCache needs to be purged periodically to support retrying of failed content fetches.
            *  This is because fetches for individual nodes within the transaction may have failed, but the transaction will still be in the
            *  cleanContentCache, which prevents it from being retried.
            *
            *  Once a transaction is purged from the cleanContentCache it will be retried automatically if it is marked dirty/new
            *  in current snapshot of the index.
            *
            *  The code below runs every two minutes and purges transactions from the
            *  cleanContentCache that is more then 20 minutes old.
            *
            */
        long purgeTime = System.currentTimeMillis();
        if (purgeTime - cleanContentLastPurged > 120000) {
            Iterator<Entry> entries = cleanContentCache.entrySet().iterator();
            while (entries.hasNext()) {
                Entry<Long, Long> entry = entries.next();
                long txnTime = entry.getValue();
                if (purgeTime - txnTime > 1200000) {
                    // Purge the clean content cache of records more then 20 minutes old.
                    entries.remove();
                }
            }
            cleanContentLastPurged = purgeTime;
        }
        long txnFloor = -1;
        // This query gets lowest txnID that has dirty content.
        // System.out.println("############### finding the transaction floor ################");
        TermQuery termQuery1 = new TermQuery(new Term(FIELD_FTSSTATUS, FTSStatus.Dirty.toString()));
        TermQuery termQuery2 = new TermQuery(new Term(FIELD_FTSSTATUS, FTSStatus.New.toString()));
        BooleanClause clause1 = new BooleanClause(termQuery1, BooleanClause.Occur.SHOULD);
        BooleanClause clause2 = new BooleanClause(termQuery2, BooleanClause.Occur.SHOULD);
        BooleanQuery.Builder builder = new BooleanQuery.Builder();
        builder.add(clause1);
        builder.add(clause2);
        BooleanQuery orQuery = builder.build();
        Sort sort = new Sort(new SortField(FIELD_INTXID, SortField.Type.LONG));
        sort = sort.rewrite(searcher);
        TopFieldCollector collector = TopFieldCollector.create(sort, 1, null, false, false, false);
        // Filter transactions that have already been processed.
        DelegatingCollector delegatingCollector = new TxnCacheFilter(cleanContentCache);
        delegatingCollector.setLastDelegate(collector);
        searcher.search(orQuery, delegatingCollector);
        if (collector.getTotalHits() == 0) {
            return docIds;
        }
        ScoreDoc[] scoreDocs = collector.topDocs().scoreDocs;
        List<LeafReaderContext> leaves = searcher.getTopReaderContext().leaves();
        int index = ReaderUtil.subIndex(scoreDocs[0].doc, leaves);
        LeafReaderContext context = leaves.get(index);
        NumericDocValues longs = context.reader().getNumericDocValues(FIELD_INTXID);
        txnFloor = longs.get(scoreDocs[0].doc - context.docBase);
        // System.out.println("################ Transaction floor:"+txnFloor);
        // Find the next N transactions
        collector = TopFieldCollector.create(new Sort(new SortField(FIELD_INTXID, SortField.Type.LONG)), rows, null, false, false, false);
        delegatingCollector = new TxnFloorFilter(txnFloor, cleanContentCache);
        delegatingCollector.setLastDelegate(collector);
        TermQuery txnQuery = new TermQuery(new Term(FIELD_DOC_TYPE, DOC_TYPE_TX));
        searcher.search(txnQuery, delegatingCollector);
        TopDocs docs = collector.topDocs();
        if (collector.getTotalHits() == 0) {
            // No new transactions to consider
            return docIds;
        }
        leaves = searcher.getTopReaderContext().leaves();
        FieldType fieldType = searcher.getSchema().getField(FIELD_INTXID).getType();
        builder = new BooleanQuery.Builder();
        for (ScoreDoc scoreDoc : docs.scoreDocs) {
            index = ReaderUtil.subIndex(scoreDoc.doc, leaves);
            context = leaves.get(index);
            longs = context.reader().getNumericDocValues(FIELD_INTXID);
            long txnID = longs.get(scoreDoc.doc - context.docBase);
            // Build up the query for the filter of transactions we need to pull the dirty content for.
            TermQuery txnIDQuery = new TermQuery(new Term(FIELD_INTXID, fieldType.readableToIndexed(Long.toString(txnID))));
            builder.add(new BooleanClause(txnIDQuery, BooleanClause.Occur.SHOULD));
        }
        BooleanQuery txnFilterQuery = builder.build();
        // Get the docs with dirty content for the transactions gathered above.
        TermQuery statusQuery1 = new TermQuery(new Term(FIELD_FTSSTATUS, FTSStatus.Dirty.toString()));
        TermQuery statusQuery2 = new TermQuery(new Term(FIELD_FTSSTATUS, FTSStatus.New.toString()));
        BooleanClause statusClause1 = new BooleanClause(statusQuery1, BooleanClause.Occur.SHOULD);
        BooleanClause statusClause2 = new BooleanClause(statusQuery2, BooleanClause.Occur.SHOULD);
        BooleanQuery.Builder builder1 = new BooleanQuery.Builder();
        builder1.add(statusClause1);
        builder1.add(statusClause2);
        BooleanQuery statusQuery = builder1.build();
        DocListCollector docListCollector = new DocListCollector();
        BooleanQuery.Builder builder2 = new BooleanQuery.Builder();
        builder2.add(statusQuery, BooleanClause.Occur.MUST);
        builder2.add(new QueryWrapperFilter(txnFilterQuery), BooleanClause.Occur.MUST);
        searcher.search(builder2.build(), docListCollector);
        IntArrayList docList = docListCollector.getDocs();
        int size = docList.size();
        // System.out.println("############### Dirty Doc Count ################:" + size);
        Set<String> fields = new HashSet<String>();
        fields.add(FIELD_SOLR4_ID);
        List<Long> processedTxns = new ArrayList<Long>();
        for (int i = 0; i < size; ++i) {
            int doc = docList.get(i);
            Document document = searcher.doc(doc, fields);
            index = ReaderUtil.subIndex(doc, leaves);
            context = leaves.get(index);
            longs = context.reader().getNumericDocValues(FIELD_INTXID);
            long txnId = longs.get(doc - context.docBase);
            if (!cleanContentCache.containsKey(txnId)) {
                processedTxns.add(txnId);
                IndexableField id = document.getField(FIELD_SOLR4_ID);
                String idString = id.stringValue();
                TenantAclIdDbId tenantAndDbId = AlfrescoSolrDataModel.decodeNodeDocumentId(idString);
                docIds.add(tenantAndDbId);
            }
        }
        long txnTime = System.currentTimeMillis();
        for (Long l : processedTxns) {
            // Save the indexVersion so we know when we can clean out this entry
            cleanContentCache.put(l, txnTime);
        }
        return docIds;
    } finally {
        refCounted.decref();
    }
}

Example 4 with DelegatingCollector

use of org.apache.solr.search.DelegatingCollector in project SearchServices by Alfresco.

the class PostFilterQuery method getFilterCollector.

public DelegatingCollector getFilterCollector(IndexSearcher searcher) {
    List<PostFilter> postFilters = new ArrayList<PostFilter>();
    getPostFilters(query, postFilters);
    Collections.sort(postFilters, new PostFilterComp());
    List<DelegatingCollector> delegatingCollectors = new ArrayList<DelegatingCollector>();
    for (PostFilter postFilter : postFilters) {
        DelegatingCollector delegatingCollector = postFilter.getFilterCollector(searcher);
        if (!(delegatingCollector instanceof AllAccessCollector)) {
            delegatingCollectors.add(delegatingCollector);
        }
    }
    if (delegatingCollectors.size() == 0) {
        return new AllAccessCollector();
    } else if (delegatingCollectors.size() == 1) {
        return delegatingCollectors.get(0);
    } else {
        return new WrapperCollector(delegatingCollectors);
    }
}

Example 5 with DelegatingCollector

use of org.apache.solr.search.DelegatingCollector in project SearchServices by Alfresco.

the class ContentSizeGroupingCollector method finish.

public void finish() throws IOException {
    NamedList<Object> analytics = new NamedList<>();
    rb.rsp.add("analytics", analytics);
    NamedList<Object> fieldCounts = new NamedList<>();
    analytics.add("contentSize()", fieldCounts);
    for (Bucket bucket : stats.getHistogram()) {
        fieldCounts.add("[" + (long) Math.ceil(bucket.leftBoundary) + " TO " + (long) Math.ceil(bucket.rightBoundary) + ">", (long) roundEven(bucket.countLeft + bucket.countRight));
    }
    if (this.delegate instanceof DelegatingCollector) {
        ((DelegatingCollector) this.delegate).finish();
    }
}