Examples with Transaction org.alfresco.solr.client.Transaction used on opensource projects

Example 16 with Transaction

use of org.alfresco.solr.client.Transaction in project SearchServices by Alfresco.

the class SolrInformationServer method getCascades.

@Override
public List<Transaction> getCascades(int num) throws IOException {
    RefCounted<SolrIndexSearcher> refCounted = null;
    try {
        refCounted = this.core.getSearcher();
        SolrIndexSearcher searcher = refCounted.get();
        Collector collector = null;
        TopFieldCollector topFieldCollector = TopFieldCollector.create(new Sort(new SortField(FIELD_TXID, SortField.Type.LONG)), num, null, false, false, false);
        collector = topFieldCollector;
        LegacyNumericRangeQuery q = LegacyNumericRangeQuery.newIntRange(FIELD_CASCADE_FLAG, 1, 1, true, true);
        DelegatingCollector delegatingCollector = new TxnCacheFilter(cleanCascadeCache);
        delegatingCollector.setLastDelegate(collector);
        collector = delegatingCollector;
        searcher.search(q, collector);
        ScoreDoc[] scoreDocs = topFieldCollector.topDocs().scoreDocs;
        Set fields = new HashSet();
        fields.add(FIELD_S_TXID);
        fields.add(FIELD_S_TXCOMMITTIME);
        List<Transaction> transactions = new ArrayList(scoreDocs.length);
        for (ScoreDoc scoreDoc : scoreDocs) {
            Transaction transaction = new Transaction();
            Document doc = searcher.doc(scoreDoc.doc, fields);
            List<IndexableField> ifields = doc.getFields();
            IndexableField txID = doc.getField(FIELD_S_TXID);
            // System.out.println("############### Cascade Document:"+doc);
            long txnID = txID.numericValue().longValue();
            cleanCascadeCache.put(txnID, null);
            transaction.setId(txnID);
            IndexableField txnCommitTime = doc.getField(FIELD_S_TXCOMMITTIME);
            transaction.setCommitTimeMs(txnCommitTime.numericValue().longValue());
            transactions.add(transaction);
        }
        return transactions;
    } finally {
        refCounted.decref();
    }
}

Example 17 with Transaction

use of org.alfresco.solr.client.Transaction in project SearchServices by Alfresco.

the class MetadataTracker method reindexTransactions.

private void reindexTransactions() throws IOException, AuthenticationException, JSONException {
    long startElapsed = System.nanoTime();
    int docCount = 0;
    while (transactionsToReindex.peek() != null) {
        Long transactionId = transactionsToReindex.poll();
        if (transactionId != null) {
            // make sure it is cleaned out so we do not miss deletes
            this.infoSrv.deleteByTransactionId(transactionId);
            Transactions transactions = client.getTransactions(null, transactionId, null, transactionId + 1, 1);
            if ((transactions.getTransactions().size() > 0) && (transactionId.equals(transactions.getTransactions().get(0).getId()))) {
                Transaction info = transactions.getTransactions().get(0);
                this.infoSrv.dirtyTransaction(info.getId());
                GetNodesParameters gnp = new GetNodesParameters();
                ArrayList<Long> txs = new ArrayList<Long>();
                txs.add(info.getId());
                gnp.setTransactionIds(txs);
                gnp.setStoreProtocol(storeRef.getProtocol());
                gnp.setStoreIdentifier(storeRef.getIdentifier());
                List<Node> nodes = client.getNodes(gnp, (int) info.getUpdates());
                for (Node node : nodes) {
                    docCount++;
                    if (log.isDebugEnabled()) {
                        log.debug(node.toString());
                    }
                    this.infoSrv.indexNode(node, true);
                    checkShutdown();
                }
                // Index the transaction doc after the node - if this is not found then a reindex will be done.
                this.infoSrv.indexTransaction(info, true);
            }
        }
        if (docCount > batchCount) {
            if (this.infoSrv.getRegisteredSearcherCount() < getMaxLiveSearchers()) {
                checkShutdown();
                long endElapsed = System.nanoTime();
                trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed);
                startElapsed = endElapsed;
                docCount = 0;
            }
        }
    }
    if (docCount > 0) {
        checkShutdown();
        // this.infoSrv.commit();
        long endElapsed = System.nanoTime();
        trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed);
    }
}

Example 18 with Transaction

use of org.alfresco.solr.client.Transaction in project SearchServices by Alfresco.

the class MetadataTracker method checkIndex.

public IndexHealthReport checkIndex(Long toTx, Long toAclTx, Long fromTime, Long toTime) throws IOException, AuthenticationException, JSONException, EncoderException {
    // DB TX Count
    long firstTransactionCommitTime = 0;
    Transactions firstTransactions = client.getTransactions(null, 0L, null, 2000L, 1);
    if (firstTransactions.getTransactions().size() > 0) {
        Transaction firstTransaction = firstTransactions.getTransactions().get(0);
        firstTransactionCommitTime = firstTransaction.getCommitTimeMs();
    }
    IOpenBitSet txIdsInDb = infoSrv.getOpenBitSetInstance();
    Long lastTxCommitTime = Long.valueOf(firstTransactionCommitTime);
    if (fromTime != null) {
        lastTxCommitTime = fromTime;
    }
    long maxTxId = 0;
    Long minTxId = null;
    Transactions transactions;
    BoundedDeque<Transaction> txnsFound = new BoundedDeque<Transaction>(100);
    long endTime = System.currentTimeMillis() + infoSrv.getHoleRetention();
    DO: do {
        transactions = getSomeTransactions(txnsFound, lastTxCommitTime, TIME_STEP_1_HR_IN_MS, 2000, endTime);
        for (Transaction info : transactions.getTransactions()) {
            // include
            if (toTime != null) {
                if (info.getCommitTimeMs() > toTime.longValue()) {
                    break DO;
                }
            }
            if (toTx != null) {
                if (info.getId() > toTx.longValue()) {
                    break DO;
                }
            }
            // bounds for later loops
            if (minTxId == null) {
                minTxId = info.getId();
            }
            if (maxTxId < info.getId()) {
                maxTxId = info.getId();
            }
            lastTxCommitTime = info.getCommitTimeMs();
            txIdsInDb.set(info.getId());
            txnsFound.add(info);
        }
    } while (transactions.getTransactions().size() > 0);
    return this.infoSrv.reportIndexTransactions(minTxId, txIdsInDb, maxTxId);
}

Example 19 with Transaction

use of org.alfresco.solr.client.Transaction in project SearchServices by Alfresco.

the class MetadataTracker method trackTransactions.

protected void trackTransactions() throws AuthenticationException, IOException, JSONException, EncoderException {
    long startElapsed = System.nanoTime();
    boolean upToDate = false;
    Transactions transactions;
    BoundedDeque<Transaction> txnsFound = new BoundedDeque<Transaction>(100);
    HashSet<Transaction> txsIndexed = new LinkedHashSet<>();
    long totalUpdatedDocs = 0;
    int docCount = 0;
    do {
        try {
            getWriteLock().acquire();
            /*
                * We acquire the tracker state again here and set it globally. This is because the
                * tracker state could have been invalidated due to a rollback by the CommitTracker.
                * In this case the state will revert to the last transaction state record in the index.
                */
            this.state = getTrackerState();
            Long fromCommitTime = getTxFromCommitTime(txnsFound, state.getLastGoodTxCommitTimeInIndex());
            transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000, state.getTimeToStopIndexing());
            setLastTxCommitTimeAndTxIdInTrackerState(transactions, state);
            log.info("Scanning transactions ...");
            if (transactions.getTransactions().size() > 0) {
                log.info(".... from " + transactions.getTransactions().get(0));
                log.info(".... to " + transactions.getTransactions().get(transactions.getTransactions().size() - 1));
            } else {
                log.info(".... none found after lastTxCommitTime " + ((txnsFound.size() > 0) ? txnsFound.getLast().getCommitTimeMs() : state.getLastIndexedTxCommitTime()));
            }
            ArrayList<Transaction> txBatch = new ArrayList<>();
            for (Transaction info : transactions.getTransactions()) {
                boolean isInIndex = (infoSrv.txnInIndex(info.getId(), true) && info.getCommitTimeMs() <= state.getLastIndexedTxCommitTime());
                if (isInIndex) {
                    txnsFound.add(info);
                } else {
                    // correctly next time
                    if (info.getCommitTimeMs() > state.getTimeToStopIndexing()) {
                        upToDate = true;
                        break;
                    }
                    txBatch.add(info);
                    if (getUpdateAndDeleteCount(txBatch) > this.transactionDocsBatchSize) {
                        docCount += indexBatchOfTransactions(txBatch);
                        totalUpdatedDocs += docCount;
                        for (Transaction scheduledTx : txBatch) {
                            txnsFound.add(scheduledTx);
                            txsIndexed.add(scheduledTx);
                        }
                        txBatch.clear();
                    }
                }
                if (docCount > batchCount) {
                    indexTransactionsAfterAsynchronous(txsIndexed, state);
                    long endElapsed = System.nanoTime();
                    trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed);
                    startElapsed = endElapsed;
                    docCount = 0;
                    // Release the write lock allowing the commit tracker to run.
                    this.getWriteLock().release();
                    // Re-acquire the write lock and keep indexing.
                    this.getWriteLock().acquire();
                }
                checkShutdown();
            }
            if (!txBatch.isEmpty()) {
                if (this.getUpdateAndDeleteCount(txBatch) > 0) {
                    docCount += indexBatchOfTransactions(txBatch);
                    totalUpdatedDocs += docCount;
                }
                for (Transaction scheduledTx : txBatch) {
                    txnsFound.add(scheduledTx);
                    txsIndexed.add(scheduledTx);
                }
                txBatch.clear();
            }
            if (txsIndexed.size() > 0) {
                indexTransactionsAfterAsynchronous(txsIndexed, state);
                long endElapsed = System.nanoTime();
                trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed);
                startElapsed = endElapsed;
                docCount = 0;
            }
        } catch (Exception e) {
            throw new IOException(e);
        } finally {
            getWriteLock().release();
        }
    } while ((transactions.getTransactions().size() > 0) && (upToDate == false));
    log.info("total number of docs with metadata updated: " + totalUpdatedDocs);
}

Example 20 with Transaction

use of org.alfresco.solr.client.Transaction in project SearchServices by Alfresco.

the class MetadataTracker method indexBatchOfTransactions.

private int indexBatchOfTransactions(List<Transaction> txBatch) throws AuthenticationException, IOException, JSONException {
    int nodeCount = 0;
    ArrayList<Transaction> nonEmptyTxs = new ArrayList<>(txBatch.size());
    GetNodesParameters gnp = new GetNodesParameters();
    ArrayList<Long> txIds = new ArrayList<Long>();
    for (Transaction tx : txBatch) {
        if (tx.getUpdates() > 0 || tx.getDeletes() > 0) {
            nonEmptyTxs.add(tx);
            txIds.add(tx.getId());
        }
    }
    gnp.setTransactionIds(txIds);
    gnp.setStoreProtocol(storeRef.getProtocol());
    gnp.setStoreIdentifier(storeRef.getIdentifier());
    gnp.setShardProperty(shardProperty);
    List<Node> nodes = client.getNodes(gnp, Integer.MAX_VALUE);
    ArrayList<Node> nodeBatch = new ArrayList<>();
    for (Node node : nodes) {
        if (log.isDebugEnabled()) {
            log.debug(node.toString());
        }
        nodeBatch.add(node);
        if (nodeBatch.size() > nodeBatchSize) {
            nodeCount += nodeBatch.size();
            NodeIndexWorkerRunnable niwr = new NodeIndexWorkerRunnable(this.threadHandler, nodeBatch, this.infoSrv);
            this.threadHandler.scheduleTask(niwr);
            nodeBatch = new ArrayList<>();
        }
    }
    if (nodeBatch.size() > 0) {
        nodeCount += nodeBatch.size();
        NodeIndexWorkerRunnable niwr = new NodeIndexWorkerRunnable(this.threadHandler, nodeBatch, this.infoSrv);
        this.threadHandler.scheduleTask(niwr);
        nodeBatch = new ArrayList<>();
    }
    return nodeCount;
}