Examples with Transactions org.alfresco.solr.client.Transactions used on opensource projects

Example 6 with Transactions

use of org.alfresco.solr.client.Transactions in project SearchServices by Alfresco.

the class MetadataTracker method checkIndex.

public IndexHealthReport checkIndex(Long toTx, Long toAclTx, Long fromTime, Long toTime) throws IOException, AuthenticationException, JSONException, EncoderException {
    // DB TX Count
    long firstTransactionCommitTime = 0;
    Transactions firstTransactions = client.getTransactions(null, 0L, null, 2000L, 1);
    if (firstTransactions.getTransactions().size() > 0) {
        Transaction firstTransaction = firstTransactions.getTransactions().get(0);
        firstTransactionCommitTime = firstTransaction.getCommitTimeMs();
    }
    IOpenBitSet txIdsInDb = infoSrv.getOpenBitSetInstance();
    Long lastTxCommitTime = Long.valueOf(firstTransactionCommitTime);
    if (fromTime != null) {
        lastTxCommitTime = fromTime;
    }
    long maxTxId = 0;
    Long minTxId = null;
    Transactions transactions;
    BoundedDeque<Transaction> txnsFound = new BoundedDeque<Transaction>(100);
    long endTime = System.currentTimeMillis() + infoSrv.getHoleRetention();
    DO: do {
        transactions = getSomeTransactions(txnsFound, lastTxCommitTime, TIME_STEP_1_HR_IN_MS, 2000, endTime);
        for (Transaction info : transactions.getTransactions()) {
            // include
            if (toTime != null) {
                if (info.getCommitTimeMs() > toTime.longValue()) {
                    break DO;
                }
            }
            if (toTx != null) {
                if (info.getId() > toTx.longValue()) {
                    break DO;
                }
            }
            // bounds for later loops
            if (minTxId == null) {
                minTxId = info.getId();
            }
            if (maxTxId < info.getId()) {
                maxTxId = info.getId();
            }
            lastTxCommitTime = info.getCommitTimeMs();
            txIdsInDb.set(info.getId());
            txnsFound.add(info);
        }
    } while (transactions.getTransactions().size() > 0);
    return this.infoSrv.reportIndexTransactions(minTxId, txIdsInDb, maxTxId);
}

Example 7 with Transactions

use of org.alfresco.solr.client.Transactions in project SearchServices by Alfresco.

the class MetadataTracker method trackTransactions.

protected void trackTransactions() throws AuthenticationException, IOException, JSONException, EncoderException {
    long startElapsed = System.nanoTime();
    boolean upToDate = false;
    Transactions transactions;
    BoundedDeque<Transaction> txnsFound = new BoundedDeque<Transaction>(100);
    HashSet<Transaction> txsIndexed = new LinkedHashSet<>();
    long totalUpdatedDocs = 0;
    int docCount = 0;
    do {
        try {
            getWriteLock().acquire();
            /*
                * We acquire the tracker state again here and set it globally. This is because the
                * tracker state could have been invalidated due to a rollback by the CommitTracker.
                * In this case the state will revert to the last transaction state record in the index.
                */
            this.state = getTrackerState();
            Long fromCommitTime = getTxFromCommitTime(txnsFound, state.getLastGoodTxCommitTimeInIndex());
            transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000, state.getTimeToStopIndexing());
            setLastTxCommitTimeAndTxIdInTrackerState(transactions, state);
            log.info("Scanning transactions ...");
            if (transactions.getTransactions().size() > 0) {
                log.info(".... from " + transactions.getTransactions().get(0));
                log.info(".... to " + transactions.getTransactions().get(transactions.getTransactions().size() - 1));
            } else {
                log.info(".... none found after lastTxCommitTime " + ((txnsFound.size() > 0) ? txnsFound.getLast().getCommitTimeMs() : state.getLastIndexedTxCommitTime()));
            }
            ArrayList<Transaction> txBatch = new ArrayList<>();
            for (Transaction info : transactions.getTransactions()) {
                boolean isInIndex = (infoSrv.txnInIndex(info.getId(), true) && info.getCommitTimeMs() <= state.getLastIndexedTxCommitTime());
                if (isInIndex) {
                    txnsFound.add(info);
                } else {
                    // correctly next time
                    if (info.getCommitTimeMs() > state.getTimeToStopIndexing()) {
                        upToDate = true;
                        break;
                    }
                    txBatch.add(info);
                    if (getUpdateAndDeleteCount(txBatch) > this.transactionDocsBatchSize) {
                        docCount += indexBatchOfTransactions(txBatch);
                        totalUpdatedDocs += docCount;
                        for (Transaction scheduledTx : txBatch) {
                            txnsFound.add(scheduledTx);
                            txsIndexed.add(scheduledTx);
                        }
                        txBatch.clear();
                    }
                }
                if (docCount > batchCount) {
                    indexTransactionsAfterAsynchronous(txsIndexed, state);
                    long endElapsed = System.nanoTime();
                    trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed);
                    startElapsed = endElapsed;
                    docCount = 0;
                    // Release the write lock allowing the commit tracker to run.
                    this.getWriteLock().release();
                    // Re-acquire the write lock and keep indexing.
                    this.getWriteLock().acquire();
                }
                checkShutdown();
            }
            if (!txBatch.isEmpty()) {
                if (this.getUpdateAndDeleteCount(txBatch) > 0) {
                    docCount += indexBatchOfTransactions(txBatch);
                    totalUpdatedDocs += docCount;
                }
                for (Transaction scheduledTx : txBatch) {
                    txnsFound.add(scheduledTx);
                    txsIndexed.add(scheduledTx);
                }
                txBatch.clear();
            }
            if (txsIndexed.size() > 0) {
                indexTransactionsAfterAsynchronous(txsIndexed, state);
                long endElapsed = System.nanoTime();
                trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed);
                startElapsed = endElapsed;
                docCount = 0;
            }
        } catch (Exception e) {
            throw new IOException(e);
        } finally {
            getWriteLock().release();
        }
    } while ((transactions.getTransactions().size() > 0) && (upToDate == false));
    log.info("total number of docs with metadata updated: " + totalUpdatedDocs);
}

Example 8 with Transactions

use of org.alfresco.solr.client.Transactions in project SearchServices by Alfresco.

the class MetadataTracker method checkRepoAndIndexConsistency.

/**
 * Checks the first and last TX time
 * @param state the state of this tracker
 * @throws AuthenticationException
 * @throws IOException
 * @throws JSONException
 */
private void checkRepoAndIndexConsistency(TrackerState state) throws AuthenticationException, IOException, JSONException {
    Transactions firstTransactions = null;
    if (state.getLastGoodTxCommitTimeInIndex() == 0) {
        state.setCheckedLastTransactionTime(true);
        state.setCheckedFirstTransactionTime(true);
        log.info("No transactions found - no verification required");
        firstTransactions = client.getTransactions(null, 0L, null, 2000L, 1);
        if (!firstTransactions.getTransactions().isEmpty()) {
            Transaction firstTransaction = firstTransactions.getTransactions().get(0);
            long firstTransactionCommitTime = firstTransaction.getCommitTimeMs();
            state.setLastGoodTxCommitTimeInIndex(firstTransactionCommitTime);
            setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions, state);
        }
    }
    if (!state.isCheckedFirstTransactionTime()) {
        firstTransactions = client.getTransactions(null, 0L, null, 2000L, 1);
        if (!firstTransactions.getTransactions().isEmpty()) {
            Transaction firstTransaction = firstTransactions.getTransactions().get(0);
            long firstTxId = firstTransaction.getId();
            long firstTransactionCommitTime = firstTransaction.getCommitTimeMs();
            int setSize = this.infoSrv.getTxDocsSize("" + firstTxId, "" + firstTransactionCommitTime);
            if (setSize == 0) {
                log.error("First transaction was not found with the correct timestamp.");
                log.error("SOLR has successfully connected to your repository  however the SOLR indexes and repository database do not match.");
                log.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
                log.error("You can also check your SOLR connection details in solrcore.properties.");
                throw new AlfrescoRuntimeException("Initial transaction not found with correct timestamp");
            } else if (setSize == 1) {
                state.setCheckedFirstTransactionTime(true);
                log.info("Verified first transaction and timestamp in index");
            } else {
                log.warn("Duplicate initial transaction found with correct timestamp");
            }
        }
    }
    // Checks that the last TxId in solr is <= last TxId in repo
    if (!state.isCheckedLastTransactionTime()) {
        if (firstTransactions == null) {
            firstTransactions = client.getTransactions(null, 0L, null, 2000L, 1);
        }
        setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions, state);
        Long maxTxnCommitTimeInRepo = firstTransactions.getMaxTxnCommitTime();
        Long maxTxnIdInRepo = firstTransactions.getMaxTxnId();
        if (maxTxnCommitTimeInRepo != null && maxTxnIdInRepo != null) {
            Transaction maxTxInIndex = this.infoSrv.getMaxTransactionIdAndCommitTimeInIndex();
            if (maxTxInIndex.getCommitTimeMs() > maxTxnCommitTimeInRepo) {
                log.error("Last transaction was found in index with timestamp later than that of repository.");
                log.error("Max Tx In Index: " + maxTxInIndex.getId() + ", In Repo: " + maxTxnIdInRepo);
                log.error("Max Tx Commit Time In Index: " + maxTxInIndex.getCommitTimeMs() + ", In Repo: " + maxTxnCommitTimeInRepo);
                log.error("SOLR has successfully connected to your repository  however the SOLR indexes and repository database do not match.");
                log.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
                log.error("You can also check your SOLR connection details in solrcore.properties.");
                throw new AlfrescoRuntimeException("Last transaction found in index with incorrect timestamp");
            } else {
                state.setCheckedLastTransactionTime(true);
                log.info("Verified last transaction timestamp in index less than or equal to that of repository.");
            }
        }
    }
}