Examples with BoundedDeque org.alfresco.solr.BoundedDeque used on opensource projects

Example 1 with BoundedDeque

use of org.alfresco.solr.BoundedDeque in project SearchServices by Alfresco.

the class AclTracker method trackAclChangeSets.

/**
 * @throws AuthenticationException
 * @throws IOException
 * @throws JSONException
 */
protected void trackAclChangeSets() throws AuthenticationException, IOException, JSONException {
    long startElapsed = System.nanoTime();
    boolean upToDate = false;
    AclChangeSets aclChangeSets;
    BoundedDeque<AclChangeSet> changeSetsFound = new BoundedDeque<AclChangeSet>(100);
    HashSet<AclChangeSet> changeSetsIndexed = new LinkedHashSet<AclChangeSet>();
    long totalAclCount = 0;
    int aclCount = 0;
    do {
        try {
            getWriteLock().acquire();
            /*
                * We acquire the tracker state again here and set it globally. This is because the
                * tracker state could have been invalidated due to a rollback by the CommitTracker.
                * In this case the state will revert to the last transaction state record in the index.
                */
            this.state = getTrackerState();
            Long fromCommitTime = getChangeSetFromCommitTime(changeSetsFound, state.getLastGoodChangeSetCommitTimeInIndex());
            aclChangeSets = getSomeAclChangeSets(changeSetsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000, state.getTimeToStopIndexing());
            setLastChangeSetIdAndCommitTimeInTrackerState(aclChangeSets, state);
            log.info("Scanning Acl change sets ...");
            if (aclChangeSets.getAclChangeSets().size() > 0) {
                log.info(".... from " + aclChangeSets.getAclChangeSets().get(0));
                log.info(".... to " + aclChangeSets.getAclChangeSets().get(aclChangeSets.getAclChangeSets().size() - 1));
            } else {
                log.info(".... none found after lastTxCommitTime " + fromCommitTime);
            }
            ArrayList<AclChangeSet> changeSetBatch = new ArrayList<AclChangeSet>();
            for (AclChangeSet changeSet : aclChangeSets.getAclChangeSets()) {
                boolean isInIndex = (changeSet.getCommitTimeMs() <= state.getLastIndexedChangeSetCommitTime() && infoSrv.aclChangeSetInIndex(changeSet.getId(), true));
                if (isInIndex) {
                    changeSetsFound.add(changeSet);
                } else {
                    // correctly next time
                    if (changeSet.getCommitTimeMs() > state.getTimeToStopIndexing()) {
                        upToDate = true;
                        break;
                    }
                    changeSetBatch.add(changeSet);
                    if (getAclCount(changeSetBatch) > changeSetAclsBatchSize) {
                        aclCount += indexBatchOfChangeSets(changeSetBatch);
                        totalAclCount += aclCount;
                        for (AclChangeSet scheduled : changeSetBatch) {
                            changeSetsFound.add(scheduled);
                            changeSetsIndexed.add(scheduled);
                        }
                        changeSetBatch.clear();
                    }
                }
                if (aclCount > batchCount) {
                    if (super.infoSrv.getRegisteredSearcherCount() < getMaxLiveSearchers()) {
                        indexAclChangeSetAfterAsynchronous(changeSetsIndexed, state);
                        long endElapsed = System.nanoTime();
                        trackerStats.addElapsedAclTime(aclCount, endElapsed - startElapsed);
                        startElapsed = endElapsed;
                        aclCount = 0;
                    }
                }
                checkShutdown();
            }
            if (!changeSetBatch.isEmpty()) {
                if (getAclCount(changeSetBatch) > 0) {
                    aclCount += indexBatchOfChangeSets(changeSetBatch);
                    totalAclCount += aclCount;
                }
                for (AclChangeSet scheduled : changeSetBatch) {
                    changeSetsFound.add(scheduled);
                    changeSetsIndexed.add(scheduled);
                }
                changeSetBatch.clear();
            }
            if (changeSetsIndexed.size() > 0) {
                indexAclChangeSetAfterAsynchronous(changeSetsIndexed, state);
                long endElapsed = System.nanoTime();
                trackerStats.addElapsedAclTime(aclCount, endElapsed - startElapsed);
                startElapsed = endElapsed;
                aclCount = 0;
            }
        } catch (InterruptedException e) {
            throw new IOException(e);
        } finally {
            getWriteLock().release();
        }
    } while ((aclChangeSets.getAclChangeSets().size() > 0) && (upToDate == false));
    log.info("total number of acls updated: " + totalAclCount);
}

Example 2 with BoundedDeque

use of org.alfresco.solr.BoundedDeque in project SearchServices by Alfresco.

the class MetadataTracker method checkIndex.

public IndexHealthReport checkIndex(Long toTx, Long toAclTx, Long fromTime, Long toTime) throws IOException, AuthenticationException, JSONException, EncoderException {
    // DB TX Count
    long firstTransactionCommitTime = 0;
    Transactions firstTransactions = client.getTransactions(null, 0L, null, 2000L, 1);
    if (firstTransactions.getTransactions().size() > 0) {
        Transaction firstTransaction = firstTransactions.getTransactions().get(0);
        firstTransactionCommitTime = firstTransaction.getCommitTimeMs();
    }
    IOpenBitSet txIdsInDb = infoSrv.getOpenBitSetInstance();
    Long lastTxCommitTime = Long.valueOf(firstTransactionCommitTime);
    if (fromTime != null) {
        lastTxCommitTime = fromTime;
    }
    long maxTxId = 0;
    Long minTxId = null;
    Transactions transactions;
    BoundedDeque<Transaction> txnsFound = new BoundedDeque<Transaction>(100);
    long endTime = System.currentTimeMillis() + infoSrv.getHoleRetention();
    DO: do {
        transactions = getSomeTransactions(txnsFound, lastTxCommitTime, TIME_STEP_1_HR_IN_MS, 2000, endTime);
        for (Transaction info : transactions.getTransactions()) {
            // include
            if (toTime != null) {
                if (info.getCommitTimeMs() > toTime.longValue()) {
                    break DO;
                }
            }
            if (toTx != null) {
                if (info.getId() > toTx.longValue()) {
                    break DO;
                }
            }
            // bounds for later loops
            if (minTxId == null) {
                minTxId = info.getId();
            }
            if (maxTxId < info.getId()) {
                maxTxId = info.getId();
            }
            lastTxCommitTime = info.getCommitTimeMs();
            txIdsInDb.set(info.getId());
            txnsFound.add(info);
        }
    } while (transactions.getTransactions().size() > 0);
    return this.infoSrv.reportIndexTransactions(minTxId, txIdsInDb, maxTxId);
}

Example 3 with BoundedDeque

use of org.alfresco.solr.BoundedDeque in project SearchServices by Alfresco.

the class MetadataTracker method trackTransactions.

protected void trackTransactions() throws AuthenticationException, IOException, JSONException, EncoderException {
    long startElapsed = System.nanoTime();
    boolean upToDate = false;
    Transactions transactions;
    BoundedDeque<Transaction> txnsFound = new BoundedDeque<Transaction>(100);
    HashSet<Transaction> txsIndexed = new LinkedHashSet<>();
    long totalUpdatedDocs = 0;
    int docCount = 0;
    do {
        try {
            getWriteLock().acquire();
            /*
                * We acquire the tracker state again here and set it globally. This is because the
                * tracker state could have been invalidated due to a rollback by the CommitTracker.
                * In this case the state will revert to the last transaction state record in the index.
                */
            this.state = getTrackerState();
            Long fromCommitTime = getTxFromCommitTime(txnsFound, state.getLastGoodTxCommitTimeInIndex());
            transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000, state.getTimeToStopIndexing());
            setLastTxCommitTimeAndTxIdInTrackerState(transactions, state);
            log.info("Scanning transactions ...");
            if (transactions.getTransactions().size() > 0) {
                log.info(".... from " + transactions.getTransactions().get(0));
                log.info(".... to " + transactions.getTransactions().get(transactions.getTransactions().size() - 1));
            } else {
                log.info(".... none found after lastTxCommitTime " + ((txnsFound.size() > 0) ? txnsFound.getLast().getCommitTimeMs() : state.getLastIndexedTxCommitTime()));
            }
            ArrayList<Transaction> txBatch = new ArrayList<>();
            for (Transaction info : transactions.getTransactions()) {
                boolean isInIndex = (infoSrv.txnInIndex(info.getId(), true) && info.getCommitTimeMs() <= state.getLastIndexedTxCommitTime());
                if (isInIndex) {
                    txnsFound.add(info);
                } else {
                    // correctly next time
                    if (info.getCommitTimeMs() > state.getTimeToStopIndexing()) {
                        upToDate = true;
                        break;
                    }
                    txBatch.add(info);
                    if (getUpdateAndDeleteCount(txBatch) > this.transactionDocsBatchSize) {
                        docCount += indexBatchOfTransactions(txBatch);
                        totalUpdatedDocs += docCount;
                        for (Transaction scheduledTx : txBatch) {
                            txnsFound.add(scheduledTx);
                            txsIndexed.add(scheduledTx);
                        }
                        txBatch.clear();
                    }
                }
                if (docCount > batchCount) {
                    indexTransactionsAfterAsynchronous(txsIndexed, state);
                    long endElapsed = System.nanoTime();
                    trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed);
                    startElapsed = endElapsed;
                    docCount = 0;
                    // Release the write lock allowing the commit tracker to run.
                    this.getWriteLock().release();
                    // Re-acquire the write lock and keep indexing.
                    this.getWriteLock().acquire();
                }
                checkShutdown();
            }
            if (!txBatch.isEmpty()) {
                if (this.getUpdateAndDeleteCount(txBatch) > 0) {
                    docCount += indexBatchOfTransactions(txBatch);
                    totalUpdatedDocs += docCount;
                }
                for (Transaction scheduledTx : txBatch) {
                    txnsFound.add(scheduledTx);
                    txsIndexed.add(scheduledTx);
                }
                txBatch.clear();
            }
            if (txsIndexed.size() > 0) {
                indexTransactionsAfterAsynchronous(txsIndexed, state);
                long endElapsed = System.nanoTime();
                trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed);
                startElapsed = endElapsed;
                docCount = 0;
            }
        } catch (Exception e) {
            throw new IOException(e);
        } finally {
            getWriteLock().release();
        }
    } while ((transactions.getTransactions().size() > 0) && (upToDate == false));
    log.info("total number of docs with metadata updated: " + totalUpdatedDocs);
}

Example 4 with BoundedDeque

use of org.alfresco.solr.BoundedDeque in project SearchServices by Alfresco.

the class AclTracker method checkIndex.

public IndexHealthReport checkIndex(Long toTx, Long toAclTx, Long fromTime, Long toTime) throws AuthenticationException, IOException, JSONException {
    // DB ACL TX Count
    long firstChangeSetCommitTimex = 0;
    AclChangeSets firstChangeSets = client.getAclChangeSets(null, 0L, null, 2000L, 1);
    if (firstChangeSets.getAclChangeSets().size() > 0) {
        AclChangeSet firstChangeSet = firstChangeSets.getAclChangeSets().get(0);
        firstChangeSetCommitTimex = firstChangeSet.getCommitTimeMs();
    }
    IOpenBitSet aclTxIdsInDb = infoSrv.getOpenBitSetInstance();
    Long lastAclTxCommitTime = Long.valueOf(firstChangeSetCommitTimex);
    if (fromTime != null) {
        lastAclTxCommitTime = fromTime;
    }
    long maxAclTxId = 0;
    Long minAclTxId = null;
    long endTime = System.currentTimeMillis() + infoSrv.getHoleRetention();
    AclChangeSets aclTransactions;
    BoundedDeque<AclChangeSet> changeSetsFound = new BoundedDeque<AclChangeSet>(100);
    DO: do {
        aclTransactions = getSomeAclChangeSets(changeSetsFound, lastAclTxCommitTime, TIME_STEP_1_HR_IN_MS, 2000, endTime);
        for (AclChangeSet set : aclTransactions.getAclChangeSets()) {
            // include
            if (toTime != null) {
                if (set.getCommitTimeMs() > toTime.longValue()) {
                    break DO;
                }
            }
            if (toAclTx != null) {
                if (set.getId() > toAclTx.longValue()) {
                    break DO;
                }
            }
            // bounds for later loops
            if (minAclTxId == null) {
                minAclTxId = set.getId();
            }
            if (maxAclTxId < set.getId()) {
                maxAclTxId = set.getId();
            }
            lastAclTxCommitTime = set.getCommitTimeMs();
            aclTxIdsInDb.set(set.getId());
            changeSetsFound.add(set);
        }
    } while (aclTransactions.getAclChangeSets().size() > 0);
    return this.infoSrv.reportAclTransactionsInIndex(minAclTxId, aclTxIdsInDb, maxAclTxId);
}