Examples with FixedRateBarrier com.unboundid.util.FixedRateBarrier used on opensource projects

Example 1 with FixedRateBarrier

use of com.unboundid.util.FixedRateBarrier in project ldapsdk by pingidentity.

the class LDAPModify method doToolProcessing.

/**
 * {@inheritDoc}
 */
@Override()
@NotNull()
public ResultCode doToolProcessing() {
    // Examine the arguments to determine the sets of controls to use for each
    // type of request.
    final ArrayList<Control> addControls = new ArrayList<>(10);
    final ArrayList<Control> deleteControls = new ArrayList<>(10);
    final ArrayList<Control> modifyControls = new ArrayList<>(10);
    final ArrayList<Control> modifyDNControls = new ArrayList<>(10);
    final ArrayList<Control> searchControls = new ArrayList<>(10);
    try {
        createRequestControls(addControls, deleteControls, modifyControls, modifyDNControls, searchControls);
    } catch (final LDAPException le) {
        Debug.debugException(le);
        for (final String line : ResultUtils.formatResult(le, true, 0, WRAP_COLUMN)) {
            err(line);
        }
        return le.getResultCode();
    }
    // If an encryption passphrase file was specified, then read its value.
    String encryptionPassphrase = null;
    if (encryptionPassphraseFile.isPresent()) {
        try {
            encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile(encryptionPassphraseFile.getValue());
        } catch (final LDAPException e) {
            Debug.debugException(e);
            wrapErr(0, WRAP_COLUMN, e.getMessage());
            return e.getResultCode();
        }
    }
    LDAPConnectionPool connectionPool = null;
    LDIFReader ldifReader = null;
    LDIFWriter rejectWriter = null;
    try {
        // before performing the bind.
        try {
            final StartAdministrativeSessionPostConnectProcessor p;
            if (useAdministrativeSession.isPresent()) {
                p = new StartAdministrativeSessionPostConnectProcessor(new StartAdministrativeSessionExtendedRequest(getToolName(), true));
            } else {
                p = null;
            }
            if (!dryRun.isPresent()) {
                connectionPool = getConnectionPool(1, 2, 0, p, null, true, new ReportBindResultLDAPConnectionPoolHealthCheck(this, true, verbose.isPresent()));
            }
        } catch (final LDAPException le) {
            Debug.debugException(le);
            // If the failure was something else, then display that failure result.
            if (le.getResultCode() != ResultCode.INVALID_CREDENTIALS) {
                for (final String line : ResultUtils.formatResult(le, true, 0, WRAP_COLUMN)) {
                    err(line);
                }
            }
            return le.getResultCode();
        }
        if (connectionPool != null) {
            connectionPool.setRetryFailedOperationsDueToInvalidConnections((!neverRetry.isPresent()));
        }
        // Report that the connection was successfully established.
        if (connectionPool != null) {
            try {
                final LDAPConnection connection = connectionPool.getConnection();
                final String hostPort = connection.getHostPort();
                connectionPool.releaseConnection(connection);
                commentToOut(INFO_LDAPMODIFY_CONNECTION_ESTABLISHED.get(hostPort));
                out();
            } catch (final LDAPException le) {
                Debug.debugException(le);
            // This should never happen.
            }
        }
        // If we should process the operations in a transaction, then start that
        // now.
        final ASN1OctetString txnID;
        if (useTransaction.isPresent()) {
            final Control[] startTxnControls;
            if (proxyAs.isPresent()) {
                // In a transaction, the proxied authorization control must only be
                // used in the start transaction request and not in any of the
                // subsequent operation requests.
                startTxnControls = new Control[] { new ProxiedAuthorizationV2RequestControl(proxyAs.getValue()) };
            } else if (proxyV1As.isPresent()) {
                // In a transaction, the proxied authorization control must only be
                // used in the start transaction request and not in any of the
                // subsequent operation requests.
                startTxnControls = new Control[] { new ProxiedAuthorizationV1RequestControl(proxyV1As.getValue()) };
            } else {
                startTxnControls = StaticUtils.NO_CONTROLS;
            }
            try {
                final StartTransactionExtendedResult startTxnResult = (StartTransactionExtendedResult) connectionPool.processExtendedOperation(new StartTransactionExtendedRequest(startTxnControls));
                if (startTxnResult.getResultCode() == ResultCode.SUCCESS) {
                    txnID = startTxnResult.getTransactionID();
                    final TransactionSpecificationRequestControl c = new TransactionSpecificationRequestControl(txnID);
                    addControls.add(c);
                    deleteControls.add(c);
                    modifyControls.add(c);
                    modifyDNControls.add(c);
                    final String txnIDString;
                    if (StaticUtils.isPrintableString(txnID.getValue())) {
                        txnIDString = txnID.stringValue();
                    } else {
                        final StringBuilder hexBuffer = new StringBuilder();
                        StaticUtils.toHex(txnID.getValue(), ":", hexBuffer);
                        txnIDString = hexBuffer.toString();
                    }
                    commentToOut(INFO_LDAPMODIFY_STARTED_TXN.get(txnIDString));
                } else {
                    commentToErr(ERR_LDAPMODIFY_CANNOT_START_TXN.get(startTxnResult.getResultString()));
                    return startTxnResult.getResultCode();
                }
            } catch (final LDAPException le) {
                Debug.debugException(le);
                commentToErr(ERR_LDAPMODIFY_CANNOT_START_TXN.get(StaticUtils.getExceptionMessage(le)));
                return le.getResultCode();
            }
        } else {
            txnID = null;
        }
        // Create an LDIF reader that will be used to read the changes to process.
        try {
            final InputStream ldifInputStream;
            if (ldifFile.isPresent()) {
                ldifInputStream = ToolUtils.getInputStreamForLDIFFiles(ldifFile.getValues(), encryptionPassphrase, getOut(), getErr()).getFirst();
            } else {
                ldifInputStream = in;
            }
            ldifReader = new LDIFReader(ldifInputStream, 0, null, null, characterSet.getValue());
        } catch (final Exception e) {
            commentToErr(ERR_LDAPMODIFY_CANNOT_CREATE_LDIF_READER.get(StaticUtils.getExceptionMessage(e)));
            return ResultCode.LOCAL_ERROR;
        }
        if (stripTrailingSpaces.isPresent()) {
            ldifReader.setTrailingSpaceBehavior(TrailingSpaceBehavior.STRIP);
        }
        // If appropriate, create a reject writer.
        if (rejectFile.isPresent()) {
            try {
                rejectWriter = new LDIFWriter(rejectFile.getValue());
                // Set the maximum allowed wrap column.  This is better than setting a
                // wrap column of zero because it will ensure that comments don't get
                // wrapped either.
                rejectWriter.setWrapColumn(Integer.MAX_VALUE);
            } catch (final Exception e) {
                Debug.debugException(e);
                commentToErr(ERR_LDAPMODIFY_CANNOT_CREATE_REJECT_WRITER.get(rejectFile.getValue().getAbsolutePath(), StaticUtils.getExceptionMessage(e)));
                return ResultCode.LOCAL_ERROR;
            }
        }
        // If appropriate, create a rate limiter.
        final FixedRateBarrier rateLimiter;
        if (ratePerSecond.isPresent()) {
            rateLimiter = new FixedRateBarrier(1000L, ratePerSecond.getValue());
        } else {
            rateLimiter = null;
        }
        // Iterate through the set of changes to process.
        boolean commitTransaction = true;
        ResultCode resultCode = null;
        final ArrayList<LDAPRequest> multiUpdateRequests = new ArrayList<>(10);
        final boolean isBulkModify = modifyEntriesMatchingFilter.isPresent() || modifyEntriesMatchingFiltersFromFile.isPresent() || modifyEntryWithDN.isPresent() || modifyEntriesWithDNsFromFile.isPresent();
        readChangeRecordLoop: while (true) {
            // If there is a rate limiter, then use it to sleep if necessary.
            if ((rateLimiter != null) && (!isBulkModify)) {
                rateLimiter.await();
            }
            // Read the next LDIF change record.  If we get an error then handle it
            // and abort if appropriate.
            final LDIFChangeRecord changeRecord;
            try {
                changeRecord = ldifReader.readChangeRecord(defaultAdd.isPresent());
            } catch (final IOException ioe) {
                Debug.debugException(ioe);
                final String message = ERR_LDAPMODIFY_IO_ERROR_READING_CHANGE.get(StaticUtils.getExceptionMessage(ioe));
                commentToErr(message);
                writeRejectedChange(rejectWriter, message, null);
                commitTransaction = false;
                resultCode = ResultCode.LOCAL_ERROR;
                break;
            } catch (final LDIFException le) {
                Debug.debugException(le);
                final StringBuilder buffer = new StringBuilder();
                if (le.mayContinueReading() && (!useTransaction.isPresent())) {
                    buffer.append(ERR_LDAPMODIFY_RECOVERABLE_LDIF_ERROR_READING_CHANGE.get(le.getLineNumber(), StaticUtils.getExceptionMessage(le)));
                } else {
                    buffer.append(ERR_LDAPMODIFY_UNRECOVERABLE_LDIF_ERROR_READING_CHANGE.get(le.getLineNumber(), StaticUtils.getExceptionMessage(le)));
                }
                if ((resultCode == null) || (resultCode == ResultCode.SUCCESS)) {
                    resultCode = ResultCode.LOCAL_ERROR;
                }
                if ((le.getDataLines() != null) && (!le.getDataLines().isEmpty())) {
                    buffer.append(StaticUtils.EOL);
                    buffer.append(StaticUtils.EOL);
                    buffer.append(ERR_LDAPMODIFY_INVALID_LINES.get());
                    buffer.append(StaticUtils.EOL);
                    for (final String s : le.getDataLines()) {
                        buffer.append(s);
                        buffer.append(StaticUtils.EOL);
                    }
                }
                final String message = buffer.toString();
                commentToErr(message);
                writeRejectedChange(rejectWriter, message, null);
                if (le.mayContinueReading() && (!useTransaction.isPresent())) {
                    continue;
                } else {
                    commitTransaction = false;
                    resultCode = ResultCode.LOCAL_ERROR;
                    break;
                }
            }
            // type.
            if (changeRecord == null) {
                break;
            }
            // the change record into a set of modifications.
            if (modifyEntriesMatchingFilter.isPresent()) {
                for (final Filter filter : modifyEntriesMatchingFilter.getValues()) {
                    final ResultCode rc = handleModifyMatchingFilter(connectionPool, changeRecord, modifyEntriesMatchingFilter.getIdentifierString(), filter, searchControls, modifyControls, rateLimiter, rejectWriter);
                    if (rc != ResultCode.SUCCESS) {
                        if ((resultCode == null) || (resultCode == ResultCode.SUCCESS) || (resultCode == ResultCode.NO_OPERATION)) {
                            resultCode = rc;
                        }
                    }
                }
            }
            if (modifyEntriesMatchingFiltersFromFile.isPresent()) {
                for (final File f : modifyEntriesMatchingFiltersFromFile.getValues()) {
                    final FilterFileReader filterReader;
                    try {
                        filterReader = new FilterFileReader(f);
                    } catch (final Exception e) {
                        Debug.debugException(e);
                        commentToErr(ERR_LDAPMODIFY_ERROR_OPENING_FILTER_FILE.get(f.getAbsolutePath(), StaticUtils.getExceptionMessage(e)));
                        return ResultCode.LOCAL_ERROR;
                    }
                    try {
                        while (true) {
                            final Filter filter;
                            try {
                                filter = filterReader.readFilter();
                            } catch (final IOException ioe) {
                                Debug.debugException(ioe);
                                commentToErr(ERR_LDAPMODIFY_IO_ERROR_READING_FILTER_FILE.get(f.getAbsolutePath(), StaticUtils.getExceptionMessage(ioe)));
                                return ResultCode.LOCAL_ERROR;
                            } catch (final LDAPException le) {
                                Debug.debugException(le);
                                commentToErr(le.getMessage());
                                if (continueOnError.isPresent()) {
                                    if ((resultCode == null) || (resultCode == ResultCode.SUCCESS) || (resultCode == ResultCode.NO_OPERATION)) {
                                        resultCode = le.getResultCode();
                                    }
                                    continue;
                                } else {
                                    return le.getResultCode();
                                }
                            }
                            if (filter == null) {
                                break;
                            }
                            final ResultCode rc = handleModifyMatchingFilter(connectionPool, changeRecord, modifyEntriesMatchingFiltersFromFile.getIdentifierString(), filter, searchControls, modifyControls, rateLimiter, rejectWriter);
                            if (rc != ResultCode.SUCCESS) {
                                if ((resultCode == null) || (resultCode == ResultCode.SUCCESS) || (resultCode == ResultCode.NO_OPERATION)) {
                                    resultCode = rc;
                                }
                            }
                        }
                    } finally {
                        try {
                            filterReader.close();
                        } catch (final Exception e) {
                            Debug.debugException(e);
                        }
                    }
                }
            }
            if (modifyEntryWithDN.isPresent()) {
                for (final DN dn : modifyEntryWithDN.getValues()) {
                    final ResultCode rc = handleModifyWithDN(connectionPool, changeRecord, modifyEntryWithDN.getIdentifierString(), dn, modifyControls, rateLimiter, rejectWriter);
                    if (rc != ResultCode.SUCCESS) {
                        if ((resultCode == null) || (resultCode == ResultCode.SUCCESS) || (resultCode == ResultCode.NO_OPERATION)) {
                            resultCode = rc;
                        }
                    }
                }
            }
            if (modifyEntriesWithDNsFromFile.isPresent()) {
                for (final File f : modifyEntriesWithDNsFromFile.getValues()) {
                    final DNFileReader dnReader;
                    try {
                        dnReader = new DNFileReader(f);
                    } catch (final Exception e) {
                        Debug.debugException(e);
                        commentToErr(ERR_LDAPMODIFY_ERROR_OPENING_DN_FILE.get(f.getAbsolutePath(), StaticUtils.getExceptionMessage(e)));
                        return ResultCode.LOCAL_ERROR;
                    }
                    try {
                        while (true) {
                            final DN dn;
                            try {
                                dn = dnReader.readDN();
                            } catch (final IOException ioe) {
                                Debug.debugException(ioe);
                                commentToErr(ERR_LDAPMODIFY_IO_ERROR_READING_DN_FILE.get(f.getAbsolutePath(), StaticUtils.getExceptionMessage(ioe)));
                                return ResultCode.LOCAL_ERROR;
                            } catch (final LDAPException le) {
                                Debug.debugException(le);
                                commentToErr(le.getMessage());
                                if (continueOnError.isPresent()) {
                                    if ((resultCode == null) || (resultCode == ResultCode.SUCCESS) || (resultCode == ResultCode.NO_OPERATION)) {
                                        resultCode = le.getResultCode();
                                    }
                                    continue;
                                } else {
                                    return le.getResultCode();
                                }
                            }
                            if (dn == null) {
                                break;
                            }
                            final ResultCode rc = handleModifyWithDN(connectionPool, changeRecord, modifyEntriesWithDNsFromFile.getIdentifierString(), dn, modifyControls, rateLimiter, rejectWriter);
                            if (rc != ResultCode.SUCCESS) {
                                if ((resultCode == null) || (resultCode == ResultCode.SUCCESS) || (resultCode == ResultCode.NO_OPERATION)) {
                                    resultCode = rc;
                                }
                            }
                        }
                    } finally {
                        try {
                            dnReader.close();
                        } catch (final Exception e) {
                            Debug.debugException(e);
                        }
                    }
                }
            }
            if (isBulkModify) {
                continue;
            }
            try {
                final ResultCode rc;
                if (changeRecord instanceof LDIFAddChangeRecord) {
                    rc = doAdd((LDIFAddChangeRecord) changeRecord, addControls, connectionPool, multiUpdateRequests, rejectWriter);
                } else if (changeRecord instanceof LDIFDeleteChangeRecord) {
                    rc = doDelete((LDIFDeleteChangeRecord) changeRecord, deleteControls, connectionPool, multiUpdateRequests, rejectWriter);
                } else if (changeRecord instanceof LDIFModifyChangeRecord) {
                    rc = doModify((LDIFModifyChangeRecord) changeRecord, modifyControls, connectionPool, multiUpdateRequests, rejectWriter);
                } else if (changeRecord instanceof LDIFModifyDNChangeRecord) {
                    rc = doModifyDN((LDIFModifyDNChangeRecord) changeRecord, modifyDNControls, connectionPool, multiUpdateRequests, rejectWriter);
                } else {
                    // This should never happen.
                    commentToErr(ERR_LDAPMODIFY_UNSUPPORTED_CHANGE_RECORD_HEADER.get());
                    for (final String line : changeRecord.toLDIF()) {
                        err("#      " + line);
                    }
                    throw new LDAPException(ResultCode.PARAM_ERROR, ERR_LDAPMODIFY_UNSUPPORTED_CHANGE_RECORD_HEADER.get() + changeRecord.toString());
                }
                if ((resultCode == null) && (rc != ResultCode.SUCCESS)) {
                    resultCode = rc;
                }
            } catch (final LDAPException le) {
                Debug.debugException(le);
                commitTransaction = false;
                if (continueOnError.isPresent()) {
                    if ((resultCode == null) || (resultCode == ResultCode.SUCCESS) || (resultCode == ResultCode.NO_OPERATION)) {
                        resultCode = le.getResultCode();
                    }
                } else {
                    resultCode = le.getResultCode();
                    break;
                }
            }
        }
        // operation, then process that now.
        if (useTransaction.isPresent()) {
            LDAPResult endTxnResult;
            final EndTransactionExtendedRequest endTxnRequest = new EndTransactionExtendedRequest(txnID, commitTransaction);
            try {
                endTxnResult = connectionPool.processExtendedOperation(endTxnRequest);
            } catch (final LDAPException le) {
                endTxnResult = le.toLDAPResult();
            }
            displayResult(endTxnResult, false);
            if (((resultCode == null) || (resultCode == ResultCode.SUCCESS)) && (endTxnResult.getResultCode() != ResultCode.SUCCESS)) {
                resultCode = endTxnResult.getResultCode();
            }
        } else if (multiUpdateErrorBehavior.isPresent()) {
            final MultiUpdateErrorBehavior errorBehavior;
            if (multiUpdateErrorBehavior.getValue().equalsIgnoreCase("atomic")) {
                errorBehavior = MultiUpdateErrorBehavior.ATOMIC;
            } else if (multiUpdateErrorBehavior.getValue().equalsIgnoreCase("abort-on-error")) {
                errorBehavior = MultiUpdateErrorBehavior.ABORT_ON_ERROR;
            } else {
                errorBehavior = MultiUpdateErrorBehavior.CONTINUE_ON_ERROR;
            }
            final Control[] multiUpdateControls;
            if (proxyAs.isPresent()) {
                multiUpdateControls = new Control[] { new ProxiedAuthorizationV2RequestControl(proxyAs.getValue()) };
            } else if (proxyV1As.isPresent()) {
                multiUpdateControls = new Control[] { new ProxiedAuthorizationV1RequestControl(proxyV1As.getValue()) };
            } else {
                multiUpdateControls = StaticUtils.NO_CONTROLS;
            }
            ExtendedResult multiUpdateResult;
            try {
                commentToOut(INFO_LDAPMODIFY_SENDING_MULTI_UPDATE_REQUEST.get());
                final MultiUpdateExtendedRequest multiUpdateRequest = new MultiUpdateExtendedRequest(errorBehavior, multiUpdateRequests, multiUpdateControls);
                multiUpdateResult = connectionPool.processExtendedOperation(multiUpdateRequest);
            } catch (final LDAPException le) {
                multiUpdateResult = new ExtendedResult(le);
            }
            displayResult(multiUpdateResult, false);
            resultCode = multiUpdateResult.getResultCode();
        }
        if (resultCode == null) {
            return ResultCode.SUCCESS;
        } else {
            return resultCode;
        }
    } finally {
        if (rejectWriter != null) {
            try {
                rejectWriter.close();
            } catch (final Exception e) {
                Debug.debugException(e);
            }
        }
        if (ldifReader != null) {
            try {
                ldifReader.close();
            } catch (final Exception e) {
                Debug.debugException(e);
            }
        }
        if (connectionPool != null) {
            try {
                connectionPool.close();
            } catch (final Exception e) {
                Debug.debugException(e);
            }
        }
    }
}

Example 2 with FixedRateBarrier

use of com.unboundid.util.FixedRateBarrier in project ldapsdk by pingidentity.

the class LDAPSearch method doToolProcessing.

/**
 * {@inheritDoc}
 */
@Override()
@NotNull()
public ResultCode doToolProcessing() {
    // If we should encrypt the output, then get the encryption passphrase.
    if (encryptOutput.isPresent()) {
        if (encryptionPassphraseFile.isPresent()) {
            try {
                encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile(encryptionPassphraseFile.getValue());
            } catch (final LDAPException e) {
                Debug.debugException(e);
                wrapErr(0, WRAP_COLUMN, e.getMessage());
                return e.getResultCode();
            }
        } else {
            try {
                encryptionPassphrase = ToolUtils.promptForEncryptionPassphrase(false, true, getOut(), getErr());
            } catch (final LDAPException e) {
                Debug.debugException(e);
                wrapErr(0, WRAP_COLUMN, e.getMessage());
                return e.getResultCode();
            }
        }
    }
    // the header to standard output.
    if (outputFile.isPresent()) {
        if (!separateOutputFilePerSearch.isPresent()) {
            try {
                OutputStream s = new FileOutputStream(outputFile.getValue());
                if (encryptOutput.isPresent()) {
                    s = new PassphraseEncryptedOutputStream(encryptionPassphrase, s);
                }
                if (compressOutput.isPresent()) {
                    s = new GZIPOutputStream(s);
                }
                if (teeResultsToStandardOut.isPresent()) {
                    outStream = new PrintStream(new TeeOutputStream(s, getOut()));
                } else {
                    outStream = new PrintStream(s);
                }
                resultWriter.updateOutputStream(outStream);
                errStream = outStream;
            } catch (final Exception e) {
                Debug.debugException(e);
                wrapErr(0, WRAP_COLUMN, ERR_LDAPSEARCH_CANNOT_OPEN_OUTPUT_FILE.get(outputFile.getValue().getAbsolutePath(), StaticUtils.getExceptionMessage(e)));
                return ResultCode.LOCAL_ERROR;
            }
            resultWriter.writeHeader();
        }
    } else {
        resultWriter.writeHeader();
    }
    // Examine the arguments to determine the sets of controls to use for each
    // type of request.
    final List<Control> searchControls = getSearchControls();
    // If appropriate, ensure that any search result entries that include
    // base64-encoded attribute values will also include comments that attempt
    // to provide a human-readable representation of that value.
    final boolean originalCommentAboutBase64EncodedValues = LDIFWriter.commentAboutBase64EncodedValues();
    LDIFWriter.setCommentAboutBase64EncodedValues(!suppressBase64EncodedValueComments.isPresent());
    LDAPConnectionPool pool = null;
    try {
        // directory server.
        if (!dryRun.isPresent()) {
            try {
                final StartAdministrativeSessionPostConnectProcessor p;
                if (useAdministrativeSession.isPresent()) {
                    p = new StartAdministrativeSessionPostConnectProcessor(new StartAdministrativeSessionExtendedRequest(getToolName(), true));
                } else {
                    p = null;
                }
                pool = getConnectionPool(1, 1, 0, p, null, true, new ReportBindResultLDAPConnectionPoolHealthCheck(this, true, false));
            } catch (final LDAPException le) {
                // This shouldn't happen since the pool won't throw an exception if an
                // attempt to create an initial connection fails.
                Debug.debugException(le);
                commentToErr(ERR_LDAPSEARCH_CANNOT_CREATE_CONNECTION_POOL.get(StaticUtils.getExceptionMessage(le)));
                return le.getResultCode();
            }
            if (retryFailedOperations.isPresent()) {
                pool.setRetryFailedOperationsDueToInvalidConnections(true);
            }
        }
        // If appropriate, create a rate limiter.
        final FixedRateBarrier rateLimiter;
        if (ratePerSecond.isPresent()) {
            rateLimiter = new FixedRateBarrier(1000L, ratePerSecond.getValue());
        } else {
            rateLimiter = null;
        }
        // requests from those URLs.
        if (ldapURLFile.isPresent()) {
            return searchWithLDAPURLs(pool, rateLimiter, searchControls);
        }
        // Get the set of requested attributes, as a combination of the
        // requestedAttribute argument values and any trailing arguments.
        final ArrayList<String> attrList = new ArrayList<>(10);
        if (requestedAttribute.isPresent()) {
            attrList.addAll(requestedAttribute.getValues());
        }
        final List<String> trailingArgs = parser.getTrailingArguments();
        if (!trailingArgs.isEmpty()) {
            final Iterator<String> trailingArgIterator = trailingArgs.iterator();
            if (!(filter.isPresent() || filterFile.isPresent())) {
                trailingArgIterator.next();
            }
            while (trailingArgIterator.hasNext()) {
                attrList.add(trailingArgIterator.next());
            }
        }
        final String[] attributes = new String[attrList.size()];
        attrList.toArray(attributes);
        // If either or both the filter or filterFile arguments are provided, then
        // use them to get the filters to process.  Otherwise, the first trailing
        // argument should be a filter.
        ResultCode resultCode = ResultCode.SUCCESS;
        if (filter.isPresent() || filterFile.isPresent()) {
            if (filter.isPresent()) {
                for (final Filter f : filter.getValues()) {
                    final ResultCode rc = searchWithFilter(pool, f, attributes, rateLimiter, searchControls);
                    if (rc != ResultCode.SUCCESS) {
                        if (resultCode == ResultCode.SUCCESS) {
                            resultCode = rc;
                        }
                        if (!continueOnError.isPresent()) {
                            return resultCode;
                        }
                    }
                }
            }
            if (filterFile.isPresent()) {
                final ResultCode rc = searchWithFilterFile(pool, attributes, rateLimiter, searchControls);
                if (rc != ResultCode.SUCCESS) {
                    if (resultCode == ResultCode.SUCCESS) {
                        resultCode = rc;
                    }
                    if (!continueOnError.isPresent()) {
                        return resultCode;
                    }
                }
            }
        } else {
            final Filter f;
            try {
                final String filterStr = parser.getTrailingArguments().iterator().next();
                f = Filter.create(filterStr);
            } catch (final LDAPException le) {
                // This should never happen.
                Debug.debugException(le);
                displayResult(le.toLDAPResult());
                return le.getResultCode();
            }
            resultCode = searchWithFilter(pool, f, attributes, rateLimiter, searchControls);
        }
        return resultCode;
    } finally {
        if (pool != null) {
            try {
                pool.close();
            } catch (final Exception e) {
                Debug.debugException(e);
            }
        }
        if (outStream != null) {
            try {
                outStream.close();
                outStream = null;
            } catch (final Exception e) {
                Debug.debugException(e);
            }
        }
        if (errStream != null) {
            try {
                errStream.close();
                errStream = null;
            } catch (final Exception e) {
                Debug.debugException(e);
            }
        }
        LDIFWriter.setCommentAboutBase64EncodedValues(originalCommentAboutBase64EncodedValues);
    }
}

Example 3 with FixedRateBarrier

use of com.unboundid.util.FixedRateBarrier in project ldapsdk by pingidentity.

the class ManageAccount method doToolProcessing.

/**
 * {@inheritDoc}
 */
@Override()
@NotNull()
public ResultCode doToolProcessing() {
    // If we should just generate a sample rate data file, then do that now.
    final FileArgument generateSampleRateFile = parser.getFileArgument(ARG_GENERATE_SAMPLE_RATE_FILE);
    if (generateSampleRateFile.isPresent()) {
        try {
            RateAdjustor.writeSampleVariableRateFile(generateSampleRateFile.getValue());
            return ResultCode.SUCCESS;
        } catch (final Exception e) {
            Debug.debugException(e);
            wrapErr(0, WRAP_COLUMN, ERR_MANAGE_ACCT_CANNOT_GENERATE_SAMPLE_RATE_FILE.get(generateSampleRateFile.getValue().getAbsolutePath(), StaticUtils.getExceptionMessage(e)));
            return ResultCode.LOCAL_ERROR;
        }
    }
    // If we need to create a fixed-rate barrier and/or use a variable rate
    // definition, then set that up.
    final IntegerArgument ratePerSecond = parser.getIntegerArgument(ARG_RATE_PER_SECOND);
    final FileArgument variableRateData = parser.getFileArgument(ARG_VARIABLE_RATE_DATA);
    if (ratePerSecond.isPresent() || variableRateData.isPresent()) {
        if (ratePerSecond.isPresent()) {
            rateLimiter = new FixedRateBarrier(1000L, ratePerSecond.getValue());
        } else {
            rateLimiter = new FixedRateBarrier(1000L, Integer.MAX_VALUE);
        }
        if (variableRateData.isPresent()) {
            try {
                rateAdjustor = RateAdjustor.newInstance(rateLimiter, ratePerSecond.getValue(), variableRateData.getValue());
            } catch (final Exception e) {
                Debug.debugException(e);
                wrapErr(0, WRAP_COLUMN, ERR_MANAGE_ACCT_CANNOT_CREATE_RATE_ADJUSTOR.get(variableRateData.getValue().getAbsolutePath(), StaticUtils.getExceptionMessage(e)));
                return ResultCode.PARAM_ERROR;
            }
        }
    }
    // Create the connection pool to use for all processing.
    final LDAPConnectionPool pool;
    final int numSearchThreads = parser.getIntegerArgument(ARG_NUM_SEARCH_THREADS).getValue();
    try {
        final int numOperationThreads = parser.getIntegerArgument(ARG_NUM_THREADS).getValue();
        pool = getConnectionPool(numOperationThreads, (numOperationThreads + numSearchThreads));
        // Explicitly disable automatic retry, since it probably won't work
        // reliably for extended operations anyway.  We'll handle retry manually.
        pool.setRetryFailedOperationsDueToInvalidConnections(false);
        // Set a maximum connection age of 30 minutes.
        pool.setMaxConnectionAgeMillis(1_800_000L);
    } catch (final LDAPException le) {
        Debug.debugException(le);
        wrapErr(0, WRAP_COLUMN, ERR_MANAGE_ACCT_CANNOT_CREATE_CONNECTION_POOL.get(getToolName(), le.getMessage()));
        return le.getResultCode();
    }
    try {
        // Create the output writer.  This should always succeed.
        outputWriter = new LDIFWriter(getOut());
        // Create the reject writer if appropriate.
        final FileArgument rejectFile = parser.getFileArgument(ARG_REJECT_FILE);
        if (rejectFile.isPresent()) {
            final BooleanArgument appendToRejectFile = parser.getBooleanArgument(ARG_APPEND_TO_REJECT_FILE);
            try {
                rejectWriter = new LDIFWriter(new FileOutputStream(rejectFile.getValue(), appendToRejectFile.isPresent()));
            } catch (final Exception e) {
                Debug.debugException(e);
                wrapErr(0, WRAP_COLUMN, ERR_MANAGE_ACCT_CANNOT_CREATE_REJECT_WRITER.get(rejectFile.getValue().getAbsolutePath(), StaticUtils.getExceptionMessage(e)));
                return ResultCode.LOCAL_ERROR;
            }
        }
        // Create the processor that will be used to actually perform the
        // manage-account operation processing for each entry.
        final ManageAccountProcessor processor;
        try {
            processor = new ManageAccountProcessor(this, pool, rateLimiter, outputWriter, rejectWriter);
        } catch (final LDAPException le) {
            Debug.debugException(le);
            wrapErr(0, WRAP_COLUMN, ERR_MANAGE_ACCT_CANNOT_CREATE_PROCESSOR.get(StaticUtils.getExceptionMessage(le)));
            return le.getResultCode();
        }
        // If we should use a rate adjustor, then start it now.
        if (rateAdjustor != null) {
            rateAdjustor.start();
        }
        // If any targetDN values were provided, then process them now.
        final DNArgument targetDN = parser.getDNArgument(ARG_TARGET_DN);
        if (targetDN.isPresent()) {
            for (final DN dn : targetDN.getValues()) {
                if (cancelRequested()) {
                    return ResultCode.USER_CANCELED;
                }
                processor.process(dn.toString());
            }
        }
        // If any DN input files were specified, then process them now.
        final FileArgument dnInputFile = parser.getFileArgument(ARG_DN_INPUT_FILE);
        if (dnInputFile.isPresent()) {
            for (final File f : dnInputFile.getValues()) {
                DNFileReader reader = null;
                try {
                    reader = new DNFileReader(f);
                    while (true) {
                        if (cancelRequested()) {
                            return ResultCode.USER_CANCELED;
                        }
                        final DN dn;
                        try {
                            dn = reader.readDN();
                        } catch (final LDAPException le) {
                            Debug.debugException(le);
                            processor.handleMessage(le.getMessage(), true);
                            continue;
                        }
                        if (dn == null) {
                            break;
                        }
                        processor.process(dn.toString());
                    }
                } catch (final Exception e) {
                    Debug.debugException(e);
                    processor.handleMessage(ERR_MANAGE_ACCT_ERROR_READING_DN_FILE.get(f.getAbsolutePath(), StaticUtils.getExceptionMessage(e)), true);
                } finally {
                    if (reader != null) {
                        try {
                            reader.close();
                        } catch (final Exception e2) {
                            Debug.debugException(e2);
                        }
                    }
                }
            }
        }
        // If any target filters were specified, then process them now.
        final FilterArgument targetFilter = parser.getFilterArgument(ARG_TARGET_FILTER);
        if (targetFilter.isPresent()) {
            searchProcessor = new ManageAccountSearchProcessor(this, processor, pool);
            for (final Filter f : targetFilter.getValues()) {
                searchProcessor.processFilter(f);
            }
        }
        // If any filter input files were specified, then process them now.
        final FileArgument filterInputFile = parser.getFileArgument(ARG_FILTER_INPUT_FILE);
        if (filterInputFile.isPresent()) {
            if (searchProcessor == null) {
                searchProcessor = new ManageAccountSearchProcessor(this, processor, pool);
            }
            for (final File f : filterInputFile.getValues()) {
                FilterFileReader reader = null;
                try {
                    reader = new FilterFileReader(f);
                    while (true) {
                        if (cancelRequested()) {
                            return ResultCode.USER_CANCELED;
                        }
                        final Filter filter;
                        try {
                            filter = reader.readFilter();
                        } catch (final LDAPException le) {
                            Debug.debugException(le);
                            processor.handleMessage(le.getMessage(), true);
                            continue;
                        }
                        if (filter == null) {
                            break;
                        }
                        searchProcessor.processFilter(filter);
                    }
                } catch (final Exception e) {
                    Debug.debugException(e);
                    processor.handleMessage(ERR_MANAGE_ACCT_ERROR_READING_FILTER_FILE.get(f.getAbsolutePath(), StaticUtils.getExceptionMessage(e)), true);
                } finally {
                    if (reader != null) {
                        try {
                            reader.close();
                        } catch (final Exception e2) {
                            Debug.debugException(e2);
                        }
                    }
                }
            }
        }
        // If any target user IDs were specified, then process them now.
        final StringArgument targetUserID = parser.getStringArgument(ARG_TARGET_USER_ID);
        if (targetUserID.isPresent()) {
            if (searchProcessor == null) {
                searchProcessor = new ManageAccountSearchProcessor(this, processor, pool);
            }
            for (final String userID : targetUserID.getValues()) {
                searchProcessor.processUserID(userID);
            }
        }
        // If any user ID input files were specified, then process them now.
        final FileArgument userIDInputFile = parser.getFileArgument(ARG_USER_ID_INPUT_FILE);
        if (userIDInputFile.isPresent()) {
            if (searchProcessor == null) {
                searchProcessor = new ManageAccountSearchProcessor(this, processor, pool);
            }
            for (final File f : userIDInputFile.getValues()) {
                BufferedReader reader = null;
                try {
                    reader = new BufferedReader(new FileReader(f));
                    while (true) {
                        if (cancelRequested()) {
                            return ResultCode.USER_CANCELED;
                        }
                        final String line = reader.readLine();
                        if (line == null) {
                            break;
                        }
                        if ((line.length() == 0) || line.startsWith("#")) {
                            continue;
                        }
                        searchProcessor.processUserID(line.trim());
                    }
                } catch (final Exception e) {
                    Debug.debugException(e);
                    processor.handleMessage(ERR_MANAGE_ACCT_ERROR_READING_USER_ID_FILE.get(f.getAbsolutePath(), StaticUtils.getExceptionMessage(e)), true);
                } finally {
                    if (reader != null) {
                        try {
                            reader.close();
                        } catch (final Exception e2) {
                            Debug.debugException(e2);
                        }
                    }
                }
            }
        }
        allFiltersProvided.set(true);
        if (searchProcessor != null) {
            searchProcessor.waitForCompletion();
        }
        allDNsProvided.set(true);
        processor.waitForCompletion();
    } finally {
        pool.close();
        if (rejectWriter != null) {
            try {
                rejectWriter.close();
            } catch (final Exception e) {
                Debug.debugException(e);
            }
        }
    }
    // if some of the operations failed.
    return ResultCode.SUCCESS;
}

Example 4 with FixedRateBarrier

use of com.unboundid.util.FixedRateBarrier in project ldapsdk by pingidentity.

the class ParallelUpdate method doShutdownHookProcessing.

/**
 * {@inheritDoc}
 */
@Override()
protected void doShutdownHookProcessing(@Nullable final ResultCode resultCode) {
    shouldAbort.set(true);
    final FixedRateBarrier b = rateLimiter;
    if (b != null) {
        b.shutdownRequested();
    }
}

Example 5 with FixedRateBarrier

use of com.unboundid.util.FixedRateBarrier in project ldapsdk by pingidentity.

the class SearchRate method doToolProcessing.

/**
 * Performs the actual processing for this tool.  In this case, it gets a
 * connection to the directory server and uses it to perform the requested
 * searches.
 *
 * @return  The result code for the processing that was performed.
 */
@Override()
@NotNull()
public ResultCode doToolProcessing() {
    // variable rate data file and return.
    if (sampleRateFile.isPresent()) {
        try {
            RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue());
            return ResultCode.SUCCESS;
        } catch (final Exception e) {
            Debug.debugException(e);
            err("An error occurred while trying to write sample variable data " + "rate file '", sampleRateFile.getValue().getAbsolutePath(), "':  ", StaticUtils.getExceptionMessage(e));
            return ResultCode.LOCAL_ERROR;
        }
    }
    // Determine the random seed to use.
    final Long seed;
    if (randomSeed.isPresent()) {
        seed = Long.valueOf(randomSeed.getValue());
    } else {
        seed = null;
    }
    // Create value patterns for the base DN, filter, LDAP URL, and proxied
    // authorization DN.
    final ValuePattern dnPattern;
    try {
        if (baseDN.getNumOccurrences() > 0) {
            dnPattern = new ValuePattern(baseDN.getValue(), seed);
        } else if (ldapURL.isPresent()) {
            dnPattern = null;
        } else {
            dnPattern = new ValuePattern("", seed);
        }
    } catch (final ParseException pe) {
        Debug.debugException(pe);
        err("Unable to parse the base DN value pattern:  ", pe.getMessage());
        return ResultCode.PARAM_ERROR;
    }
    final ValuePattern filterPattern;
    try {
        if (filter.isPresent()) {
            filterPattern = new ValuePattern(filter.getValue(), seed);
        } else {
            filterPattern = null;
        }
    } catch (final ParseException pe) {
        Debug.debugException(pe);
        err("Unable to parse the filter pattern:  ", pe.getMessage());
        return ResultCode.PARAM_ERROR;
    }
    final ValuePattern ldapURLPattern;
    try {
        if (ldapURL.isPresent()) {
            ldapURLPattern = new ValuePattern(ldapURL.getValue(), seed);
        } else {
            ldapURLPattern = null;
        }
    } catch (final ParseException pe) {
        Debug.debugException(pe);
        err("Unable to parse the LDAP URL pattern:  ", pe.getMessage());
        return ResultCode.PARAM_ERROR;
    }
    final ValuePattern authzIDPattern;
    if (proxyAs.isPresent()) {
        try {
            authzIDPattern = new ValuePattern(proxyAs.getValue(), seed);
        } catch (final ParseException pe) {
            Debug.debugException(pe);
            err("Unable to parse the proxied authorization pattern:  ", pe.getMessage());
            return ResultCode.PARAM_ERROR;
        }
    } else {
        authzIDPattern = null;
    }
    // Get the alias dereference policy to use.
    final DereferencePolicy derefPolicy;
    final String derefValue = StaticUtils.toLowerCase(dereferencePolicy.getValue());
    if (derefValue.equals("always")) {
        derefPolicy = DereferencePolicy.ALWAYS;
    } else if (derefValue.equals("search")) {
        derefPolicy = DereferencePolicy.SEARCHING;
    } else if (derefValue.equals("find")) {
        derefPolicy = DereferencePolicy.FINDING;
    } else {
        derefPolicy = DereferencePolicy.NEVER;
    }
    // Get the set of controls to include in search requests.
    final ArrayList<Control> controlList = new ArrayList<>(5);
    if (assertionFilter.isPresent()) {
        controlList.add(new AssertionRequestControl(assertionFilter.getValue()));
    }
    if (sortOrder.isPresent()) {
        final ArrayList<SortKey> sortKeys = new ArrayList<>(5);
        final StringTokenizer tokenizer = new StringTokenizer(sortOrder.getValue(), ",");
        while (tokenizer.hasMoreTokens()) {
            String token = tokenizer.nextToken().trim();
            final boolean ascending;
            if (token.startsWith("+")) {
                ascending = true;
                token = token.substring(1);
            } else if (token.startsWith("-")) {
                ascending = false;
                token = token.substring(1);
            } else {
                ascending = true;
            }
            final String attributeName;
            final String matchingRuleID;
            final int colonPos = token.indexOf(':');
            if (colonPos < 0) {
                attributeName = token;
                matchingRuleID = null;
            } else {
                attributeName = token.substring(0, colonPos);
                matchingRuleID = token.substring(colonPos + 1);
            }
            sortKeys.add(new SortKey(attributeName, matchingRuleID, (!ascending)));
        }
        controlList.add(new ServerSideSortRequestControl(sortKeys));
    }
    if (control.isPresent()) {
        controlList.addAll(control.getValues());
    }
    // Get the attributes to return.
    final String[] attrs;
    if (attributes.isPresent()) {
        final List<String> attrList = attributes.getValues();
        attrs = new String[attrList.size()];
        attrList.toArray(attrs);
    } else {
        attrs = StaticUtils.NO_STRINGS;
    }
    // If the --ratePerSecond option was specified, then limit the rate
    // accordingly.
    FixedRateBarrier fixedRateBarrier = null;
    if (ratePerSecond.isPresent() || variableRateData.isPresent()) {
        // We might not have a rate per second if --variableRateData is specified.
        // The rate typically doesn't matter except when we have warm-up
        // intervals.  In this case, we'll run at the max rate.
        final int intervalSeconds = collectionInterval.getValue();
        final int ratePerInterval = (ratePerSecond.getValue() == null) ? Integer.MAX_VALUE : ratePerSecond.getValue() * intervalSeconds;
        fixedRateBarrier = new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval);
    }
    // If --variableRateData was specified, then initialize a RateAdjustor.
    RateAdjustor rateAdjustor = null;
    if (variableRateData.isPresent()) {
        try {
            rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier, ratePerSecond.getValue(), variableRateData.getValue());
        } catch (final IOException | IllegalArgumentException e) {
            Debug.debugException(e);
            err("Initializing the variable rates failed: " + e.getMessage());
            return ResultCode.PARAM_ERROR;
        }
    }
    // If the --maxOutstandingRequests option was specified, then create the
    // semaphore used to enforce that limit.
    final Semaphore asyncSemaphore;
    if (maxOutstandingRequests.isPresent()) {
        asyncSemaphore = new Semaphore(maxOutstandingRequests.getValue());
    } else {
        asyncSemaphore = null;
    }
    // Determine whether to include timestamps in the output and if so what
    // format should be used for them.
    final boolean includeTimestamp;
    final String timeFormat;
    if (timestampFormat.getValue().equalsIgnoreCase("with-date")) {
        includeTimestamp = true;
        timeFormat = "dd/MM/yyyy HH:mm:ss";
    } else if (timestampFormat.getValue().equalsIgnoreCase("without-date")) {
        includeTimestamp = true;
        timeFormat = "HH:mm:ss";
    } else {
        includeTimestamp = false;
        timeFormat = null;
    }
    // Determine whether any warm-up intervals should be run.
    final long totalIntervals;
    final boolean warmUp;
    int remainingWarmUpIntervals = warmUpIntervals.getValue();
    if (remainingWarmUpIntervals > 0) {
        warmUp = true;
        totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals;
    } else {
        warmUp = true;
        totalIntervals = 0L + numIntervals.getValue();
    }
    // Create the table that will be used to format the output.
    final OutputFormat outputFormat;
    if (csvFormat.isPresent()) {
        outputFormat = OutputFormat.CSV;
    } else {
        outputFormat = OutputFormat.COLUMNS;
    }
    final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp, timeFormat, outputFormat, " ", new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", "Searches/Sec"), new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", "Avg Dur ms"), new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", "Entries/Srch"), new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", "Errors/Sec"), new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", "Searches/Sec"), new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", "Avg Dur ms"));
    // Create values to use for statistics collection.
    final AtomicLong searchCounter = new AtomicLong(0L);
    final AtomicLong entryCounter = new AtomicLong(0L);
    final AtomicLong errorCounter = new AtomicLong(0L);
    final AtomicLong searchDurations = new AtomicLong(0L);
    final ResultCodeCounter rcCounter = new ResultCodeCounter();
    // Determine the length of each interval in milliseconds.
    final long intervalMillis = 1000L * collectionInterval.getValue();
    // Create the threads to use for the searches.
    final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1);
    final SearchRateThread[] threads = new SearchRateThread[numThreads.getValue()];
    for (int i = 0; i < threads.length; i++) {
        final LDAPConnection connection;
        try {
            connection = getConnection();
        } catch (final LDAPException le) {
            Debug.debugException(le);
            err("Unable to connect to the directory server:  ", StaticUtils.getExceptionMessage(le));
            return le.getResultCode();
        }
        threads[i] = new SearchRateThread(this, i, connection, asynchronousMode.isPresent(), dnPattern, scope.getValue(), derefPolicy, sizeLimit.getValue(), timeLimitSeconds.getValue(), typesOnly.isPresent(), filterPattern, attrs, ldapURLPattern, authzIDPattern, simplePageSize.getValue(), controlList, iterationsBeforeReconnect.getValue(), runningThreads, barrier, searchCounter, entryCounter, searchDurations, errorCounter, rcCounter, fixedRateBarrier, asyncSemaphore);
        threads[i].start();
    }
    // Display the table header.
    for (final String headerLine : formatter.getHeaderLines(true)) {
        out(headerLine);
    }
    // which case, we'll start it after the warm-up is complete.
    if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0)) {
        rateAdjustor.start();
    }
    // Indicate that the threads can start running.
    try {
        barrier.await();
    } catch (final Exception e) {
        Debug.debugException(e);
    }
    long overallStartTime = System.nanoTime();
    long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis;
    boolean setOverallStartTime = false;
    long lastDuration = 0L;
    long lastNumEntries = 0L;
    long lastNumErrors = 0L;
    long lastNumSearches = 0L;
    long lastEndTime = System.nanoTime();
    for (long i = 0; i < totalIntervals; i++) {
        if (rateAdjustor != null) {
            if (!rateAdjustor.isAlive()) {
                out("All of the rates in " + variableRateData.getValue().getName() + " have been completed.");
                break;
            }
        }
        final long startTimeMillis = System.currentTimeMillis();
        final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis;
        nextIntervalStartTime += intervalMillis;
        if (sleepTimeMillis > 0) {
            sleeper.sleep(sleepTimeMillis);
        }
        if (stopRequested.get()) {
            break;
        }
        final long endTime = System.nanoTime();
        final long intervalDuration = endTime - lastEndTime;
        final long numSearches;
        final long numEntries;
        final long numErrors;
        final long totalDuration;
        if (warmUp && (remainingWarmUpIntervals > 0)) {
            numSearches = searchCounter.getAndSet(0L);
            numEntries = entryCounter.getAndSet(0L);
            numErrors = errorCounter.getAndSet(0L);
            totalDuration = searchDurations.getAndSet(0L);
        } else {
            numSearches = searchCounter.get();
            numEntries = entryCounter.get();
            numErrors = errorCounter.get();
            totalDuration = searchDurations.get();
        }
        final long recentNumSearches = numSearches - lastNumSearches;
        final long recentNumEntries = numEntries - lastNumEntries;
        final long recentNumErrors = numErrors - lastNumErrors;
        final long recentDuration = totalDuration - lastDuration;
        final double numSeconds = intervalDuration / 1_000_000_000.0d;
        final double recentSearchRate = recentNumSearches / numSeconds;
        final double recentErrorRate = recentNumErrors / numSeconds;
        final double recentAvgDuration;
        final double recentEntriesPerSearch;
        if (recentNumSearches > 0L) {
            recentEntriesPerSearch = 1.0d * recentNumEntries / recentNumSearches;
            recentAvgDuration = 1.0d * recentDuration / recentNumSearches / 1_000_000;
        } else {
            recentEntriesPerSearch = 0.0d;
            recentAvgDuration = 0.0d;
        }
        if (warmUp && (remainingWarmUpIntervals > 0)) {
            out(formatter.formatRow(recentSearchRate, recentAvgDuration, recentEntriesPerSearch, recentErrorRate, "warming up", "warming up"));
            remainingWarmUpIntervals--;
            if (remainingWarmUpIntervals == 0) {
                out("Warm-up completed.  Beginning overall statistics collection.");
                setOverallStartTime = true;
                if (rateAdjustor != null) {
                    rateAdjustor.start();
                }
            }
        } else {
            if (setOverallStartTime) {
                overallStartTime = lastEndTime;
                setOverallStartTime = false;
            }
            final double numOverallSeconds = (endTime - overallStartTime) / 1_000_000_000.0d;
            final double overallSearchRate = numSearches / numOverallSeconds;
            final double overallAvgDuration;
            if (numSearches > 0L) {
                overallAvgDuration = 1.0d * totalDuration / numSearches / 1_000_000;
            } else {
                overallAvgDuration = 0.0d;
            }
            out(formatter.formatRow(recentSearchRate, recentAvgDuration, recentEntriesPerSearch, recentErrorRate, overallSearchRate, overallAvgDuration));
            lastNumSearches = numSearches;
            lastNumEntries = numEntries;
            lastNumErrors = numErrors;
            lastDuration = totalDuration;
        }
        final List<ObjectPair<ResultCode, Long>> rcCounts = rcCounter.getCounts(true);
        if ((!suppressErrors.isPresent()) && (!rcCounts.isEmpty())) {
            err("\tError Results:");
            for (final ObjectPair<ResultCode, Long> p : rcCounts) {
                err("\t", p.getFirst().getName(), ":  ", p.getSecond());
            }
        }
        lastEndTime = endTime;
    }
    // Shut down the RateAdjustor if we have one.
    if (rateAdjustor != null) {
        rateAdjustor.shutDown();
    }
    // Stop all of the threads.
    ResultCode resultCode = ResultCode.SUCCESS;
    for (final SearchRateThread t : threads) {
        t.signalShutdown();
    }
    for (final SearchRateThread t : threads) {
        final ResultCode r = t.waitForShutdown();
        if (resultCode == ResultCode.SUCCESS) {
            resultCode = r;
        }
    }
    return resultCode;
}