Examples with TraceStopwatch datawave.util.time.TraceStopwatch used on opensource projects

Example 1 with TraceStopwatch

use of datawave.util.time.TraceStopwatch in project datawave by NationalSecurityAgency.

the class EventMapper method map.

public void map(K1 key, V1 value, Context context) throws IOException, InterruptedException {
    TraceStopwatch eventMapperTimer = null;
    if (metricsEnabled) {
        eventMapperTimer = new TraceStopwatch("Time in EventMapper");
        eventMapperTimer.start();
    }
    // ensure this datatype's handlers etc are loaded such that the dataTypeDiscardIntervalCache and validators are filled as well
    List<DataTypeHandler<K1>> typeHandlers = loadDataType(value.getDataType().typeName(), context);
    // This is a little bit fragile, but there is no other way
    // to get the context on a partitioner, and we are only
    // using this to set some counters that collect stats.
    MultiTableRangePartitioner.setContext(context);
    Long myInterval = dataTypeDiscardIntervalCache.get(value.getDataType().typeName());
    // setup the configuration on the event
    // this is automatically done by the sequence reader....
    // value.setConf(context.getConfiguration());
    // Flag to control whether a reprocessed event caused an NDC.push
    boolean reprocessedNDCPush = false;
    byte[] rawData = value.getRawData();
    if (rawData != null) {
        long rawDataBytes = rawData.length;
        getCounter(context, IngestInput.LINE_BYTES.toString(), "TOTAL").increment(rawDataBytes);
        long minBytes = getCounter(context, IngestInput.LINE_BYTES.toString(), "MIN").getValue();
        if (rawDataBytes < minBytes) {
            getCounter(context, IngestInput.LINE_BYTES.toString(), "MIN").setValue(rawDataBytes);
        }
        long maxBytes = getCounter(context, IngestInput.LINE_BYTES.toString(), "MAX").getValue();
        if (rawDataBytes > maxBytes) {
            getCounter(context, IngestInput.LINE_BYTES.toString(), "MAX").setValue(rawDataBytes);
        }
    }
    // First lets clear this event from the error table if we are reprocessing a previously errored event
    if (value.getAuxData() instanceof EventErrorSummary) {
        EventErrorSummary errorSummary = (EventErrorSummary) (value.getAuxData());
        value.setAuxData(null);
        // pass the processedCount through via the aux properties
        value.setAuxProperty(ErrorDataTypeHandler.PROCESSED_COUNT, Integer.toString(errorSummary.getProcessedCount() + 1));
        // delete these keys from the error table. If this fails then nothing will have changed
        if (log.isInfoEnabled())
            log.info("Purging event from the " + errorSummary.getTableName() + " table");
        try {
            // Load error dataType into typeMap
            loadDataType(TypeRegistry.ERROR_PREFIX, context);
            // purge event
            errorSummary.purge(contextWriter, context, value, typeMap);
            // Set the original file value from the event in the error table
            Collection<String> origFiles = errorSummary.getEventFields().get(SEQUENCE_FILE_FIELDNAME);
            if (!origFiles.isEmpty()) {
                NDC.push(origFiles.iterator().next());
                reprocessedNDCPush = true;
            }
        } catch (Exception e) {
            contextWriter.rollback();
            log.error("Failed to clean event from error table.  Terminating map", e);
            throw new IOException("Failed to clean event from error table, Terminating map", e);
        } finally {
            contextWriter.commit(context);
            context.progress();
        }
    } else {
        // pass the processedCount through via the aux properties
        value.setAuxProperty(ErrorDataTypeHandler.PROCESSED_COUNT, "1");
    }
    // Determine whether the event date is greater than the interval. Excluding fatal error events.
    if (!value.fatalError() && null != myInterval && 0L != myInterval && (value.getDate() < (now.get() - myInterval))) {
        if (log.isInfoEnabled())
            log.info("Event with time " + value.getDate() + " older than specified interval of " + (now.get() - myInterval) + ", skipping...");
        getCounter(context, IngestInput.OLD_EVENT).increment(1);
        return;
    }
    // Add the list of handlers with the ALL specified handlers
    List<DataTypeHandler<K1>> handlers = new ArrayList<>();
    handlers.addAll(typeHandlers);
    handlers.addAll(loadDataType(TypeRegistry.ALL_PREFIX, context));
    // Always include any event errors in the counters
    for (String error : value.getErrors()) {
        getCounter(context, IngestInput.EVENT_ERROR_TYPE.name(), error).increment(1);
    }
    // switch over to the errorHandlerList if still a fatal error
    if (value.fatalError()) {
        // now clear out the handlers to avoid processing this event
        handlers.clear();
        if (!value.ignorableError()) {
            // since this is not an ignorable error, lets add the error handlers back into the list
            handlers.addAll(loadDataType(TypeRegistry.ERROR_PREFIX, context));
            getCounter(context, IngestInput.EVENT_FATAL_ERROR).increment(1);
            getCounter(context, IngestInput.EVENT_FATAL_ERROR.name(), "ValidationError").increment(1);
        } else {
            getCounter(context, IngestInput.EVENT_IGNORABLE_ERROR).increment(1);
            getCounter(context, IngestInput.EVENT_IGNORABLE_ERROR.name(), "IgnorableError").increment(1);
        }
        context.progress();
    }
    Multimap<String, NormalizedContentInterface> fields = HashMultimap.create();
    try {
        processEvent(key, value, handlers, fields, context);
    } catch (Exception e) {
        // Rollback anything written for this event
        contextWriter.rollback();
        // Fail job on constraint violations
        if (e instanceof ConstraintChecker.ConstraintViolationException) {
            throw ((RuntimeException) e);
        }
        // ensure they know we are still working on it
        context.progress();
        // log error
        log.error("Runtime exception processing event", e);
        // first set the exception on the event if not a field normalization error in which case the fields contain the errors
        if (!(e instanceof FieldNormalizationError)) {
            value.setAuxData(e);
        }
        for (DataTypeHandler<K1> handler : loadDataType(TypeRegistry.ERROR_PREFIX, context)) {
            if (log.isTraceEnabled())
                log.trace("executing handler: " + handler.getClass().getName());
            try {
                executeHandler(key, value, fields, handler, context);
                context.progress();
            } catch (Exception e2) {
                // This is a real bummer, we had a critical exception attempting to throw the event into the error table.
                // lets terminate this job
                log.error("Failed to process error data handlers for an event", e2);
                throw new IOException("Failed to process error data handlers for an event", e2);
            }
        }
        // now create some counters
        getCounter(context, IngestProcess.RUNTIME_EXCEPTION).increment(1);
        List<String> exceptions = getExceptionSynopsis(e);
        for (String exception : exceptions) {
            getCounter(context, IngestProcess.RUNTIME_EXCEPTION.name(), exception).increment(1);
        }
    } finally {
        // Remove ORIG_FILE from NDC that was populated by reprocessing events from the error tables
        if (reprocessedNDCPush) {
            NDC.pop();
        }
        // cleanup the context writer
        contextWriter.commit(context);
        context.progress();
    }
    getCounter(context, IngestOutput.EVENTS_PROCESSED.name(), value.getDataType().typeName().toUpperCase()).increment(1);
    offset++;
    if (metricsEnabled && eventMapperTimer != null) {
        eventMapperTimer.stop();
        long timeInEventMapper = eventMapperTimer.elapsed(TimeUnit.MILLISECONDS);
        metricsLabels.clear();
        metricsLabels.put("dataType", value.getDataType().typeName());
        metricsService.collect(Metric.MILLIS_IN_EVENT_MAPPER, metricsLabels.get(), fields, timeInEventMapper);
    }
}

Example 2 with TraceStopwatch

use of datawave.util.time.TraceStopwatch in project datawave by NationalSecurityAgency.

the class EventMapper method executeHandler.

@SuppressWarnings("unchecked")
public void executeHandler(K1 key, RawRecordContainer event, Multimap<String, NormalizedContentInterface> fields, DataTypeHandler<K1> handler, Context context) throws Exception {
    long count = 0;
    TraceStopwatch handlerTimer = null;
    // Handler based metrics
    if (metricsEnabled) {
        handlerTimer = new TraceStopwatch("Time in handler");
        handlerTimer.start();
    }
    // In the setup we determined whether or not we were performing bulk ingest. This tells us which
    // method to call on the DataTypeHandler interface.
    Multimap<BulkIngestKey, Value> r;
    if (!(handler instanceof ExtendedDataTypeHandler)) {
        r = handler.processBulk(key, event, fields, new ContextWrappedStatusReporter(getContext(context)));
        if (r == null) {
            getCounter(context, IngestInput.EVENT_FATAL_ERROR).increment(1);
            getCounter(context, IngestInput.EVENT_FATAL_ERROR.name(), "NullMultiMap").increment(1);
        } else {
            contextWriter.write(r, context);
            count = r.size();
        }
    } else {
        count = ((ExtendedDataTypeHandler<K1, K2, V2>) handler).process(key, event, fields, context, contextWriter);
        if (count == -1) {
            getCounter(context, IngestInput.EVENT_FATAL_ERROR).increment(1);
            getCounter(context, IngestInput.EVENT_FATAL_ERROR.name(), "NegOneCount").increment(1);
        }
    }
    // Update the counters
    if (count > 0) {
        getCounter(context, IngestOutput.ROWS_CREATED.name(), handler.getClass().getSimpleName()).increment(count);
        getCounter(context, IngestOutput.ROWS_CREATED).increment(count);
    }
    if (handler.getMetadata() != null) {
        handler.getMetadata().addEvent(handler.getHelper(event.getDataType()), event, fields, now.get());
    }
    if (metricsEnabled && handlerTimer != null) {
        handlerTimer.stop();
        long handlerTime = handlerTimer.elapsed(TimeUnit.MILLISECONDS);
        metricsLabels.clear();
        metricsLabels.put("dataType", event.getDataType().typeName());
        metricsLabels.put("handler", handler.getClass().getName());
        metricsService.collect(Metric.MILLIS_IN_HANDLER, metricsLabels.get(), fields, handlerTime);
        if (contextWriter instanceof KeyValueCountingContextWriter) {
            ((KeyValueCountingContextWriter) contextWriter).writeMetrics(event, fields, handler);
        }
    }
}

Example 3 with TraceStopwatch

use of datawave.util.time.TraceStopwatch in project datawave by NationalSecurityAgency.

the class QueryStopwatch method newStopwatch.

/**
 * Creates a new Stopwatch for use but does not start it
 *
 * @return
 */
private TraceStopwatch newStopwatch(String header) {
    checkNotNull(header);
    TraceStopwatch sw = new TraceStopwatch(header);
    watches.add(Maps.immutableEntry(header, sw));
    return sw;
}

Example 4 with TraceStopwatch

use of datawave.util.time.TraceStopwatch in project datawave by NationalSecurityAgency.

the class ShardQueryLogic method setupQuery.

@Override
public void setupQuery(GenericQueryConfiguration genericConfig) throws Exception {
    if (!ShardQueryConfiguration.class.isAssignableFrom(genericConfig.getClass())) {
        throw new QueryException("Did not receive a ShardQueryConfiguration instance!!");
    }
    ShardQueryConfiguration config = (ShardQueryConfiguration) genericConfig;
    final QueryStopwatch timers = config.getTimers();
    TraceStopwatch stopwatch = timers.newStartedStopwatch("ShardQueryLogic - Setup Query");
    // Ensure we have all of the information needed to run a query
    if (!config.canRunQuery()) {
        log.warn("The given query '" + config + "' could not be run, most likely due to not matching any records in the global index.");
        // Stub out an iterator to correctly present "no results"
        this.iterator = new Iterator<Map.Entry<Key, Value>>() {

            @Override
            public boolean hasNext() {
                return false;
            }

            @Override
            public Map.Entry<Key, Value> next() {
                return null;
            }

            @Override
            public void remove() {
                return;
            }
        };
        this.scanner = null;
        stopwatch.stop();
        log.info(getStopwatchHeader(config));
        List<String> timings = timers.summarizeAsList();
        for (String timing : timings) {
            log.info(timing);
        }
        return;
    }
    // Instantiate the scheduler for the queries
    this.scheduler = getScheduler(config, scannerFactory);
    this.scanner = null;
    this.iterator = this.scheduler.iterator();
    if (!config.isSortedUIDs()) {
        this.iterator = new DedupingIterator(this.iterator);
    }
    stopwatch.stop();
    log.info(getStopwatchHeader(config));
    List<String> timings = timers.summarizeAsList();
    for (String timing : timings) {
        log.info(timing);
    }
}

Example 5 with TraceStopwatch

use of datawave.util.time.TraceStopwatch in project datawave by NationalSecurityAgency.

the class ShardQueryLogic method loadQueryParameters.

protected void loadQueryParameters(ShardQueryConfiguration config, Query settings) throws QueryException {
    TraceStopwatch stopwatch = config.getTimers().newStartedStopwatch("ShardQueryLogic - Parse query parameters");
    boolean rawDataOnly = false;
    String rawDataOnlyStr = settings.findParameter(QueryParameters.RAW_DATA_ONLY).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(rawDataOnlyStr)) {
        rawDataOnly = Boolean.valueOf(rawDataOnlyStr);
        // note that if any of these other options are set, then it overrides the settings here
        if (rawDataOnly) {
            // set the grouping context to trye to ensure we get the full field names
            this.setIncludeGroupingContext(true);
            config.setIncludeGroupingContext(true);
            // set the hierarchy fields to false as they are generated fields
            this.setIncludeHierarchyFields(false);
            config.setIncludeHierarchyFields(false);
            // set the datatype field to false as it is a generated field
            this.setIncludeDataTypeAsField(false);
            config.setIncludeDataTypeAsField(false);
            // do not include the record id
            this.setIncludeRecordId(false);
            config.setIncludeRecordId(false);
            // set the hit list to false as it is a generated field
            this.setHitList(false);
            config.setHitList(false);
            // set the raw types to true to avoid any type transformations of the values
            config.setRawTypes(true);
            // do not filter masked values
            this.setFilterMaskedValues(false);
            config.setFilterMaskedValues(false);
            // do not reduce the response
            this.setReducedResponse(false);
            config.setReducedResponse(false);
            // clear the content field names to prevent content field transformations (see DocumentTransformer)
            this.setContentFieldNames(Collections.EMPTY_LIST);
            // clear the model name to avoid field name translations
            this.setModelName(null);
            config.setModelName(null);
        }
    }
    // Get the datatype set if specified
    String typeList = settings.findParameter(QueryParameters.DATATYPE_FILTER_SET).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(typeList)) {
        HashSet<String> typeFilter = new HashSet<>();
        typeFilter.addAll(Arrays.asList(StringUtils.split(typeList, Constants.PARAM_VALUE_SEP)));
        if (log.isDebugEnabled()) {
            log.debug("Type Filter: " + typeFilter);
        }
        config.setDatatypeFilter(typeFilter);
    }
    // Get the list of fields to project up the stack. May be null.
    String projectFields = settings.findParameter(QueryParameters.RETURN_FIELDS).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(projectFields)) {
        List<String> projectFieldsList = Arrays.asList(StringUtils.split(projectFields, Constants.PARAM_VALUE_SEP));
        // Only set the projection fields if we were actually given some
        if (!projectFieldsList.isEmpty()) {
            config.setProjectFields(new HashSet<>(projectFieldsList));
            if (log.isDebugEnabled()) {
                final int maxLen = 100;
                // Trim down the projection if it's stupid long
                projectFields = maxLen < projectFields.length() ? projectFields.substring(0, maxLen) + "[TRUNCATED]" : projectFields;
                log.debug("Projection fields: " + projectFields);
            }
        }
    }
    // if the TRANFORM_CONTENT_TO_UID is false, then unset the list of content field names preventing the DocumentTransformer from
    // transforming them.
    String transformContentStr = settings.findParameter(QueryParameters.TRANFORM_CONTENT_TO_UID).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(transformContentStr)) {
        if (!Boolean.valueOf(transformContentStr)) {
            setContentFieldNames(Collections.EMPTY_LIST);
        }
    }
    // Get the list of blacklisted fields. May be null.
    String tBlacklistedFields = settings.findParameter(QueryParameters.BLACKLISTED_FIELDS).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(tBlacklistedFields)) {
        List<String> blacklistedFieldsList = Arrays.asList(StringUtils.split(tBlacklistedFields, Constants.PARAM_VALUE_SEP));
        // Only set the blacklisted fields if we were actually given some
        if (!blacklistedFieldsList.isEmpty()) {
            if (!config.getProjectFields().isEmpty()) {
                throw new QueryException("Whitelist and blacklist projection options are mutually exclusive");
            }
            config.setBlacklistedFields(new HashSet<>(blacklistedFieldsList));
            if (log.isDebugEnabled()) {
                log.debug("Blacklisted fields: " + tBlacklistedFields);
            }
        }
    }
    // Get the LIMIT_FIELDS parameter if given
    String limitFields = settings.findParameter(QueryParameters.LIMIT_FIELDS).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(limitFields)) {
        List<String> limitFieldsList = Arrays.asList(StringUtils.split(limitFields, Constants.PARAM_VALUE_SEP));
        // Only set the limit fields if we were actually given some
        if (!limitFieldsList.isEmpty()) {
            config.setLimitFields(new HashSet<>(limitFieldsList));
        }
    }
    String limitFieldsPreQueryEvaluation = settings.findParameter(QueryOptions.LIMIT_FIELDS_PRE_QUERY_EVALUATION).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(limitFieldsPreQueryEvaluation)) {
        Boolean limitFieldsPreQueryEvaluationValue = Boolean.parseBoolean(limitFieldsPreQueryEvaluation);
        this.setLimitFieldsPreQueryEvaluation(limitFieldsPreQueryEvaluationValue);
        config.setLimitFieldsPreQueryEvaluation(limitFieldsPreQueryEvaluationValue);
    }
    String limitFieldsField = settings.findParameter(QueryOptions.LIMIT_FIELDS_FIELD).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(limitFieldsField)) {
        this.setLimitFieldsField(limitFieldsField);
        config.setLimitFieldsField(limitFieldsField);
    }
    // Get the GROUP_FIELDS parameter if given
    String groupFields = settings.findParameter(QueryParameters.GROUP_FIELDS).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(groupFields)) {
        List<String> groupFieldsList = Arrays.asList(StringUtils.split(groupFields, Constants.PARAM_VALUE_SEP));
        // Only set the group fields if we were actually given some
        if (!groupFieldsList.isEmpty()) {
            this.setGroupFields(new HashSet<>(groupFieldsList));
            config.setGroupFields(new HashSet<>(groupFieldsList));
            config.setProjectFields(new HashSet<>(groupFieldsList));
        }
    }
    String groupFieldsBatchSizeString = settings.findParameter(QueryParameters.GROUP_FIELDS_BATCH_SIZE).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(groupFieldsBatchSizeString)) {
        int groupFieldsBatchSize = Integer.parseInt(groupFieldsBatchSizeString);
        this.setGroupFieldsBatchSize(groupFieldsBatchSize);
        config.setGroupFieldsBatchSize(groupFieldsBatchSize);
    }
    // Get the UNIQUE_FIELDS parameter if given
    String uniqueFieldsParam = settings.findParameter(QueryParameters.UNIQUE_FIELDS).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(uniqueFieldsParam)) {
        UniqueFields uniqueFields = UniqueFields.from(uniqueFieldsParam);
        // Only set the unique fields if we were actually given some
        if (!uniqueFields.isEmpty()) {
            this.setUniqueFields(uniqueFields);
            config.setUniqueFields(uniqueFields);
        }
    }
    // Get the HIT_LIST parameter if given
    String hitListString = settings.findParameter(QueryParameters.HIT_LIST).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(hitListString)) {
        Boolean hitListBool = Boolean.parseBoolean(hitListString);
        config.setHitList(hitListBool);
    }
    // Get the BYPASS_ACCUMULO parameter if given
    String bypassAccumuloString = settings.findParameter(BYPASS_ACCUMULO).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(bypassAccumuloString)) {
        Boolean bypassAccumuloBool = Boolean.parseBoolean(bypassAccumuloString);
        config.setBypassAccumulo(bypassAccumuloBool);
    }
    // Get the DATE_INDEX_TIME_TRAVEL parameter if given
    String dateIndexTimeTravelString = settings.findParameter(QueryOptions.DATE_INDEX_TIME_TRAVEL).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(dateIndexTimeTravelString)) {
        Boolean dateIndexTimeTravel = Boolean.parseBoolean(dateIndexTimeTravelString);
        config.setDateIndexTimeTravel(dateIndexTimeTravel);
    }
    // get the RAW_TYPES parameter if given
    String rawTypesString = settings.findParameter(QueryParameters.RAW_TYPES).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(rawTypesString)) {
        Boolean rawTypesBool = Boolean.parseBoolean(rawTypesString);
        config.setRawTypes(rawTypesBool);
    }
    // Get the FILTER_MASKED_VALUES spring setting
    String filterMaskedValuesStr = settings.findParameter(QueryParameters.FILTER_MASKED_VALUES).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(filterMaskedValuesStr)) {
        Boolean filterMaskedValuesBool = Boolean.parseBoolean(filterMaskedValuesStr);
        this.setFilterMaskedValues(filterMaskedValuesBool);
        config.setFilterMaskedValues(filterMaskedValuesBool);
    }
    // Get the INCLUDE_DATATYPE_AS_FIELD spring setting
    String includeDatatypeAsFieldStr = settings.findParameter(QueryParameters.INCLUDE_DATATYPE_AS_FIELD).getParameterValue().trim();
    if (((org.apache.commons.lang.StringUtils.isNotBlank(includeDatatypeAsFieldStr) && Boolean.valueOf(includeDatatypeAsFieldStr))) || (this.getIncludeDataTypeAsField() && !rawDataOnly)) {
        config.setIncludeDataTypeAsField(true);
    }
    // Get the INCLUDE_RECORD_ID spring setting
    String includeRecordIdStr = settings.findParameter(QueryParameters.INCLUDE_RECORD_ID).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(includeRecordIdStr)) {
        boolean includeRecordIdBool = Boolean.parseBoolean(includeRecordIdStr) && !rawDataOnly;
        this.setIncludeRecordId(includeRecordIdBool);
        config.setIncludeRecordId(includeRecordIdBool);
    }
    // Get the INCLUDE_HIERARCHY_FIELDS spring setting
    String includeHierarchyFieldsStr = settings.findParameter(QueryParameters.INCLUDE_HIERARCHY_FIELDS).getParameterValue().trim();
    if (((org.apache.commons.lang.StringUtils.isNotBlank(includeHierarchyFieldsStr) && Boolean.valueOf(includeHierarchyFieldsStr))) || (this.getIncludeHierarchyFields() && !rawDataOnly)) {
        config.setIncludeHierarchyFields(true);
        final Map<String, String> options = this.getHierarchyFieldOptions();
        config.setHierarchyFieldOptions(options);
    }
    // Get the query profile to allow us to select the tune profile of the query
    String queryProfile = settings.findParameter(QueryParameters.QUERY_PROFILE).getParameterValue().trim();
    if ((org.apache.commons.lang.StringUtils.isNotBlank(queryProfile))) {
        selectedProfile = configuredProfiles.get(queryProfile);
        if (null == selectedProfile) {
            throw new QueryException(QueryParameters.QUERY_PROFILE + " has been specified but " + queryProfile + " is not a selectable profile");
        }
    }
    // Get the include.grouping.context = true/false spring setting
    String includeGroupingContextStr = settings.findParameter(QueryParameters.INCLUDE_GROUPING_CONTEXT).getParameterValue().trim();
    if (((org.apache.commons.lang.StringUtils.isNotBlank(includeGroupingContextStr) && Boolean.valueOf(includeGroupingContextStr))) || (this.getIncludeGroupingContext() && !rawDataOnly)) {
        config.setIncludeGroupingContext(true);
    }
    // Check if the default modelName and modelTableNames have been overridden by custom parameters.
    String parameterModelName = settings.findParameter(QueryParameters.PARAMETER_MODEL_NAME).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(parameterModelName)) {
        this.setModelName(parameterModelName);
    }
    config.setModelName(this.getModelName());
    String parameterModelTableName = settings.findParameter(QueryParameters.PARAMETER_MODEL_TABLE_NAME).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(parameterModelTableName)) {
        this.setModelTableName(parameterModelTableName);
    }
    if (null != config.getModelName() && null == config.getModelTableName()) {
        throw new IllegalArgumentException(QueryParameters.PARAMETER_MODEL_NAME + " has been specified but " + QueryParameters.PARAMETER_MODEL_TABLE_NAME + " is missing. Both are required to use a model");
    }
    configureDocumentAggregation(settings);
    config.setLimitTermExpansionToModel(this.isExpansionLimitedToModelContents());
    String reducedResponseStr = settings.findParameter(QueryOptions.REDUCED_RESPONSE).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(reducedResponseStr)) {
        Boolean reducedResponseValue = Boolean.parseBoolean(reducedResponseStr);
        this.setReducedResponse(reducedResponseValue);
        config.setReducedResponse(reducedResponseValue);
    }
    final String postProcessingClasses = settings.findParameter(QueryOptions.POSTPROCESSING_CLASSES).getParameterValue().trim();
    final String postProcessingOptions = settings.findParameter(QueryOptions.POSTPROCESSING_OPTIONS).getParameterValue().trim();
    // build the post p
    if (org.apache.commons.lang.StringUtils.isNotBlank(postProcessingClasses)) {
        List<String> filterClasses = config.getFilterClassNames();
        if (null == filterClasses) {
            filterClasses = new ArrayList<>();
        }
        for (String fClassName : StringUtils.splitIterable(postProcessingClasses, ',', true)) {
            filterClasses.add(fClassName);
        }
        config.setFilterClassNames(filterClasses);
        final Map<String, String> options = this.getFilterOptions();
        if (null != options) {
            config.putFilterOptions(options);
        }
        if (org.apache.commons.lang.StringUtils.isNotBlank(postProcessingOptions)) {
            for (String filterOptionStr : StringUtils.splitIterable(postProcessingOptions, ',', true)) {
                if (org.apache.commons.lang.StringUtils.isNotBlank(filterOptionStr)) {
                    final String filterValueString = settings.findParameter(filterOptionStr).getParameterValue().trim();
                    if (org.apache.commons.lang.StringUtils.isNotBlank(filterValueString)) {
                        config.putFilterOptions(filterOptionStr, filterValueString);
                    }
                }
            }
        }
    }
    String tCompressServerSideResults = settings.findParameter(QueryOptions.COMPRESS_SERVER_SIDE_RESULTS).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(tCompressServerSideResults)) {
        boolean compress = Boolean.parseBoolean(tCompressServerSideResults);
        config.setCompressServerSideResults(compress);
    }
    // Configure index-only filter functions to be enabled if not already set to such a state
    config.setIndexOnlyFilterFunctionsEnabled(this.isIndexOnlyFilterFunctionsEnabled());
    // Set the ReturnType for Documents coming out of the iterator stack
    config.setReturnType(DocumentSerialization.getReturnType(settings));
    QueryLogicTransformer transformer = getTransformer(settings);
    if (transformer instanceof WritesQueryMetrics) {
        String logTimingDetailsStr = settings.findParameter(QueryOptions.LOG_TIMING_DETAILS).getParameterValue().trim();
        if (org.apache.commons.lang.StringUtils.isNotBlank(logTimingDetailsStr)) {
            setLogTimingDetails(Boolean.valueOf(logTimingDetailsStr));
        }
        if (getLogTimingDetails()) {
            // we have to collect the timing details on the iterator stack in order to log them
            setCollectTimingDetails(true);
        } else {
            String collectTimingDetailsStr = settings.findParameter(QueryOptions.COLLECT_TIMING_DETAILS).getParameterValue().trim();
            if (org.apache.commons.lang.StringUtils.isNotBlank(collectTimingDetailsStr)) {
                setCollectTimingDetails(Boolean.valueOf(collectTimingDetailsStr));
            }
        }
    } else {
        // if the transformer can not process the timing metrics, then turn them off
        setLogTimingDetails(false);
        setCollectTimingDetails(false);
    }
    stopwatch.stop();
    if (null != selectedProfile) {
        selectedProfile.configure(this);
        selectedProfile.configure(config);
        selectedProfile.configure(planner);
    }
}