Examples with VisitationContext datawave.query.tables.edge.contexts.VisitationContext used on opensource projects

Example 1 with VisitationContext

use of datawave.query.tables.edge.contexts.VisitationContext in project datawave by NationalSecurityAgency.

the class DefaultExtendedEdgeQueryLogic method setupQuery.

@Override
public void setupQuery(GenericQueryConfiguration configuration) throws Exception {
    EdgeExtendedSummaryConfiguration localConf = (EdgeExtendedSummaryConfiguration) configuration;
    config = (EdgeExtendedSummaryConfiguration) configuration;
    prefilterValues = null;
    EdgeExtendedSummaryConfiguration.dateType dateFilterType = localConf.getDateRangeType();
    if (log.isTraceEnabled()) {
        log.trace("Performing edge table query: " + config.getQueryString());
    }
    // TODO check to see if overriding I/O necessary
    if (allowOverrideIO && localConf.isOverRideInput()) {
        this.summaryInputType = localConf.isSummaryInputType();
    }
    if (allowOverrideIO && localConf.isOverRideOutput()) {
        this.summaryOutputType = localConf.isAggregateResults();
    }
    boolean includeStats = localConf.includeStats();
    String queryString = config.getQueryString();
    MetadataHelper metadataHelper = super.prepareMetadataHelper(config.getConnector(), config.getModelTableName(), config.getAuthorizations());
    loadQueryModel(metadataHelper, config);
    // list of source values...no field names to translate
    if (this.summaryInputType == false) {
        queryString = applyQueryModel(queryString);
    }
    // set the modified queryString back into the config, for easy access
    config.setQueryString(queryString);
    String normalizedQuery = "";
    String statsNormalizedQuery = "";
    QueryData qData = configureRanges(queryString);
    setRanges(qData.getRanges());
    VisitationContext context = null;
    if (this.summaryInputType == false) {
        try {
            context = normalizeJexlQuery(queryString, false);
            normalizedQuery = context.getNormalizedQuery().toString();
            statsNormalizedQuery = context.getNormalizedStatsQuery().toString();
            if (log.isTraceEnabled()) {
                log.trace("Jexl after normalizing both vertices: " + normalizedQuery);
            }
        } catch (JexlException ex) {
            try {
                log.error("Error parsing user query.", ex);
            } catch (Exception ex2) {
                log.error("Exception thrown by logger (???)");
            }
        }
    }
    if ((null == normalizedQuery || normalizedQuery.equals("")) && qData.getRanges().size() < 1) {
        throw new IllegalStateException("Query string is empty after initial processing, no ranges or filters can be generated to execute.");
    }
    addIterators(qData, getDateBasedIterators(config.getBeginDate(), config.getEndDate(), currentIteratorPriority, dateFilterSkipLimit, dateFilterScanLimit, dateFilterType));
    if (!normalizedQuery.equals("")) {
        if (log.isTraceEnabled()) {
            log.trace("Query being sent to the filter iterator: " + normalizedQuery);
        }
        IteratorSetting edgeIteratorSetting = new IteratorSetting(currentIteratorPriority, EdgeFilterIterator.class.getSimpleName() + "_" + currentIteratorPriority, EdgeFilterIterator.class);
        edgeIteratorSetting.addOption(EdgeFilterIterator.JEXL_OPTION, normalizedQuery);
        edgeIteratorSetting.addOption(EdgeFilterIterator.PROTOBUF_OPTION, "TRUE");
        if (!statsNormalizedQuery.equals("")) {
            edgeIteratorSetting.addOption(EdgeFilterIterator.JEXL_STATS_OPTION, statsNormalizedQuery);
        }
        if (prefilterValues != null) {
            String value = serializePrefilter();
            edgeIteratorSetting.addOption(EdgeFilterIterator.PREFILTER_WHITELIST, value);
        }
        if (includeStats) {
            edgeIteratorSetting.addOption(EdgeFilterIterator.INCLUDE_STATS_OPTION, "TRUE");
        } else {
            edgeIteratorSetting.addOption(EdgeFilterIterator.INCLUDE_STATS_OPTION, "FALSE");
        }
        addIterator(qData, edgeIteratorSetting);
    }
    if (log.isTraceEnabled()) {
        log.trace("Configuring connection: tableName: " + config.getTableName() + ", auths: " + config.getAuthorizations());
    }
    BatchScanner scanner = createBatchScanner(config);
    if (log.isTraceEnabled()) {
        log.trace("Using the following ranges: " + qData.getRanges());
    }
    if (context != null && context.isHasAllCompleteColumnFamilies()) {
        for (Text columnFamily : context.getColumnFamilies()) {
            scanner.fetchColumnFamily(columnFamily);
        }
    }
    scanner.setRanges(qData.getRanges());
    addCustomFilters(qData, currentIteratorPriority);
    for (IteratorSetting setting : qData.getSettings()) {
        scanner.addScanIterator(setting);
    }
    this.scanner = scanner;
    iterator = scanner.iterator();
}

Example 2 with VisitationContext

use of datawave.query.tables.edge.contexts.VisitationContext in project datawave by NationalSecurityAgency.

the class EdgeTableRangeBuildingVisitor method computeVisitaionContext.

/**
 * This method creates a new VisitationContext object to be returned and loads the final list of queryContexts which are then used to build the ranges and
 * normalized query One of the main purposes of this method is to create the normalized query that is used to filter column families from ranges. This is a
 * problem when there are multiple query contexts because the whitelist will exclude certain column family values, which will affect what gets returned by
 * the query. This is addressed by the columnFamilyAreDifferent boolean which is passed down to populateQuery()
 */
private VisitationContext computeVisitaionContext(List<QueryContext> queryContexts) {
    // if both edge types and edge relationships are complete (not regex) for each query context then we can use the
    // batch scanners built in fetch column method to do the filtering for us so we can drop the edge types, and relations
    // from the normalized query that goes to the edge filter iterator.
    boolean includColumnFamilyTerms = false;
    boolean columnFamilyAreDifferent = false;
    // If the sink field appears in a query contexts, 'otherContext' list then it will have to be included in the normalized
    // query string sent to the edge filter iterator
    boolean includeSink = false;
    boolean includeSource = false;
    for (QueryContext queryContext : queryContexts) {
        if (queryContext.hasCompleteColumnFamily() == false) {
            includColumnFamilyTerms = true;
            break;
        }
    }
    if (queryContexts.size() > 1) {
        int i;
        QueryContext.ColumnContext firstColumn = (queryContexts.get(0).getColumnContext());
        for (i = 1; i < queryContexts.size(); i++) {
            QueryContext.ColumnContext currentContext = queryContexts.get(i).getColumnContext();
            if ((firstColumn != null && currentContext == null) || (firstColumn == null && currentContext != null)) {
                columnFamilyAreDifferent = true;
                break;
            }
            if (firstColumn != null) {
                if (!(firstColumn.equals(queryContexts.get(i).getColumnContext()))) {
                    columnFamilyAreDifferent = true;
                    break;
                }
            }
        }
    }
    VisitationContext vContext = new VisitationContext(includeStats);
    vContext.setHasAllCompleteColumnFamilies(!includColumnFamilyTerms);
    for (QueryContext qContext : queryContexts) {
        includeSink = vContext.updateQueryRanges(qContext);
        includeSink = includeSink || sawEquivalenceRegexSink || columnFamilyAreDifferent;
        // If there is only one query context you don't need to include the source, you do if there are multiple QCs
        if (queryContexts.size() > 1) {
            // If there are multiple query contexts, that means that there are multiple sources. If there are multiple
            // sources, then they need to be included with sinks.
            includeSource = includeSink || sawEquivalenceRegexSource || columnFamilyAreDifferent;
        } else if (queryContexts.size() == 1) {
            // If there is only one source, you don't need to include it with the sink. It is implied.
            includeSource = sawEquivalenceRegexSource || columnFamilyAreDifferent;
        }
        // boolean for source and sink inclusion for normalized query
        // boolean to include column family terms to the normalized query
        // boolean to create white list for column family terms
        vContext.updateQueryStrings(qContext, includeSource, includeSink, includColumnFamilyTerms, !columnFamilyAreDifferent);
        if (!includColumnFamilyTerms) {
            vContext.buildColumnFamilyList(qContext, includeStats);
        }
    }
    vContext.setTermCount(termCount);
    return vContext;
}

Example 3 with VisitationContext

use of datawave.query.tables.edge.contexts.VisitationContext in project datawave by NationalSecurityAgency.

the class EdgeQueryLogic method setupQuery.

@Override
public void setupQuery(GenericQueryConfiguration configuration) throws Exception {
    config = (EdgeQueryConfiguration) configuration;
    prefilterValues = null;
    EdgeQueryConfiguration.dateType dateFilterType = ((EdgeQueryConfiguration) configuration).getDateRangeType();
    log.debug("Performing edge table query: " + config.getQueryString());
    boolean includeStats = ((EdgeQueryConfiguration) configuration).includeStats();
    String queryString = config.getQueryString();
    String normalizedQuery = null;
    String statsNormalizedQuery = null;
    queryString = fixQueryString(queryString);
    QueryData qData = configureRanges(queryString);
    setRanges(qData.getRanges());
    VisitationContext context = null;
    try {
        context = normalizeJexlQuery(queryString, false);
        normalizedQuery = context.getNormalizedQuery().toString();
        statsNormalizedQuery = context.getNormalizedStatsQuery().toString();
        log.debug("Jexl after normalizing SOURCE and SINK: " + normalizedQuery);
    } catch (JexlException ex) {
        log.error("Error parsing user query.", ex);
    }
    if ((null == normalizedQuery || normalizedQuery.equals("")) && qData.getRanges().size() < 1) {
        throw new IllegalStateException("Query string is empty after initial processing, no ranges or filters can be generated to execute.");
    }
    addIterators(qData, getDateBasedIterators(config.getBeginDate(), config.getEndDate(), currentIteratorPriority, dateFilterSkipLimit, dateFilterScanLimit, dateFilterType));
    if (!normalizedQuery.equals("")) {
        log.debug("Query being sent to the filter iterator: " + normalizedQuery);
        IteratorSetting edgeIteratorSetting = new IteratorSetting(currentIteratorPriority, EdgeFilterIterator.class.getSimpleName() + "_" + currentIteratorPriority, EdgeFilterIterator.class);
        edgeIteratorSetting.addOption(EdgeFilterIterator.JEXL_OPTION, normalizedQuery);
        edgeIteratorSetting.addOption(EdgeFilterIterator.PROTOBUF_OPTION, "TRUE");
        if (!statsNormalizedQuery.equals("")) {
            edgeIteratorSetting.addOption(EdgeFilterIterator.JEXL_STATS_OPTION, statsNormalizedQuery);
        }
        if (prefilterValues != null) {
            String value = serializePrefilter();
            edgeIteratorSetting.addOption(EdgeFilterIterator.PREFILTER_WHITELIST, value);
        }
        if (includeStats) {
            edgeIteratorSetting.addOption(EdgeFilterIterator.INCLUDE_STATS_OPTION, "TRUE");
        } else {
            edgeIteratorSetting.addOption(EdgeFilterIterator.INCLUDE_STATS_OPTION, "FALSE");
        }
        addIterator(qData, edgeIteratorSetting);
    }
    log.debug("Configuring connection: tableName: " + config.getTableName() + ", auths: " + config.getAuthorizations());
    BatchScanner scanner = createBatchScanner(config);
    log.debug("Using the following ranges: " + qData.getRanges());
    if (context != null && context.isHasAllCompleteColumnFamilies()) {
        for (Text columnFamily : context.getColumnFamilies()) {
            scanner.fetchColumnFamily(columnFamily);
        }
    }
    scanner.setRanges(qData.getRanges());
    addCustomFilters(qData, currentIteratorPriority);
    for (IteratorSetting setting : qData.getSettings()) {
        scanner.addScanIterator(setting);
    }
    this.scanner = scanner;
    iterator = scanner.iterator();
}