Examples with ParseException org.apache.commons.jexl2.parser.ParseException used on opensource projects

Example 1 with ParseException

use of org.apache.commons.jexl2.parser.ParseException in project datawave by NationalSecurityAgency.

the class VisitorFunction method apply.

@Override
@Nullable
public ScannerChunk apply(@Nullable ScannerChunk input) {
    SessionOptions options = input.getOptions();
    ScannerChunk newSettings = new ScannerChunk(null, input.getRanges(), input.getLastKnownLocation());
    SessionOptions newOptions = new SessionOptions(options);
    for (IteratorSetting setting : options.getIterators()) {
        final String query = setting.getOptions().get(QueryOptions.QUERY);
        if (null != query) {
            IteratorSetting newIteratorSetting = new IteratorSetting(setting.getPriority(), setting.getName(), setting.getIteratorClass());
            newIteratorSetting.addOptions(setting.getOptions());
            try {
                ASTJexlScript script = null;
                boolean evaluatedPreviously = previouslyExecutable(query);
                boolean madeChange = false;
                if (!evaluatedPreviously && config.isCleanupShardsAndDaysQueryHints()) {
                    script = JexlASTHelper.parseAndFlattenJexlQuery(query);
                    script = DateIndexCleanupVisitor.cleanup(script);
                    madeChange = true;
                }
                String newQuery = evaluatedPreviously ? previouslyExpanded.get(query) : query;
                List<String> debug = null;
                if (log.isTraceEnabled())
                    debug = Lists.newArrayList();
                if (!config.isDisableWhindexFieldMappings() && !evaluatedPreviously) {
                    if (null == script)
                        script = JexlASTHelper.parseAndFlattenJexlQuery(query);
                    // apply the whindex using the shard date
                    ASTJexlScript rebuiltScript = WhindexVisitor.apply(script, config, getEarliestBeginDate(newSettings.getRanges()), metadataHelper);
                    // if the query changed, save it, and mark it as such
                    if (!TreeEqualityVisitor.isEqual(script, rebuiltScript)) {
                        log.debug("[" + config.getQuery().getId() + "] The WhindexVisitor updated the query: " + JexlStringBuildingVisitor.buildQuery(script));
                        script = rebuiltScript;
                        madeChange = true;
                    }
                }
                if (!config.isBypassExecutabilityCheck() || !evaluatedPreviously) {
                    if (null == script)
                        script = JexlASTHelper.parseAndFlattenJexlQuery(query);
                    if (!ExecutableDeterminationVisitor.isExecutable(script, config, indexedFields, indexOnlyFields, nonEventFields, true, debug, this.metadataHelper)) {
                        if (log.isTraceEnabled()) {
                            log.trace("Need to pull up non-executable query: " + JexlStringBuildingVisitor.buildQuery(script));
                            for (String debugStatement : debug) {
                                log.trace(debugStatement);
                            }
                            DefaultQueryPlanner.logQuery(script, "Failing query:");
                        }
                        script = (ASTJexlScript) PullupUnexecutableNodesVisitor.pullupDelayedPredicates(script, true, config, indexedFields, indexOnlyFields, nonEventFields, metadataHelper);
                        madeChange = true;
                        STATE state = ExecutableDeterminationVisitor.getState(script, config, indexedFields, indexOnlyFields, nonEventFields, true, debug, metadataHelper);
                        /**
                         * We could achieve better performance if we live with the small number of queries that error due to the full table scan exception.
                         *
                         * Either look at improving PushdownUnexecutableNodesVisitor or avoid the process altogether.
                         */
                        if (state != STATE.EXECUTABLE) {
                            if (log.isTraceEnabled()) {
                                log.trace("Need to push down non-executable query: " + JexlStringBuildingVisitor.buildQuery(script));
                                for (String debugStatement : debug) {
                                    log.trace(debugStatement);
                                }
                            }
                            script = (ASTJexlScript) PushdownUnexecutableNodesVisitor.pushdownPredicates(script, true, config, indexedFields, indexOnlyFields, nonEventFields, metadataHelper);
                        }
                        state = ExecutableDeterminationVisitor.getState(script, config, indexedFields, indexOnlyFields, nonEventFields, true, debug, metadataHelper);
                        if (state != STATE.EXECUTABLE) {
                            if (state == STATE.ERROR) {
                                log.warn("After expanding the query, it is determined that the query cannot be executed due to index-only fields mixed with expressions that cannot be run against the index.");
                                BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.INDEX_ONLY_FIELDS_MIXED_INVALID_EXPRESSIONS);
                                throw new InvalidQueryException(qe);
                            }
                            log.warn("After expanding the query, it is determined that the query cannot be executed against the field index and a full table scan is required");
                            if (!config.getFullTableScanEnabled()) {
                                if (log.isTraceEnabled()) {
                                    log.trace("Full Table fail of " + JexlStringBuildingVisitor.buildQuery(script));
                                    for (String debugStatement : debug) {
                                        log.trace(debugStatement);
                                    }
                                    DefaultQueryPlanner.logQuery(script, "Failing query:");
                                }
                                PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.FULL_TABLE_SCAN_REQUIRED_BUT_DISABLED);
                                throw new DatawaveFatalQueryException(qe);
                            }
                        }
                        if (log.isTraceEnabled()) {
                            for (String debugStatement : debug) {
                                log.trace(debugStatement);
                            }
                            DefaultQueryPlanner.logQuery(script, "Query pushing down large fielded lists:");
                        }
                    }
                }
                if (config.getSerializeQueryIterator()) {
                    serializeQuery(newIteratorSetting);
                } else {
                    if (!evaluatedPreviously) {
                        // if we have an hdfs configuration, then we can pushdown large fielded lists to an ivarator
                        if (config.getHdfsSiteConfigURLs() != null && setting.getOptions().get(QueryOptions.BATCHED_QUERY) == null) {
                            if (null == script)
                                script = JexlASTHelper.parseAndFlattenJexlQuery(query);
                            try {
                                script = pushdownLargeFieldedLists(config, script);
                                madeChange = true;
                            } catch (IOException ioe) {
                                log.error("Unable to pushdown large fielded lists....leaving in expanded form", ioe);
                            }
                        }
                    }
                }
                // only recompile the script if changes were made to the query
                if (madeChange)
                    newQuery = JexlStringBuildingVisitor.buildQuery(script);
                try {
                    previouslyExpanded.put(query, newQuery);
                } catch (NullPointerException npe) {
                    throw new DatawaveFatalQueryException(String.format("New query is null! madeChange: %b, qid: %s", madeChange, setting.getOptions().get(QueryOptions.QUERY_ID)), npe);
                }
                // test the final script for thresholds
                DefaultQueryPlanner.validateQuerySize("VisitorFunction", script, config, false);
                newIteratorSetting.addOption(QueryOptions.QUERY, newQuery);
                newOptions.removeScanIterator(setting.getName());
                newOptions.addScanIterator(newIteratorSetting);
                if (log.isDebugEnabled()) {
                    log.debug("VisitorFunction result: " + newSettings.getRanges());
                }
                if (log.isTraceEnabled()) {
                    DefaultQueryPlanner.logTrace(PrintingVisitor.formattedQueryStringList(script), "VistorFunction::apply method");
                } else if (log.isDebugEnabled()) {
                    DefaultQueryPlanner.logDebug(PrintingVisitor.formattedQueryStringList(script, DefaultQueryPlanner.maxChildNodesToPrint), "VistorFunction::apply method");
                }
            } catch (ParseException e) {
                throw new DatawaveFatalQueryException(e);
            }
        }
    }
    newSettings.setOptions(newOptions);
    return newSettings;
}

Example 2 with ParseException

use of org.apache.commons.jexl2.parser.ParseException in project datawave by NationalSecurityAgency.

the class PushdownFunction method apply.

public List<ScannerChunk> apply(QueryData qd) {
    Multimap<String, QueryPlan> serverPlan = ArrayListMultimap.create();
    List<ScannerChunk> chunks = Lists.newArrayList();
    try {
        redistributeQueries(serverPlan, tl, new QueryPlan(qd));
        for (String server : serverPlan.keySet()) {
            Collection<QueryPlan> plans = serverPlan.get(server);
            Set<QueryPlan> reducedSet = Sets.newHashSet(plans);
            for (QueryPlan plan : reducedSet) {
                Integer hashCode = plan.hashCode();
                if (queryPlanSet.contains(hashCode)) {
                    continue;
                } else
                    queryPlanSet.clear();
                queryPlanSet.add(hashCode);
                try {
                    SessionOptions options = new SessionOptions();
                    if (log.isTraceEnabled()) {
                        log.trace("setting ranges" + plan.getRanges());
                        log.trace("range set size" + plan.getSettings().size());
                    }
                    for (IteratorSetting setting : plan.getSettings()) {
                        options.addScanIterator(setting);
                    }
                    for (IteratorSetting setting : customSettings) {
                        options.addScanIterator(setting);
                    }
                    for (String cf : plan.getColumnFamilies()) {
                        options.fetchColumnFamily(new Text(cf));
                    }
                    options.setQueryConfig(this.config);
                    chunks.add(new ScannerChunk(options, Lists.newArrayList(plan.getRanges()), server));
                } catch (Exception e) {
                    log.error(e);
                    throw new AccumuloException(e);
                }
            }
        }
    } catch (AccumuloException e) {
        throw new RuntimeException(e);
    } catch (AccumuloSecurityException e) {
        throw new RuntimeException(e);
    } catch (TableNotFoundException e) {
        throw new RuntimeException(e);
    } catch (ParseException e) {
        throw new RuntimeException(e);
    }
    return chunks;
}

Example 3 with ParseException

use of org.apache.commons.jexl2.parser.ParseException in project datawave by NationalSecurityAgency.

the class PushdownScheduler method concatIterators.

/**
 * @return
 * @throws ParseException
 * @throws TableNotFoundException
 * @throws AccumuloSecurityException
 * @throws AccumuloException
 */
protected Iterator<Entry<Key, Value>> concatIterators() throws AccumuloException, AccumuloSecurityException, TableNotFoundException, ParseException {
    String tableName = config.getShardTableName();
    Set<Authorizations> auths = config.getAuthorizations();
    TabletLocator tl;
    Instance instance = config.getConnector().getInstance();
    if (instance instanceof InMemoryInstance) {
        tl = new InMemoryTabletLocator();
        tableId = config.getTableName();
    } else {
        tableId = Tables.getTableId(instance, tableName);
        Credentials credentials = new Credentials(config.getConnector().whoami(), new PasswordToken(config.getAccumuloPassword()));
        tl = TabletLocator.getLocator(new ClientContext(instance, credentials, AccumuloConfiguration.getDefaultConfiguration()), tableId);
    }
    Iterator<List<ScannerChunk>> chunkIter = Iterators.transform(getQueryDataIterator(), new PushdownFunction(tl, config, settings, tableId));
    try {
        session = scannerFactory.newQueryScanner(tableName, auths, config.getQuery());
        if (config.getBypassAccumulo()) {
            session.setDelegatedInitializer(RfileResource.class);
        }
        if (config.getSpeculativeScanning()) {
            session.setSpeculativeScanning(true);
        }
        session.addVisitor(new VisitorFunction(config, metadataHelper));
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    session.setScanLimit(config.getMaxDocScanTimeout());
    if (config.getBackoffEnabled()) {
        session.setBackoffEnabled(true);
    }
    session.setChunkIter(chunkIter);
    session.setTabletLocator(tl);
    session.updateIdentifier(config.getQuery().getId().toString());
    return session;
}

Example 4 with ParseException

use of org.apache.commons.jexl2.parser.ParseException in project datawave by NationalSecurityAgency.

the class JexlASTHelperTest method testDereferenceMarkerNodeSafely.

// dereference marked node while preserving the final wrapper layer
@Test
public void testDereferenceMarkerNodeSafely() throws ParseException {
    String query = "(((((_Value_ = true) && (FOO =~ 'a.*')))))";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(query);
    JexlNode child = script.jjtGetChild(0);
    JexlNode test = JexlASTHelper.dereferenceSafely(child);
    Assert.assertEquals("((_Value_ = true) && (FOO =~ 'a.*'))", JexlStringBuildingVisitor.buildQueryWithoutParse(test));
}

Example 5 with ParseException

use of org.apache.commons.jexl2.parser.ParseException in project datawave by NationalSecurityAgency.

the class JexlASTHelperTest method testIdentifierParse.

private void testIdentifierParse(String query, Set<String> expectedIdentifiers) {
    try {
        ASTJexlScript script = JexlASTHelper.parseJexlQuery(query);
        Set<String> fields = JexlASTHelper.getIdentifierNames(script);
        assertEquals("Expected fields but was", expectedIdentifiers, fields);
    } catch (ParseException e) {
        e.printStackTrace();
    }
}