Examples with LuceneToJexlQueryParser datawave.query.language.parser.jexl.LuceneToJexlQueryParser used on opensource projects

Example 11 with LuceneToJexlQueryParser

use of datawave.query.language.parser.jexl.LuceneToJexlQueryParser in project datawave by NationalSecurityAgency.

the class DiscoveryLogic method initialize.

@Override
public GenericQueryConfiguration initialize(Connector connection, Query settings, Set<Authorizations> auths) throws Exception {
    DiscoveryQueryConfiguration config = new DiscoveryQueryConfiguration(this, settings);
    this.scannerFactory = new ScannerFactory(connection);
    this.metadataHelper = initializeMetadataHelper(connection, config.getMetadataTableName(), auths);
    if (StringUtils.isEmpty(settings.getQuery())) {
        throw new IllegalArgumentException("Query cannot be null");
    }
    if (log.isDebugEnabled()) {
        log.debug("Query parameters set to " + settings.getParameters());
    }
    // Check if the default modelName and modelTableNames have been overriden by custom parameters.
    if (null != settings.findParameter(QueryParameters.PARAMETER_MODEL_NAME) && !settings.findParameter(QueryParameters.PARAMETER_MODEL_NAME).getParameterValue().trim().isEmpty()) {
        modelName = settings.findParameter(QueryParameters.PARAMETER_MODEL_NAME).getParameterValue().trim();
    }
    if (null != settings.findParameter(QueryParameters.PARAMETER_MODEL_TABLE_NAME) && !settings.findParameter(QueryParameters.PARAMETER_MODEL_TABLE_NAME).getParameterValue().trim().isEmpty()) {
        modelTableName = settings.findParameter(QueryParameters.PARAMETER_MODEL_TABLE_NAME).getParameterValue().trim();
    }
    // Check if user would like counts separated by column visibility
    if (null != settings.findParameter(SEPARATE_COUNTS_BY_COLVIS) && !settings.findParameter(SEPARATE_COUNTS_BY_COLVIS).getParameterValue().trim().isEmpty()) {
        separateCountsByColVis = Boolean.valueOf(settings.findParameter(SEPARATE_COUNTS_BY_COLVIS).getParameterValue().trim());
        config.setSeparateCountsByColVis(separateCountsByColVis);
    }
    // Check if user would like to show reference counts instead of term counts
    if (null != settings.findParameter(SHOW_REFERENCE_COUNT) && !settings.findParameter(SHOW_REFERENCE_COUNT).getParameterValue().trim().isEmpty()) {
        showReferenceCount = Boolean.valueOf(settings.findParameter(SHOW_REFERENCE_COUNT).getParameterValue().trim());
        config.setShowReferenceCount(showReferenceCount);
    }
    this.queryModel = metadataHelper.getQueryModel(modelTableName, modelName, null);
    // get the data type filter set if any
    if (null != settings.findParameter(QueryParameters.DATATYPE_FILTER_SET) && !settings.findParameter(QueryParameters.DATATYPE_FILTER_SET).getParameterValue().trim().isEmpty()) {
        Set<String> dataTypeFilter = new HashSet<>(Arrays.asList(StringUtils.split(settings.findParameter(QueryParameters.DATATYPE_FILTER_SET).getParameterValue().trim(), Constants.PARAM_VALUE_SEP)));
        config.setDatatypeFilter(dataTypeFilter);
        if (log.isDebugEnabled()) {
            log.debug("Data type filter set to " + dataTypeFilter);
        }
    }
    // Set the connector
    config.setConnector(connection);
    // Set the auths
    config.setAuthorizations(auths);
    // set the table names
    if (getIndexTableName() != null) {
        config.setIndexTableName(getIndexTableName());
    }
    if (getReverseIndexTableName() != null) {
        config.setReverseIndexTableName(getReverseIndexTableName());
    }
    // Get the ranges
    config.setBeginDate(settings.getBeginDate());
    config.setEndDate(settings.getEndDate());
    if (null == config.getBeginDate() || null == config.getEndDate()) {
        config.setBeginDate(new Date(0));
        config.setEndDate(new Date(Long.MAX_VALUE));
        log.warn("Dates not specified, using entire date range");
    }
    // start with a trimmed version of the query, converted to JEXL
    LuceneToJexlQueryParser parser = new LuceneToJexlQueryParser();
    parser.setAllowLeadingWildCard(this.isAllowLeadingWildcard());
    QueryNode node = parser.parse(settings.getQuery().trim());
    // TODO: Validate that this is a simple list of terms type of query
    config.setQueryString(node.getOriginalQuery());
    if (log.isDebugEnabled()) {
        log.debug("Original Query = " + settings.getQuery().trim());
        log.debug("JEXL Query = " + node.getOriginalQuery());
    }
    // Parse & flatten the query
    ASTJexlScript script = JexlASTHelper.parseAndFlattenJexlQuery(config.getQueryString());
    script = CaseSensitivityVisitor.upperCaseIdentifiers(config, metadataHelper, script);
    Set<String> dataTypes = config.getDatatypeFilter();
    Set<String> allFields;
    allFields = metadataHelper.getAllFields(dataTypes);
    script = QueryModelVisitor.applyModel(script, getQueryModel(), allFields);
    QueryValues literalsAndPatterns = FindLiteralsAndPatternsVisitor.find(script);
    Stopwatch timer = new Stopwatch();
    timer.start();
    // no caching for getAllNormalizers, so try some magic with getFields...
    Multimap<String, Type<?>> dataTypeMap = ArrayListMultimap.create(metadataHelper.getFieldsToDatatypes(config.getDatatypeFilter()));
    /*
         * we have a mapping of FIELD->DataType, but not a mapping of ANYFIELD->DataType which should be all dataTypes
         */
    dataTypeMap.putAll(Constants.ANY_FIELD, uniqueByType(dataTypeMap.values()));
    timer.stop();
    log.debug("Took " + timer.elapsed(TimeUnit.MILLISECONDS) + "ms to get all the dataTypes.");
    config.setLiterals(normalize(new LiteralNormalization(), literalsAndPatterns.getLiterals(), dataTypeMap));
    config.setPatterns(normalize(new PatternNormalization(), literalsAndPatterns.getPatterns(), dataTypeMap));
    config.setRanges(normalizeRanges(new LiteralNormalization(), literalsAndPatterns.getRanges(), dataTypeMap));
    if (log.isDebugEnabled()) {
        log.debug("Normalized Literals = " + config.getLiterals());
        log.debug("Normalized Patterns = " + config.getPatterns());
    }
    return config;
}

Example 12 with LuceneToJexlQueryParser

use of datawave.query.language.parser.jexl.LuceneToJexlQueryParser in project datawave by NationalSecurityAgency.

the class DefaultEdgeEventQueryLogicTest method testParseWithLucene.

@Test
public void testParseWithLucene() throws Exception {
    Query query = new QueryImpl();
    Map<String, QueryParser> parsers = new HashMap<>();
    parsers.put("LUCENE", new LuceneToJexlQueryParser());
    logic.setQuerySyntaxParsers(parsers);
    query.setQuery("SOURCE:sourceValue SINK:targetValue TYPE:TEST1 RELATION:REL1-REL2 ATTRIBUTE1:SOURCE1-SOURCE2");
    query.addParameter(QueryParameters.QUERY_SYNTAX, "LUCENE");
    String transformed = logic.getEventQuery(query);
    assertEquals("(SOURCEFIELD == 'sourceValue' AND TARGETFIELD == 'targetValue' AND ENRICHFIELD == 'enrichValue')", transformed);
}

Example 13 with LuceneToJexlQueryParser

use of datawave.query.language.parser.jexl.LuceneToJexlQueryParser in project datawave by NationalSecurityAgency.

the class TreeFlatteningRebuildingVisitorTest method depthNoStackTraceOrTest.

@Test
public void depthNoStackTraceOrTest() throws Exception {
    final int numTerms = 10000;
    // 13 == "abc_" + 5 + " OR "
    final StringBuilder sb = new StringBuilder(13 * numTerms);
    sb.append("abc_").append(StringUtils.leftPad(Integer.toString(numTerms, 10), 5, '0'));
    for (int i = 2; i <= numTerms; i++) {
        sb.append(" OR ").append(i);
    }
    Assert.assertNotNull(TreeFlatteningRebuildingVisitor.flattenAll(new Parser(new StringReader(";")).parse(new StringReader(new LuceneToJexlQueryParser().parse(sb.toString()).toString()), null)));
}

Example 14 with LuceneToJexlQueryParser

use of datawave.query.language.parser.jexl.LuceneToJexlQueryParser in project datawave by NationalSecurityAgency.

the class FilterFieldsQueryTest method testAnyFieldLuceneInclude.

@Test
public void testAnyFieldLuceneInclude() throws Exception {
    log.info("------  testAnyFieldLuceneInclude  ------");
    String state = "ohio";
    String anyState = this.dataManager.convertAnyField(EQ_OP + "'" + state + "'");
    for (final TestCities city : TestCities.values()) {
        String query = CityField.CITY.name() + ":" + city.name() + AND_OP + " #INCLUDE(" + Constants.ANY_FIELD + ",ohio)";
        String expectQuery = CityField.CITY.name() + EQ_OP + "'" + city.name() + "'" + AND_OP + anyState;
        this.logic.setParser(new LuceneToJexlQueryParser());
        runTest(query, expectQuery, true, false);
    }
}

Example 15 with LuceneToJexlQueryParser

use of datawave.query.language.parser.jexl.LuceneToJexlQueryParser in project datawave by NationalSecurityAgency.

the class FilterFieldsQueryTest method testAnyFieldLuceneText.

@Test
public void testAnyFieldLuceneText() throws Exception {
    log.info("------  testAnyFieldLuceneText  ------");
    String state = "ohio";
    String anyState = this.dataManager.convertAnyField(EQ_OP + "'" + state + "'");
    for (final TestCities city : TestCities.values()) {
        String query = CityField.CITY.name() + ":" + city.name() + AND_OP + " #TEXT(Ohio)";
        String expectQuery = CityField.CITY.name() + EQ_OP + "'" + city.name() + "'" + AND_OP + anyState;
        this.logic.setParser(new LuceneToJexlQueryParser());
        runTest(query, expectQuery, true, false);
    }
}