Examples with GenericQueryConfiguration datawave.webservice.query.configuration.GenericQueryConfiguration used on opensource projects

Example 1 with GenericQueryConfiguration

use of datawave.webservice.query.configuration.GenericQueryConfiguration in project datawave by NationalSecurityAgency.

the class GroupingTest method runTestQueryWithGrouping.

protected BaseQueryResponse runTestQueryWithGrouping(Map<String, Integer> expected, String querystr, Date startDate, Date endDate, Map<String, String> extraParms, Connector connector) throws Exception {
    log.debug("runTestQueryWithGrouping");
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(querystr);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    DocumentTransformer transformer = (DocumentTransformer) (logic.getTransformer(settings));
    TransformIterator iter = new DatawaveTransformIterator(logic.iterator(), transformer);
    List<Object> eventList = new ArrayList<>();
    while (iter.hasNext()) {
        eventList.add(iter.next());
    }
    BaseQueryResponse response = transformer.createResponse(eventList);
    // un-comment to look at the json output
    ObjectMapper mapper = new ObjectMapper();
    mapper.enable(MapperFeature.USE_WRAPPER_NAME_AS_PROPERTY_NAME);
    mapper.writeValue(temporaryFolder.newFile(), response);
    Assert.assertTrue(response instanceof DefaultEventQueryResponse);
    DefaultEventQueryResponse eventQueryResponse = (DefaultEventQueryResponse) response;
    Assert.assertEquals("Got the wrong number of events", expected.size(), (long) eventQueryResponse.getReturnedEvents());
    for (EventBase event : eventQueryResponse.getEvents()) {
        String firstKey = "";
        String secondKey = "";
        Integer value = null;
        for (Object field : event.getFields()) {
            FieldBase fieldBase = (FieldBase) field;
            switch(fieldBase.getName()) {
                case "COUNT":
                    value = Integer.valueOf(fieldBase.getValueString());
                    break;
                case "GENDER":
                case "GEN":
                case "BIRTHDAY":
                    firstKey = fieldBase.getValueString();
                    break;
                case "AGE":
                case "AG":
                case "RECORD":
                    secondKey = fieldBase.getValueString();
                    break;
            }
        }
        log.debug("mapping is " + firstKey + "-" + secondKey + " count:" + value);
        String key;
        if (!firstKey.isEmpty() && !secondKey.isEmpty()) {
            key = firstKey + "-" + secondKey;
        } else if (!firstKey.isEmpty()) {
            key = firstKey;
        } else {
            key = secondKey;
        }
        Assert.assertEquals(expected.get(key), value);
    }
    return response;
}

Example 2 with GenericQueryConfiguration

use of datawave.webservice.query.configuration.GenericQueryConfiguration in project datawave by NationalSecurityAgency.

the class CompositeFunctionsTest method runTestQuery.

protected void runTestQuery(List<String> expected, String querystr, Date startDate, Date endDate, Map<String, String> extraParms, Connector connector, ShardQueryLogic logic) throws Exception {
    log.debug("runTestQuery");
    log.trace("Creating QueryImpl");
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(querystr);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    logic.setMaxEvaluationPipelines(1);
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    HashSet<String> expectedSet = new HashSet<>(expected);
    HashSet<String> resultSet;
    resultSet = new HashSet<>();
    Set<Document> docs = new HashSet<>();
    for (Entry<Key, Value> entry : logic) {
        Document d = deserializer.apply(entry).getValue();
        log.debug(entry.getKey() + " => " + d);
        Attribute<?> attr = d.get("UUID");
        if (attr == null) {
            attr = d.get("UUID.0");
        }
        Assert.assertNotNull("Result Document did not contain a 'UUID'", attr);
        Assert.assertTrue("Expected result to be an instance of DatwawaveTypeAttribute, was: " + attr.getClass().getName(), attr instanceof TypeAttribute || attr instanceof PreNormalizedAttribute);
        TypeAttribute<?> UUIDAttr = (TypeAttribute<?>) attr;
        String UUID = UUIDAttr.getType().getDelegate().toString();
        Assert.assertTrue("Received unexpected UUID: " + UUID, expected.contains(UUID));
        resultSet.add(UUID);
        docs.add(d);
    }
    if (expected.size() > resultSet.size()) {
        expectedSet.addAll(expected);
        expectedSet.removeAll(resultSet);
        for (String s : expectedSet) {
            log.warn("Missing: " + s);
        }
    }
    if (!expected.containsAll(resultSet)) {
        log.error("Expected results " + expected + " differ form actual results " + resultSet);
    }
    Assert.assertTrue("Expected results " + expected + " differ form actual results " + resultSet, expected.containsAll(resultSet));
    Assert.assertEquals("Unexpected number of records", expected.size(), resultSet.size());
}

Example 3 with GenericQueryConfiguration

use of datawave.webservice.query.configuration.GenericQueryConfiguration in project datawave by NationalSecurityAgency.

the class HitsAreAlwaysIncludedCommonalityTokenTest method runTestQuery.

protected void runTestQuery(Connector connector, String queryString, Date startDate, Date endDate, Map<String, String> extraParms, Collection<String> goodResults) throws Exception {
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(queryString);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    Set<Document> docs = new HashSet<>();
    for (Entry<Key, Value> entry : logic) {
        Document d = deserializer.apply(entry).getValue();
        log.trace(entry.getKey() + " => " + d);
        docs.add(d);
        Attribute hitAttribute = d.get(JexlEvaluation.HIT_TERM_FIELD);
        if (hitAttribute instanceof Attributes) {
            Attributes attributes = (Attributes) hitAttribute;
            for (Attribute attr : attributes.getAttributes()) {
                if (attr instanceof Content) {
                    Content content = (Content) attr;
                    Assert.assertTrue(goodResults.contains(content.getContent()));
                }
            }
        } else if (hitAttribute instanceof Content) {
            Content content = (Content) hitAttribute;
            Assert.assertTrue(goodResults.contains(content.getContent()));
        }
        // remove from goodResults as we find the expected return fields
        log.debug("goodResults: " + goodResults);
        Map<String, Attribute<? extends Comparable<?>>> dictionary = d.getDictionary();
        log.debug("dictionary:" + dictionary);
        for (Entry<String, Attribute<? extends Comparable<?>>> dictionaryEntry : dictionary.entrySet()) {
            Attribute<? extends Comparable<?>> attribute = dictionaryEntry.getValue();
            if (attribute instanceof Attributes) {
                for (Attribute attr : ((Attributes) attribute).getAttributes()) {
                    String toFind = dictionaryEntry.getKey() + ":" + attr;
                    boolean found = goodResults.remove(toFind);
                    if (found)
                        log.debug("removed " + toFind);
                    else
                        log.debug("Did not remove " + toFind);
                }
            } else {
                String toFind = dictionaryEntry.getKey() + ":" + dictionaryEntry.getValue();
                boolean found = goodResults.remove(toFind);
                if (found)
                    log.debug("removed " + toFind);
                else
                    log.debug("Did not remove " + toFind);
            }
        }
        Assert.assertTrue(goodResults + " was not empty", goodResults.isEmpty());
    }
    Assert.assertTrue("No docs were returned!", !docs.isEmpty());
}

Example 4 with GenericQueryConfiguration

use of datawave.webservice.query.configuration.GenericQueryConfiguration in project datawave by NationalSecurityAgency.

the class IvaratorInterruptTest method runTestQuery.

protected void runTestQuery(List<String> expected, String querystr, Date startDate, Date endDate, Map<String, String> extraParms) throws Exception {
    log.debug("runTestQuery");
    log.trace("Creating QueryImpl");
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(querystr);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    logic.setMaxEvaluationPipelines(1);
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    HashSet<String> expectedSet = new HashSet<>(expected);
    HashSet<String> resultSet;
    resultSet = new HashSet<>();
    Set<Document> docs = new HashSet<>();
    for (Map.Entry<Key, Value> entry : logic) {
        Document d = deserializer.apply(entry).getValue();
        log.debug(entry.getKey() + " => " + d);
        Attribute<?> attr = d.get("UUID");
        if (attr == null)
            attr = d.get("UUID.0");
        Assert.assertNotNull("Result Document did not contain a 'UUID'", attr);
        Assert.assertTrue("Expected result to be an instance of DatwawaveTypeAttribute, was: " + attr.getClass().getName(), attr instanceof TypeAttribute || attr instanceof PreNormalizedAttribute);
        TypeAttribute<?> UUIDAttr = (TypeAttribute<?>) attr;
        String UUID = UUIDAttr.getType().getDelegate().toString();
        Assert.assertTrue("Received unexpected UUID: " + UUID, expected.contains(UUID));
        resultSet.add(UUID);
        docs.add(d);
    }
    if (expected.size() > resultSet.size()) {
        expectedSet.addAll(expected);
        expectedSet.removeAll(resultSet);
        for (String s : expectedSet) {
            log.warn("Missing: " + s);
        }
    }
    if (!expected.containsAll(resultSet)) {
        log.error("Expected results " + expected + " differ form actual results " + resultSet);
    }
    Assert.assertTrue("Expected results " + expected + " differ form actual results " + resultSet, expected.containsAll(resultSet));
    Assert.assertEquals("Unexpected number of records", expected.size(), resultSet.size());
}

Example 5 with GenericQueryConfiguration

use of datawave.webservice.query.configuration.GenericQueryConfiguration in project datawave by NationalSecurityAgency.

the class BulkResultsJobConfiguration method _initializeConfiguration.

@Override
public void _initializeConfiguration(Job job, Path jobDir, String jobId, Map<String, String> runtimeParameters, DatawavePrincipal serverPrincipal) throws IOException, QueryException {
    String queryId = runtimeParameters.get("queryId");
    SerializationFormat format = SerializationFormat.valueOf(runtimeParameters.get("format"));
    String outputFormatParameter = runtimeParameters.get("outputFormat");
    if (outputFormatParameter != null && outputFormatParameter.equalsIgnoreCase("TEXT")) {
        this.outputFormatClass = TextOutputFormat.class;
    }
    if (runtimeParameters.containsKey("outputTableName"))
        this.tableName = runtimeParameters.get("outputTableName");
    // Initialize the Query
    QueryLogic<?> logic;
    GenericQueryConfiguration queryConfig;
    String base64EncodedQuery;
    Class<? extends Query> queryImplClass;
    Set<Authorizations> runtimeQueryAuthorizations;
    try {
        QuerySettings settings = setupQuery(sid, queryId, principal);
        logic = settings.getLogic();
        queryConfig = settings.getQueryConfig();
        base64EncodedQuery = settings.getBase64EncodedQuery();
        queryImplClass = settings.getQueryImplClass();
        runtimeQueryAuthorizations = settings.getRuntimeQueryAuthorizations();
    } catch (QueryException qe) {
        log.error("Error getting Query for id: " + queryId, qe);
        throw qe;
    } catch (Exception e) {
        log.error("Error setting up Query for id: " + queryId, e);
        throw new QueryException(e);
    }
    // Setup and run the MapReduce job
    try {
        setupJob(job, jobDir, queryConfig, logic, base64EncodedQuery, queryImplClass, runtimeQueryAuthorizations, serverPrincipal);
        if (null == this.tableName) {
            // Setup job for output to HDFS
            // set the mapper
            job.setMapperClass(BulkResultsFileOutputMapper.class);
            job.getConfiguration().set(BulkResultsFileOutputMapper.RESULT_SERIALIZATION_FORMAT, format.name());
            // Setup the output
            job.setOutputFormatClass(outputFormatClass);
            job.setOutputKeyClass(Key.class);
            job.setOutputValueClass(Value.class);
            if (this.outputFormatClass.equals(SequenceFileOutputFormat.class)) {
                SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK);
            } else if (this.outputFormatClass.equals(TextOutputFormat.class)) {
                // if we are writing Text output to hdfs, we don't want to write key-tab-value, we want just the value
                // this property gets fetched in the Mapper to skip writing the key
                job.setOutputKeyClass(NullWritable.class);
            }
            job.setNumReduceTasks(0);
            SequenceFileOutputFormat.setOutputPath(job, new Path(this.getResultsDir()));
        } else {
            // Setup job for output to table.
            // set the mapper
            job.setMapperClass(BulkResultsTableOutputMapper.class);
            job.getConfiguration().set(BulkResultsTableOutputMapper.TABLE_NAME, tableName);
            job.getConfiguration().set(BulkResultsFileOutputMapper.RESULT_SERIALIZATION_FORMAT, format.name());
            // Setup the output
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(Mutation.class);
            job.setNumReduceTasks(0);
            job.setOutputFormatClass(AccumuloOutputFormat.class);
            AccumuloOutputFormat.setZooKeeperInstance(job, ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers));
            AccumuloOutputFormat.setConnectorInfo(job, user, new PasswordToken(password));
            AccumuloOutputFormat.setCreateTables(job, true);
            AccumuloOutputFormat.setDefaultTableName(job, tableName);
            // AccumuloOutputFormat.loglevel
            AccumuloOutputFormat.setLogLevel(job, Level.INFO);
            // AccumuloOutputFormat.maxlatency
            // AccumuloOutputFormat.maxmemory
            // AccumuloOutputFormat.writethreads
            AccumuloOutputFormat.setBatchWriterOptions(job, new BatchWriterConfig().setMaxLatency(30, TimeUnit.SECONDS).setMaxMemory(10485760).setMaxWriteThreads(2));
        }
    } catch (WebApplicationException wex) {
        throw wex;
    } catch (Exception e) {
        log.error("Error starting job", e);
        throw new QueryException(DatawaveErrorCode.JOB_STARTING_ERROR, e);
    }
}