Examples with DatawaveTransformIterator datawave.webservice.query.iterator.DatawaveTransformIterator used on opensource projects

Example 1 with DatawaveTransformIterator

use of datawave.webservice.query.iterator.DatawaveTransformIterator in project datawave by NationalSecurityAgency.

the class GroupingTest method runTestQueryWithGrouping.

protected BaseQueryResponse runTestQueryWithGrouping(Map<String, Integer> expected, String querystr, Date startDate, Date endDate, Map<String, String> extraParms, Connector connector) throws Exception {
    log.debug("runTestQueryWithGrouping");
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(querystr);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    DocumentTransformer transformer = (DocumentTransformer) (logic.getTransformer(settings));
    TransformIterator iter = new DatawaveTransformIterator(logic.iterator(), transformer);
    List<Object> eventList = new ArrayList<>();
    while (iter.hasNext()) {
        eventList.add(iter.next());
    }
    BaseQueryResponse response = transformer.createResponse(eventList);
    // un-comment to look at the json output
    ObjectMapper mapper = new ObjectMapper();
    mapper.enable(MapperFeature.USE_WRAPPER_NAME_AS_PROPERTY_NAME);
    mapper.writeValue(temporaryFolder.newFile(), response);
    Assert.assertTrue(response instanceof DefaultEventQueryResponse);
    DefaultEventQueryResponse eventQueryResponse = (DefaultEventQueryResponse) response;
    Assert.assertEquals("Got the wrong number of events", expected.size(), (long) eventQueryResponse.getReturnedEvents());
    for (EventBase event : eventQueryResponse.getEvents()) {
        String firstKey = "";
        String secondKey = "";
        Integer value = null;
        for (Object field : event.getFields()) {
            FieldBase fieldBase = (FieldBase) field;
            switch(fieldBase.getName()) {
                case "COUNT":
                    value = Integer.valueOf(fieldBase.getValueString());
                    break;
                case "GENDER":
                case "GEN":
                case "BIRTHDAY":
                    firstKey = fieldBase.getValueString();
                    break;
                case "AGE":
                case "AG":
                case "RECORD":
                    secondKey = fieldBase.getValueString();
                    break;
            }
        }
        log.debug("mapping is " + firstKey + "-" + secondKey + " count:" + value);
        String key;
        if (!firstKey.isEmpty() && !secondKey.isEmpty()) {
            key = firstKey + "-" + secondKey;
        } else if (!firstKey.isEmpty()) {
            key = firstKey;
        } else {
            key = secondKey;
        }
        Assert.assertEquals(expected.get(key), value);
    }
    return response;
}

Example 2 with DatawaveTransformIterator

use of datawave.webservice.query.iterator.DatawaveTransformIterator in project datawave by NationalSecurityAgency.

the class UniqueTest method runTestQueryWithUniqueness.

protected void runTestQueryWithUniqueness(Set<Set<String>> expected, String querystr, Date startDate, Date endDate, Map<String, String> extraParms, Connector connector) throws Exception {
    log.debug("runTestQueryWithUniqueness");
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(querystr);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    DocumentTransformer transformer = (DocumentTransformer) (logic.getTransformer(settings));
    TransformIterator iter = new DatawaveTransformIterator(logic.iterator(), transformer);
    List<Object> eventList = new ArrayList<>();
    while (iter.hasNext()) {
        eventList.add(iter.next());
    }
    BaseQueryResponse response = transformer.createResponse(eventList);
    // un-comment to look at the json output
    // ObjectMapper mapper = new ObjectMapper();
    // mapper.enable(MapperFeature.USE_WRAPPER_NAME_AS_PROPERTY_NAME);
    // mapper.writeValue(new File("/tmp/grouped2.json"), response);
    Assert.assertTrue(response instanceof DefaultEventQueryResponse);
    DefaultEventQueryResponse eventQueryResponse = (DefaultEventQueryResponse) response;
    for (EventBase event : eventQueryResponse.getEvents()) {
        boolean found = false;
        for (Iterator<Set<String>> it = expected.iterator(); it.hasNext(); ) {
            Set<String> expectedSet = it.next();
            if (expectedSet.contains(event.getMetadata().getInternalId())) {
                it.remove();
                found = true;
                break;
            }
        }
        Assert.assertTrue(found);
    }
    Assert.assertTrue(expected.isEmpty());
}

Example 3 with DatawaveTransformIterator

use of datawave.webservice.query.iterator.DatawaveTransformIterator in project datawave by NationalSecurityAgency.

the class GroupingTestWithModel method runTestQueryWithGrouping.

protected BaseQueryResponse runTestQueryWithGrouping(Map<String, Integer> expected, String querystr, Date startDate, Date endDate, Map<String, String> extraParms, Connector connector) throws Exception {
    log.debug("runTestQueryWithGrouping");
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(querystr);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    DocumentTransformer transformer = (DocumentTransformer) (logic.getTransformer(settings));
    TransformIterator iter = new DatawaveTransformIterator(logic.iterator(), transformer);
    List<Object> eventList = new ArrayList<>();
    while (iter.hasNext()) {
        eventList.add(iter.next());
    }
    BaseQueryResponse response = transformer.createResponse(eventList);
    // un-comment to look at the json output
    ObjectMapper mapper = new ObjectMapper();
    mapper.enable(MapperFeature.USE_WRAPPER_NAME_AS_PROPERTY_NAME);
    mapper.writeValue(temporaryFolder.newFile(), response);
    Assert.assertTrue(response instanceof DefaultEventQueryResponse);
    DefaultEventQueryResponse eventQueryResponse = (DefaultEventQueryResponse) response;
    Assert.assertEquals("Got the wrong number of events", expected.size(), (long) eventQueryResponse.getReturnedEvents());
    for (EventBase event : eventQueryResponse.getEvents()) {
        String genderKey = "";
        String ageKey = "";
        Integer value = null;
        for (Object field : event.getFields()) {
            FieldBase fieldBase = (FieldBase) field;
            switch(fieldBase.getName()) {
                case "COUNT":
                    value = Integer.valueOf(fieldBase.getValueString());
                    break;
                case "GENDER":
                    genderKey = fieldBase.getValueString();
                    break;
                case "GEN":
                    genderKey = fieldBase.getValueString();
                    break;
                case "AGE":
                    ageKey = fieldBase.getValueString();
                    break;
                case "AG":
                    ageKey = fieldBase.getValueString();
                    break;
            }
        }
        log.debug("mapping is " + genderKey + "-" + ageKey + " count:" + value);
        String key;
        if (!genderKey.isEmpty() && !ageKey.isEmpty()) {
            key = genderKey + "-" + ageKey;
        } else if (!genderKey.isEmpty()) {
            key = genderKey;
        } else {
            key = ageKey;
        }
        Assert.assertEquals(expected.get(key), value);
    }
    return response;
}