Examples with DocumentTransformer datawave.query.transformer.DocumentTransformer used on opensource projects

Example 1 with DocumentTransformer

use of datawave.query.transformer.DocumentTransformer in project datawave by NationalSecurityAgency.

the class ParentQueryLogic method getTransformer.

@Override
public QueryLogicTransformer getTransformer(Query settings) {
    DocumentTransformer transformer = new ParentDocumentTransformer(this, settings, this.getMarkingFunctions(), this.getResponseObjectFactory(), this.isReducedResponse());
    transformer.setEventQueryDataDecoratorTransformer(eventQueryDataDecoratorTransformer);
    transformer.setQm(queryModel);
    return transformer;
}

Example 2 with DocumentTransformer

use of datawave.query.transformer.DocumentTransformer in project datawave by NationalSecurityAgency.

the class UniqueTest method runTestQueryWithUniqueness.

protected void runTestQueryWithUniqueness(Set<Set<String>> expected, String querystr, Date startDate, Date endDate, Map<String, String> extraParms, Connector connector) throws Exception {
    log.debug("runTestQueryWithUniqueness");
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(querystr);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    DocumentTransformer transformer = (DocumentTransformer) (logic.getTransformer(settings));
    TransformIterator iter = new DatawaveTransformIterator(logic.iterator(), transformer);
    List<Object> eventList = new ArrayList<>();
    while (iter.hasNext()) {
        eventList.add(iter.next());
    }
    BaseQueryResponse response = transformer.createResponse(eventList);
    // un-comment to look at the json output
    // ObjectMapper mapper = new ObjectMapper();
    // mapper.enable(MapperFeature.USE_WRAPPER_NAME_AS_PROPERTY_NAME);
    // mapper.writeValue(new File("/tmp/grouped2.json"), response);
    Assert.assertTrue(response instanceof DefaultEventQueryResponse);
    DefaultEventQueryResponse eventQueryResponse = (DefaultEventQueryResponse) response;
    for (EventBase event : eventQueryResponse.getEvents()) {
        boolean found = false;
        for (Iterator<Set<String>> it = expected.iterator(); it.hasNext(); ) {
            Set<String> expectedSet = it.next();
            if (expectedSet.contains(event.getMetadata().getInternalId())) {
                it.remove();
                found = true;
                break;
            }
        }
        Assert.assertTrue(found);
    }
    Assert.assertTrue(expected.isEmpty());
}

Example 3 with DocumentTransformer

use of datawave.query.transformer.DocumentTransformer in project datawave by NationalSecurityAgency.

the class ShardQueryLogic method getTransformer.

@Override
public QueryLogicTransformer getTransformer(Query settings) {
    MarkingFunctions markingFunctions = this.getMarkingFunctions();
    ResponseObjectFactory responseObjectFactory = this.getResponseObjectFactory();
    boolean reducedInSettings = false;
    String reducedResponseStr = settings.findParameter(QueryOptions.REDUCED_RESPONSE).getParameterValue().trim();
    if (org.apache.commons.lang.StringUtils.isNotBlank(reducedResponseStr)) {
        reducedInSettings = Boolean.parseBoolean(reducedResponseStr);
    }
    boolean reduced = (this.isReducedResponse() || reducedInSettings);
    DocumentTransformer transformer = new DocumentTransformer(this, settings, markingFunctions, responseObjectFactory, reduced);
    transformer.setEventQueryDataDecoratorTransformer(eventQueryDataDecoratorTransformer);
    transformer.setContentFieldNames(getConfig().getContentFieldNames());
    transformer.setLogTimingDetails(this.getLogTimingDetails());
    transformer.setCardinalityConfiguration(cardinalityConfiguration);
    transformer.setPrimaryToSecondaryFieldMap(primaryToSecondaryFieldMap);
    transformer.setQm(queryModel);
    if (getConfig() != null) {
        transformer.setProjectFields(getConfig().getProjectFields());
        transformer.setBlacklistedFields(getConfig().getBlacklistedFields());
        if (getConfig().getUniqueFields() != null && !getConfig().getUniqueFields().isEmpty()) {
            transformer.addTransform(new UniqueTransform(this, getConfig().getUniqueFields()));
        }
        if (getConfig().getGroupFields() != null && !getConfig().getGroupFields().isEmpty()) {
            transformer.addTransform(new GroupingTransform(this, getConfig().getGroupFields()));
        }
    }
    return transformer;
}

Example 4 with DocumentTransformer

use of datawave.query.transformer.DocumentTransformer in project datawave by NationalSecurityAgency.

the class TestCardinalityWithQuery method runQuery.

@Test
public void runQuery() throws Exception {
    SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
    QueryImpl q = new QueryImpl();
    q.setBeginDate(format.parse("20180101"));
    q.setEndDate(format.parse("20200101"));
    q.setQuery("ID == 'id-001'");
    q.setQueryAuthorizations(auths.toString());
    q.setPagesize(Integer.MAX_VALUE);
    q.setId(UUID.randomUUID());
    /*
         * The reduced response parameter causes components of the attribute metadata, including timestamp, not to be written or read when serializing and
         * deserializing
         */
    q.addParameter("reduced.response", "true");
    q.setQueryLogicName("EventQuery");
    RunningQuery query = new RunningQuery(connector, AccumuloConnectionFactory.Priority.NORMAL, logic, q, "", datawavePrincipal, new QueryMetricFactoryImpl());
    TransformIterator<?, ?> it = query.getTransformIterator();
    AbstractQueryLogicTransformer<?, ?> et = (DocumentTransformer) it.getTransformer();
    CardinalityConfiguration cconf = new CardinalityConfiguration();
    Set<String> cardFields = new HashSet<String>();
    cardFields.add("ID");
    cconf.setCardinalityFields(cardFields);
    Map<String, String> cmap = new HashMap<String, String>();
    cmap.put("ID", "ID");
    cconf.setCardinalityFieldReverseMapping(cmap);
    String path = temporaryFolder.toAbsolutePath().toString();
    cconf.setOutputFileDirectory(path);
    cconf.setCardinalityUidField("ID");
    cconf.setFlushThreshold(1);
    ((DocumentTransformer) it.getTransformer()).setCardinalityConfiguration(cconf);
    EventQueryResponseBase response = (EventQueryResponseBase) et.createResponse(query.next());
    // Wait for cardinality info to be written to the file
    Thread.currentThread().sleep(1000);
    Assert.assertTrue(response.getReturnedEvents().intValue() > 0);
    boolean createdDocFile = false;
    boolean readDocFile = false;
    File folder = new File(path);
    for (File f : folder.listFiles()) {
        if (f.getName().contains("document")) {
            createdDocFile = true;
            CardinalityRecord rcc = CardinalityRecord.readFromDisk(f);
            if (null != rcc) {
                for (DateFieldValueCardinalityRecord dr : rcc.getCardinalityMap().values()) {
                    Assert.assertEquals("20190101", dr.getEventDate());
                    readDocFile = true;
                }
            }
        }
    }
    Assert.assertTrue(createdDocFile);
    Assert.assertTrue(readDocFile);
}