Examples with TimingMetadata datawave.query.attributes.TimingMetadata used on opensource projects

Example 1 with TimingMetadata

use of datawave.query.attributes.TimingMetadata in project datawave by NationalSecurityAgency.

the class LogTiming method addTimingMetadata.

public static void addTimingMetadata(Document document, QuerySpan querySpan) {
    if (document != null && querySpan != null) {
        TimingMetadata timingMetadata = new TimingMetadata();
        synchronized (querySpan) {
            timingMetadata.setHost(host);
            timingMetadata.setSourceCount(querySpan.getSourceCount());
            timingMetadata.setSeekCount(querySpan.getSeekCount());
            timingMetadata.setNextCount(querySpan.getNextCount());
            if (querySpan.getYield()) {
                timingMetadata.setYieldCount(1L);
            } else {
                timingMetadata.setYieldCount(0L);
            }
            long totalStageTimers = querySpan.getStageTimerTotal();
            // do not report timers that are less than 5% of the total
            double threshold = totalStageTimers * 0.05;
            for (Entry<String, Long> e : querySpan.getStageTimers().entrySet()) {
                if (e.getValue().longValue() >= threshold) {
                    timingMetadata.addStageTimer(e.getKey(), new Numeric(e.getValue(), document.getMetadata(), document.isToKeep()));
                }
            }
            querySpan.reset();
        }
        document.put(TIMING_METADATA, timingMetadata);
    }
}

Example 2 with TimingMetadata

use of datawave.query.attributes.TimingMetadata in project datawave by NationalSecurityAgency.

the class UniqueTransformTest method testUniquenessWithTimingMetric.

@Test
public void testUniquenessWithTimingMetric() {
    List<Document> input = new ArrayList<>();
    List<Document> expected = new ArrayList<>();
    String MARKER_STRING = "\u2735FinalDocument\u2735";
    TimingMetadata timingMetadata = new TimingMetadata();
    timingMetadata.setNextCount(5l);
    givenInputDocument(MARKER_STRING).withKeyValue(LogTiming.TIMING_METADATA, timingMetadata.toString()).isExpectedToBeUnique();
    givenInputDocument().withKeyValue("ATTR0", randomValues.get(0)).isExpectedToBeUnique();
    givenInputDocument().withKeyValue("ATTR1", randomValues.get(1)).isExpectedToBeUnique();
    givenInputDocument().withKeyValue("ATTR1", randomValues.get(2));
    givenValueTransformerForFields(UniqueGranularity.ALL, "Attr0");
    assertUniqueDocuments();
}

Example 3 with TimingMetadata

use of datawave.query.attributes.TimingMetadata in project datawave by NationalSecurityAgency.

the class QueryLogicTestHarness method assertLogicResults.

// =============================================
// assert methods
/**
 * Determines if the correct results were obtained for a query.
 *
 * @param logic
 *            key/value response data
 * @param expected
 *            list of key values expected within response data
 * @param checkers
 *            list of additional validation methods
 */
public void assertLogicResults(BaseQueryLogic<Map.Entry<Key, Value>> logic, Collection<String> expected, List<DocumentChecker> checkers) {
    Set<String> actualResults = new HashSet<>();
    if (log.isDebugEnabled()) {
        log.debug("    ======  expected id(s)  ======");
        for (String e : expected) {
            log.debug("id(" + e + ")");
        }
    }
    for (Map.Entry<Key, Value> entry : logic) {
        if (FinalDocumentTrackingIterator.isFinalDocumentKey(entry.getKey())) {
            continue;
        }
        final Document document = this.deserializer.apply(entry).getValue();
        // check all of the types to ensure that all are keepers as defined in the
        // AttributeFactory class
        int count = 0;
        for (Attribute<? extends Comparable<?>> attribute : document.getAttributes()) {
            if (attribute instanceof TimingMetadata) {
            // ignore
            } else if (attribute instanceof Attributes) {
                Attributes attrs = (Attributes) attribute;
                Collection<Class<?>> types = new HashSet<>();
                for (Attribute<? extends Comparable<?>> attr : attrs.getAttributes()) {
                    count++;
                    if (attr instanceof TypeAttribute) {
                        Type<? extends Comparable<?>> type = ((TypeAttribute<?>) attr).getType();
                        if (Objects.nonNull(type)) {
                            types.add(type.getClass());
                        }
                    }
                }
                Assert.assertEquals(AttributeFactory.getKeepers(types), types);
            } else {
                count++;
            }
        }
        // ignore empty documents (possible when only passing FinalDocument back)
        if (count == 0) {
            continue;
        }
        // parse the document
        String extractedResult = this.parser.parse(entry.getKey(), document);
        log.debug("result(" + extractedResult + ") key(" + entry.getKey() + ") document(" + document + ")");
        // verify expected results
        Assert.assertNotNull("extracted result", extractedResult);
        Assert.assertFalse("duplicate result(" + extractedResult + ") key(" + entry.getKey() + ")", actualResults.contains(extractedResult));
        actualResults.add(extractedResult);
        // perform any custom assert checks on document
        for (final DocumentChecker check : checkers) {
            check.assertValid(document);
        }
    }
    log.info("total records found(" + actualResults.size() + ") expected(" + expected.size() + ")");
    // ensure that the complete expected result set exists
    if (expected.size() > actualResults.size()) {
        final Set<String> notFound = new HashSet<>(expected);
        notFound.removeAll(actualResults);
        for (final String m : notFound) {
            log.error("missing result(" + m + ")");
        }
    } else if (expected.size() < actualResults.size()) {
        final Set<String> extra = new HashSet<>(actualResults);
        extra.removeAll(expected);
        for (final String r : extra) {
            log.error("unexpected result(" + r + ")");
        }
    }
    Assert.assertEquals("results do not match expected", expected.size(), actualResults.size());
    Assert.assertTrue("expected and actual values do not match", expected.containsAll(actualResults));
    Assert.assertTrue("expected and actual values do not match", actualResults.containsAll(expected));
}

Example 4 with TimingMetadata

use of datawave.query.attributes.TimingMetadata in project datawave by NationalSecurityAgency.

the class DocumentTransformerSupport method extractMetrics.

protected void extractMetrics(Document document, Key documentKey) {
    Map<String, Attribute<? extends Comparable<?>>> dictionary = document.getDictionary();
    Attribute<? extends Comparable<?>> timingMetadataAttribute = dictionary.get(LogTiming.TIMING_METADATA);
    if (timingMetadataAttribute != null && timingMetadataAttribute instanceof TimingMetadata) {
        TimingMetadata timingMetadata = (TimingMetadata) timingMetadataAttribute;
        long currentSourceCount = timingMetadata.getSourceCount();
        long currentNextCount = timingMetadata.getNextCount();
        long currentSeekCount = timingMetadata.getSeekCount();
        long currentYieldCount = timingMetadata.getYieldCount();
        String host = timingMetadata.getHost();
        sourceCount += currentSourceCount;
        nextCount += currentNextCount;
        seekCount += currentSeekCount;
        yieldCount += currentYieldCount;
        Map<String, Long> stageTimers = timingMetadata.getStageTimers();
        if (stageTimers.containsKey(QuerySpan.Stage.DocumentSpecificTree.toString())) {
            docRanges++;
        } else if (stageTimers.containsKey(QuerySpan.Stage.FieldIndexTree.toString())) {
            fiRanges++;
        }
        if (logTimingDetails || log.isTraceEnabled()) {
            StringBuilder sb = new StringBuilder();
            sb.append("retrieved document from host:").append(host).append(" at key:").append(documentKey.toStringNoTime()).append(" stageTimers:").append(stageTimers);
            sb.append(" sourceCount:").append(currentSourceCount).append(" nextCount:").append(currentNextCount).append(" seekCount:").append(currentSeekCount).append(" yieldCount:").append(currentYieldCount);
            if (log.isTraceEnabled()) {
                log.trace(sb.toString());
            } else {
                log.info(sb.toString());
            }
        }
        if (dictionary.size() == 1) {
            // this document contained only timing metadata
            throw new EmptyObjectException();
        }
    }
}