Examples with Matcher nl.basjes.parse.useragent.analyze.Matcher used on opensource projects

Example 1 with Matcher

use of nl.basjes.parse.useragent.analyze.Matcher in project yauaa by nielsbasjes.

the class AbstractUserAgentAnalyzerTester method analyzeMatcherImpactAllTests.

public void analyzeMatcherImpactAllTests() {
    if (getTestCases() == null) {
        return;
    }
    initializeMatchers();
    DebugUserAgent agent = new DebugUserAgent(getWantedFieldNames());
    setVerbose(false);
    agent.setDebug(false);
    Map<String, MatcherImpact> impactOverview = new TreeMap<>();
    List<MatcherImpact> impactList = new ArrayList<>();
    getAllMatchers().stream().sorted(Comparator.comparing(Matcher::getSourceFileName).thenComparingLong(Matcher::getSourceFileLineNumber)).forEach(matcher -> {
        MatcherImpact matcherImpact = new MatcherImpact();
        matcherImpact.neededInputs = matcher.getActionsThatRequireInput();
        matcherImpact.name = matcher.getMatcherSourceLocation();
        impactOverview.put(matcher.getMatcherSourceLocation(), matcherImpact);
        impactList.add(matcherImpact);
    });
    for (TestCase test : getTestCases()) {
        String userAgentString = test.getUserAgent();
        agent.setUserAgentString(userAgentString);
        parse(agent);
        impactOverview.forEach((n, i) -> i.tests++);
        getTouchedMatchers().forEach(m -> {
            MatcherImpact impact = impactOverview.get(m.getMatcherSourceLocation());
            impact.touched++;
            if (m.getActionsThatRequireInput() == m.getActionsThatRequireInputAndReceivedInput()) {
                impact.enoughInputs++;
                if (!m.getUsedMatches().isEmpty()) {
                    impact.used++;
                }
            }
        });
    }
    impactList.forEach(i -> LOG.info("%s", i));
}

Example 2 with Matcher

use of nl.basjes.parse.useragent.analyze.Matcher in project yauaa by nielsbasjes.

the class AbstractUserAgentAnalyzerTester method getUsedMatches.

public synchronized List<Match> getUsedMatches(MutableUserAgent userAgent) {
    // Reset all Matchers
    for (Matcher matcher : getAllMatchers()) {
        matcher.reset();
        matcher.setVerboseTemporarily(false);
    }
    flattener.parse(userAgent);
    List<Match> allMatches = new ArrayList<>(128);
    for (Matcher matcher : getAllMatchers()) {
        allMatches.addAll(matcher.getUsedMatches());
    }
    return allMatches;
}

Example 3 with Matcher

use of nl.basjes.parse.useragent.analyze.Matcher in project yauaa by nielsbasjes.

the class DebugUserAgent method analyzeMatchersResult.

boolean analyzeMatchersResult() {
    boolean passed = true;
    for (String fieldName : getAvailableFieldNamesSorted()) {
        Map<Long, String> receivedValues = new HashMap<>(32);
        for (Pair<UserAgent, Matcher> pair : appliedMatcherResults) {
            UserAgent result = pair.getLeft();
            AgentField partialField = result.get(fieldName);
            if (partialField != null && partialField.getConfidence() >= 0) {
                String previousValue = receivedValues.get(partialField.getConfidence());
                if (previousValue != null) {
                    if (!previousValue.equals(partialField.getValue())) {
                        if (passed) {
                            LOG.error("***********************************************************");
                            LOG.error("***        REALLY IMPORTANT ERRORS IN THE RULESET       ***");
                            LOG.error("*** YOU MUST CHANGE THE CONFIDENCE LEVELS OF YOUR RULES ***");
                            LOG.error("***********************************************************");
                        }
                        passed = false;
                        LOG.error("Found different value for \"{}\" with SAME confidence {}: \"{}\" and \"{}\"", fieldName, partialField.getConfidence(), previousValue, partialField.getValue());
                    }
                } else {
                    receivedValues.put(partialField.getConfidence(), partialField.getValue());
                }
            }
        }
    }
    return passed;
}

Example 4 with Matcher

use of nl.basjes.parse.useragent.analyze.Matcher in project yauaa by nielsbasjes.

the class AbstractUserAgentAnalyzerDirect method parse.

/**
 * Parses and analyzes the useragent string provided in the MutableUserAgent instance.
 * NOTE: This method is internally synchronized because the way the analyzer works is not reentrant.
 * @param userAgent The MutableUserAgent instance that is to be parsed and that gets all results
 * @return An ImmutableUserAgent copy of the results that is suitable for further usage and caching.
 */
@Nonnull
public ImmutableUserAgent parse(MutableUserAgent userAgent) {
    initializeMatchers();
    String useragentString = userAgent.getUserAgentString();
    if (useragentString != null && useragentString.length() > userAgentMaxLength) {
        setAsHacker(userAgent, 100);
        userAgent.setForced(HACKER_ATTACK_VECTOR, "Buffer overflow", 100);
        return new ImmutableUserAgent(hardCodedPostProcessing(userAgent));
    }
    synchronized (this) {
        // Reset all Matchers
        reset();
        if (userAgent.isDebug()) {
            for (Matcher matcher : allMatchers) {
                matcher.setVerboseTemporarily(true);
            }
        }
        try {
            userAgent = flattener.parse(userAgent);
            inform(SYNTAX_ERROR, userAgent.getValue(SYNTAX_ERROR), null);
            if (verbose) {
                LOG.info("=========== Checking all Touched Matchers: {}", touchedMatchers.size());
            }
            // Fire all Analyzers with any input
            for (Matcher matcher : touchedMatchers) {
                matcher.analyze(userAgent);
            }
            if (verbose) {
                LOG.info("=========== Checking all Zero Input Matchers: {}", zeroInputMatchers.size());
            }
            // Fire all Analyzers that should not get input
            for (Matcher matcher : zeroInputMatchers) {
                matcher.analyze(userAgent);
            }
            userAgent.processSetAll();
        } catch (RuntimeException rte) {
            // If this occurs then someone has found a previously undetected problem.
            // So this is a safety for something that 'can' but 'should not' occur.
            userAgent.reset();
            setAsHacker(userAgent, 10000);
            userAgent.setForced(HACKER_ATTACK_VECTOR, "Yauaa Exploit", 10000);
        }
    }
    return new ImmutableUserAgent(hardCodedPostProcessing(userAgent));
}

Example 5 with Matcher

use of nl.basjes.parse.useragent.analyze.Matcher in project yauaa by nielsbasjes.

the class AbstractUserAgentAnalyzerDirect method finalizeLoadingRules.

protected void finalizeLoadingRules() {
    logVersion();
    flattener = new UserAgentTreeFlattener(this);
    if (wantedFieldNames != null) {
        int wantedSize = wantedFieldNames.size();
        if (wantedFieldNames.contains(SET_ALL_FIELDS)) {
            wantedSize--;
        }
        LOG.info("Building all needed matchers for the requested {} fields.", wantedSize);
    } else {
        LOG.info("Building all matchers for all possible fields.");
    }
    Map<String, MatcherConfig> matcherConfigs = config.getMatcherConfigs();
    if (matcherConfigs.isEmpty()) {
        throw new InvalidParserConfigurationException("No matchers were loaded at all.");
    }
    allMatchers.clear();
    for (Map.Entry<String, MatcherConfig> matcherConfigEntry : matcherConfigs.entrySet()) {
        MatcherConfig matcherConfig = matcherConfigEntry.getValue();
        try {
            allMatchers.add(new Matcher(this, wantedFieldNames, matcherConfig));
        } catch (UselessMatcherException ume) {
        // skippedMatchers++;
        }
    }
    verifyWeAreNotAskingForImpossibleFields();
    if (!delayInitialization) {
        initializeMatchers();
    }
}