Examples with Source net.htmlparser.jericho.Source used on opensource projects

Example 71 with Source

use of net.htmlparser.jericho.Source in project zaproxy by zaproxy.

the class SpiderHtmlFormParserUnitTest method shouldFailToParseAnUndefinedMessage.

@Test(expected = NullPointerException.class)
public void shouldFailToParseAnUndefinedMessage() {
    // Given
    HttpMessage undefinedMessage = null;
    SpiderHtmlFormParser htmlParser = createSpiderHtmlFormParser();
    Source source = createSource(createMessageWith("NoForms.html"));
    // When
    htmlParser.parseResource(undefinedMessage, source, BASE_DEPTH);
// Then = NullPointerException
}

Example 72 with Source

use of net.htmlparser.jericho.Source in project zaproxy by zaproxy.

the class SpiderHtmlFormParserUnitTest method shouldUseAbsolutePathBaseHtmlUrlWhenParsingGetFormWithRelativeAction.

@Test
public void shouldUseAbsolutePathBaseHtmlUrlWhenParsingGetFormWithRelativeAction() {
    // Given
    SpiderHtmlFormParser htmlParser = createSpiderHtmlFormParser();
    TestSpiderParserListener listener = createTestSpiderParserListener();
    htmlParser.addSpiderParserListener(listener);
    HttpMessage msg = createMessageWith("GET", "FormWithHtmlBase.html", "action/relative", "/base/absolute/path/", "/a/b.html");
    Source source = createSource(msg);
    // When
    boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH);
    // Then
    assertThat(completelyParsed, is(equalTo(false)));
    assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1)));
    assertThat(listener.getUrlsFound(), contains("http://example.com/base/absolute/path/action/relative?q=Search&submit=Submit"));
}

Example 73 with Source

use of net.htmlparser.jericho.Source in project zaproxy by zaproxy.

the class ExtensionAntiCSRF method getTokenValue.

public String getTokenValue(HttpMessage tokenMsg, String tokenName) {
    String response = tokenMsg.getResponseHeader().toString() + tokenMsg.getResponseBody().toString();
    Source source = new Source(response);
    List<Element> formElements = source.getAllElements(HTMLElementName.FORM);
    if (formElements != null && formElements.size() > 0) {
        for (Element formElement : formElements) {
            List<Element> inputElements = formElement.getAllElements(HTMLElementName.INPUT);
            if (inputElements != null && inputElements.size() > 0) {
                // Loop through all of the INPUT elements
                for (Element inputElement : inputElements) {
                    String id = inputElement.getAttributeValue("ID");
                    if (id != null && id.equalsIgnoreCase(tokenName)) {
                        return inputElement.getAttributeValue("VALUE");
                    }
                    String name = inputElement.getAttributeValue("NAME");
                    if (name != null && name.equalsIgnoreCase(tokenName)) {
                        return inputElement.getAttributeValue("VALUE");
                    }
                }
            }
        }
    }
    return null;
}

Example 74 with Source

use of net.htmlparser.jericho.Source in project zaproxy by zaproxy.

the class SpiderHtmlFormParser method parseResource.

@Override
public boolean parseResource(HttpMessage message, Source source, int depth) {
    log.debug("Parsing an HTML message for forms...");
    // If form processing is disabled, don't parse anything
    if (!param.isProcessForm()) {
        return false;
    }
    // Prepare the source, if not provided
    if (source == null) {
        source = new Source(message.getResponseBody().toString());
    }
    // Get the context (base url)
    String baseURL = message.getRequestHeader().getURI().toString();
    uri = message.getRequestHeader().getURI();
    // Try to see if there's any BASE tag that could change the base URL
    Element base = source.getFirstElement(HTMLElementName.BASE);
    if (base != null) {
        if (log.isDebugEnabled()) {
            log.debug("Base tag was found in HTML: " + base.getDebugInfo());
        }
        String href = base.getAttributeValue("href");
        if (href != null && !href.isEmpty()) {
            baseURL = URLCanonicalizer.getCanonicalURL(href, baseURL);
        }
    }
    // Go through the forms
    List<Element> forms = source.getAllElements(HTMLElementName.FORM);
    for (Element form : forms) {
        //Clear the attributes for each form and store their key and values
        envAttributes.clear();
        for (Attribute att : form.getAttributes()) {
            envAttributes.put(att.getKey(), att.getValue());
        }
        // Get method and action
        String method = form.getAttributeValue("method");
        String action = form.getAttributeValue("action");
        log.debug("Found new form with method: '" + method + "' and action: " + action);
        // If no action, skip the form
        if (action == null) {
            log.debug("No form 'action' defined. Using base URL: " + baseURL);
            action = baseURL;
        }
        // If POSTing forms is not enabled, skip processing of forms with POST method
        if (!param.isPostForm() && method != null && method.trim().equalsIgnoreCase(METHOD_POST)) {
            log.debug("Skipping form with POST method because of user settings.");
            continue;
        }
        // Clear the fragment, if any, as it does not have any relevance for the server
        if (action.contains("#")) {
            int fs = action.lastIndexOf("#");
            action = action.substring(0, fs);
        }
        url = URLCanonicalizer.getCanonicalURL(action, baseURL);
        FormData formData = prepareFormDataSet(form.getFormFields());
        // Process the case of a POST method
        if (method != null && method.trim().equalsIgnoreCase(METHOD_POST)) {
            // Build the absolute canonical URL
            String fullURL = URLCanonicalizer.getCanonicalURL(action, baseURL);
            if (fullURL == null) {
                return false;
            }
            log.debug("Canonical URL constructed using '" + action + "': " + fullURL);
            /*
				 * Ignore encoding, as we will not POST files anyway, so using
				 * "application/x-www-form-urlencoded" is adequate
				 */
            // String encoding = form.getAttributeValue("enctype");
            // if (encoding != null && encoding.equals("multipart/form-data"))
            String baseRequestBody = buildEncodedUrlQuery(formData.getFields());
            if (formData.getSubmitFields().isEmpty()) {
                notifyPostResourceFound(message, depth, fullURL, baseRequestBody);
                continue;
            }
            for (HtmlParameter submitField : formData.getSubmitFields()) {
                notifyPostResourceFound(message, depth, fullURL, appendEncodedUrlQueryParameter(baseRequestBody, submitField));
            }
        } else // Process anything else as a GET method
        {
            // Process the final URL
            if (action.contains("?")) {
                if (action.endsWith("?")) {
                    processGetForm(message, depth, action, baseURL, formData);
                } else {
                    processGetForm(message, depth, action + "&", baseURL, formData);
                }
            } else {
                processGetForm(message, depth, action + "?", baseURL, formData);
            }
        }
    }
    return false;
}

Example 75 with Source

use of net.htmlparser.jericho.Source in project zaproxy by zaproxy.

the class PassiveScanThread method run.

@Override
public void run() {
    historyTable = Model.getSingleton().getDb().getTableHistory();
    session = Model.getSingleton().getSession();
    // Get the last id - in case we've just opened an existing session
    currentId = this.getLastHistoryId();
    lastId = currentId;
    while (!shutDown) {
        try {
            if (href != null || lastId > currentId) {
                currentId++;
            } else {
                // Either just started or there are no new records 
                try {
                    Thread.sleep(mainSleep);
                    if (shutDown) {
                        return;
                    }
                    lastId = this.getLastHistoryId();
                } catch (InterruptedException e) {
                    // New URL, but give it a chance to be processed first
                    try {
                        Thread.sleep(postSleep);
                    } catch (InterruptedException e2) {
                    // Ignore
                    }
                }
            }
            try {
                href = getHistoryReference(currentId);
            //historyRecord = historyTable.read(currentId);
            } catch (Exception e) {
                if (shutDown) {
                    return;
                }
                logger.error("Failed to read record " + currentId + " from History table", e);
            }
            if (href != null && (!pscanOptions.isScanOnlyInScope() || session.isInScope(href))) {
                try {
                    // Parse the record
                    HttpMessage msg = href.getHttpMessage();
                    String response = msg.getResponseHeader().toString() + msg.getResponseBody().toString();
                    Source src = new Source(response);
                    for (PassiveScanner scanner : scannerList.list()) {
                        try {
                            if (shutDown) {
                                return;
                            }
                            if (scanner.isEnabled() && scanner.appliesToHistoryType(href.getHistoryType())) {
                                scanner.setParent(this);
                                scanner.scanHttpRequestSend(msg, href.getHistoryId());
                                if (msg.isResponseFromTargetHost()) {
                                    scanner.scanHttpResponseReceive(msg, href.getHistoryId(), src);
                                }
                            }
                        } catch (Throwable e) {
                            if (shutDown) {
                                return;
                            }
                            logger.error("Scanner " + scanner.getName() + " failed on record " + currentId + " from History table: " + href.getMethod() + " " + href.getURI(), e);
                        }
                    }
                } catch (Exception e) {
                    logger.error("Parser failed on record " + currentId + " from History table", e);
                }
            }
        } catch (Exception e) {
            if (shutDown) {
                return;
            }
            logger.error("Failed on record " + currentId + " from History table", e);
        }
    }
}