Examples with SpiderParser org.zaproxy.zap.spider.parser.SpiderParser used on opensource projects

Example 1 with SpiderParser

use of org.zaproxy.zap.spider.parser.SpiderParser in project zaproxy by zaproxy.

the class SpiderThread method startSpider.

/**
	 * Start spider.
	 */
private void startSpider() {
    spider = new Spider(id, extension, spiderParams, extension.getModel().getOptionsParam().getConnectionParam(), extension.getModel(), this.scanContext);
    // Register this thread as a Spider Listener, so it gets notified of events and is able
    // to manipulate the UI accordingly
    spider.addSpiderListener(this);
    // Add the pending listeners
    for (SpiderListener l : pendingSpiderListeners) {
        spider.addSpiderListener(l);
    }
    // Add the list of (regex) URIs that should be excluded
    List<String> excludeList = new ArrayList<>();
    excludeList.addAll(extension.getExcludeList());
    excludeList.addAll(extension.getModel().getSession().getExcludeFromSpiderRegexs());
    excludeList.addAll(extension.getModel().getSession().getGlobalExcludeURLRegexs());
    spider.setExcludeList(excludeList);
    // Add seeds accordingly
    addSeeds();
    spider.setScanAsUser(scanUser);
    // Add any custom parsers and filters specified
    if (this.customSpiderParsers != null) {
        for (SpiderParser sp : this.customSpiderParsers) {
            spider.addCustomParser(sp);
        }
    }
    if (this.customFetchFilters != null) {
        for (FetchFilter ff : this.customFetchFilters) {
            spider.addFetchFilter(ff);
        }
    }
    if (this.customParseFilters != null) {
        for (ParseFilter pf : this.customParseFilters) {
            spider.addParseFilter(pf);
        }
    }
    // Start the spider
    spider.start();
}

Example 2 with SpiderParser

use of org.zaproxy.zap.spider.parser.SpiderParser in project zaproxy by zaproxy.

the class SpiderController method prepareDefaultParsers.

private void prepareDefaultParsers() {
    this.parsers = new LinkedList<>();
    SpiderParser parser;
    // If parsing of robots.txt is enabled
    if (spider.getSpiderParam().isParseRobotsTxt()) {
        parser = new SpiderRobotstxtParser(spider.getSpiderParam());
        parsers.add(parser);
    }
    // If parsing of sitemap.xml is enabled		
    if (spider.getSpiderParam().isParseSitemapXml()) {
        if (log.isDebugEnabled())
            log.debug("Adding SpiderSitemapXMLParser");
        parser = new SpiderSitemapXMLParser(spider.getSpiderParam());
        parsers.add(parser);
    } else {
        if (log.isDebugEnabled())
            log.debug("NOT Adding SpiderSitemapXMLParser");
    }
    // If parsing of SVN entries is enabled
    if (spider.getSpiderParam().isParseSVNEntries()) {
        parser = new SpiderSVNEntriesParser(spider.getSpiderParam());
        parsers.add(parser);
    }
    // If parsing of GIT entries is enabled
    if (spider.getSpiderParam().isParseGit()) {
        parser = new SpiderGitParser(spider.getSpiderParam());
        parsers.add(parser);
    }
    // Redirect requests parser
    parser = new SpiderRedirectParser();
    parsers.add(parser);
    // Simple HTML parser
    parser = new SpiderHtmlParser(spider.getSpiderParam());
    this.parsers.add(parser);
    // HTML Form parser
    parser = new SpiderHtmlFormParser(spider.getSpiderParam(), spider.getExtensionSpider().getValueGenerator());
    this.parsers.add(parser);
    Config.CurrentCompatibilityMode.setFormFieldNameCaseInsensitive(false);
    // Prepare the parsers for OData ATOM files
    parser = new SpiderODataAtomParser();
    this.parsers.add(parser);
    // Prepare the parsers for simple non-HTML files
    parser = new SpiderTextParser();
    this.parsers.add(parser);
    this.parsersUnmodifiableView = Collections.unmodifiableList(parsers);
}

Example 3 with SpiderParser

use of org.zaproxy.zap.spider.parser.SpiderParser in project zaproxy by zaproxy.

the class SpiderTask method processResource.

/**
	 * Process a resource, searching for links (uris) to other resources.
	 * 
	 * @param message the HTTP Message
	 */
private void processResource(HttpMessage message) {
    List<SpiderParser> parsers = parent.getController().getParsers();
    // Prepare the Jericho source
    Source source = new Source(message.getResponseBody().toString());
    // Get the full path of the file
    String path = null;
    try {
        path = message.getRequestHeader().getURI().getPath();
    } catch (URIException e) {
    } finally {
        // Handle null paths.
        if (path == null)
            path = "";
    }
    // Parse the resource
    boolean alreadyConsumed = false;
    for (SpiderParser parser : parsers) {
        if (parser.canParseResource(message, path, alreadyConsumed)) {
            if (log.isDebugEnabled())
                log.debug("Parser " + parser + " can parse resource '" + path + "'");
            if (parser.parseResource(message, source, depth))
                alreadyConsumed = true;
        } else {
            if (log.isDebugEnabled())
                log.debug("Parser " + parser + " cannot parse resource '" + path + "'");
        }
    }
}

Example 4 with SpiderParser

use of org.zaproxy.zap.spider.parser.SpiderParser in project zaproxy by zaproxy.

the class SpiderController method init.

public void init() {
    visitedGet.clear();
    visitedPost.clear();
    for (SpiderParser parser : parsers) {
        parser.addSpiderParserListener(this);
    }
}

Example 5 with SpiderParser

use of org.zaproxy.zap.spider.parser.SpiderParser in project zaproxy by zaproxy.

the class SpiderController method reset.

/**
	 * Clears the previous process.
	 */
public void reset() {
    visitedGet.clear();
    visitedPost.clear();
    for (SpiderParser parser : parsers) {
        parser.removeSpiderParserListener(this);
    }
}