Examples with ParseFilter org.zaproxy.zap.spider.filters.ParseFilter used on opensource projects

Example 1 with ParseFilter

use of org.zaproxy.zap.spider.filters.ParseFilter in project zaproxy by zaproxy.

the class SpiderThread method startSpider.

/**
	 * Start spider.
	 */
private void startSpider() {
    spider = new Spider(id, extension, spiderParams, extension.getModel().getOptionsParam().getConnectionParam(), extension.getModel(), this.scanContext);
    // Register this thread as a Spider Listener, so it gets notified of events and is able
    // to manipulate the UI accordingly
    spider.addSpiderListener(this);
    // Add the pending listeners
    for (SpiderListener l : pendingSpiderListeners) {
        spider.addSpiderListener(l);
    }
    // Add the list of (regex) URIs that should be excluded
    List<String> excludeList = new ArrayList<>();
    excludeList.addAll(extension.getExcludeList());
    excludeList.addAll(extension.getModel().getSession().getExcludeFromSpiderRegexs());
    excludeList.addAll(extension.getModel().getSession().getGlobalExcludeURLRegexs());
    spider.setExcludeList(excludeList);
    // Add seeds accordingly
    addSeeds();
    spider.setScanAsUser(scanUser);
    // Add any custom parsers and filters specified
    if (this.customSpiderParsers != null) {
        for (SpiderParser sp : this.customSpiderParsers) {
            spider.addCustomParser(sp);
        }
    }
    if (this.customFetchFilters != null) {
        for (FetchFilter ff : this.customFetchFilters) {
            spider.addFetchFilter(ff);
        }
    }
    if (this.customParseFilters != null) {
        for (ParseFilter pf : this.customParseFilters) {
            spider.addParseFilter(pf);
        }
    }
    // Start the spider
    spider.start();
}

Example 2 with ParseFilter

use of org.zaproxy.zap.spider.filters.ParseFilter in project zaproxy by zaproxy.

the class SpiderTask method run.

@Override
public void run() {
    if (reference == null) {
        log.warn("Null URI. Skipping crawling task: " + this);
        parent.postTaskExecution();
        return;
    }
    // Log the task start
    if (log.isDebugEnabled()) {
        log.debug("Spider Task Started. Processing uri at depth " + depth + " using already constructed message:  " + reference.getURI());
    }
    // Check if the should stop
    if (parent.isStopped()) {
        log.debug("Spider process is stopped. Skipping crawling task...");
        deleteHistoryReference();
        parent.postTaskExecution();
        return;
    }
    // Check if the crawling process is paused and do any "before execution" processing
    parent.preTaskExecution();
    // Fetch the resource
    HttpMessage msg = null;
    try {
        msg = fetchResource();
    } catch (Exception e) {
        // The exception was already logged, in fetchResource, with the URL (which we dont have here)
        parent.postTaskExecution();
        return;
    }
    // Check if the should stop
    if (parent.isStopped()) {
        log.debug("Spider process is stopped. Skipping crawling task...");
        parent.postTaskExecution();
        return;
    }
    // Check if the crawling process is paused
    parent.checkPauseAndWait();
    // Check the parse filters to see if the resource should be skipped from parsing
    boolean isFiltered = false;
    for (ParseFilter filter : parent.getController().getParseFilters()) {
        if (filter.isFiltered(msg)) {
            if (log.isDebugEnabled()) {
                log.debug("Resource fetched, but will not be parsed due to a ParseFilter rule: " + msg.getRequestHeader().getURI());
            }
            isFiltered = true;
            break;
        }
    }
    if (!isFiltered) {
        // Notify the SpiderListeners that a resource was read
        parent.notifyListenersReadURI(msg);
    }
    // Check if the should stop
    if (parent.isStopped()) {
        log.debug("Spider process is stopped. Skipping crawling task...");
        parent.postTaskExecution();
        return;
    }
    // Check if the crawling process is paused
    parent.checkPauseAndWait();
    // Process resource, if this is not the maximum depth
    if (!isFiltered && depth < parent.getSpiderParam().getMaxDepth()) {
        processResource(msg);
    }
    // Update the progress and check if the spidering process should stop
    parent.postTaskExecution();
    log.debug("Spider Task finished.");
}

Example 3 with ParseFilter

use of org.zaproxy.zap.spider.filters.ParseFilter in project zaproxy by zaproxy.

the class Spider method init.

/**
	 * Initialize the spider.
	 */
private void init() {
    this.paused = false;
    this.stopped = true;
    this.tasksDoneCount = 0;
    this.tasksTotalCount = 0;
    this.initialized = false;
    // Add a default fetch filter and any custom ones
    defaultFetchFilter = new DefaultFetchFilter();
    this.addFetchFilter(defaultFetchFilter);
    for (FetchFilter filter : extension.getCustomFetchFilters()) {
        this.addFetchFilter(filter);
    }
    // Add a default parse filter and any custom ones
    this.addParseFilter(new DefaultParseFilter());
    for (ParseFilter filter : extension.getCustomParseFilters()) this.addParseFilter(filter);
    // Add the scan context, if any
    defaultFetchFilter.setScanContext(this.scanContext);
    defaultFetchFilter.setDomainsAlwaysInScope(spiderParam.getDomainsAlwaysInScopeEnabled());
}

Example 4 with ParseFilter

use of org.zaproxy.zap.spider.filters.ParseFilter in project zaproxy by zaproxy.

the class SpiderScanController method startScan.

@Override
public int startScan(String name, Target target, User user, Object[] contextSpecificObjects) {
    spiderScansLock.lock();
    try {
        int id = this.scanIdCounter++;
        SpiderParam spiderParams = extension.getSpiderParam();
        List<SpiderParser> customSpiderParsers = new ArrayList<SpiderParser>();
        List<FetchFilter> customFetchFilters = new ArrayList<FetchFilter>();
        List<ParseFilter> customParseFilters = new ArrayList<ParseFilter>();
        URI startUri = null;
        if (contextSpecificObjects != null) {
            for (Object obj : contextSpecificObjects) {
                if (obj instanceof SpiderParam) {
                    log.debug("Setting custom spider params");
                    spiderParams = (SpiderParam) obj;
                } else if (obj instanceof SpiderParser) {
                    customSpiderParsers.add((SpiderParser) obj);
                } else if (obj instanceof FetchFilter) {
                    customFetchFilters.add((FetchFilter) obj);
                } else if (obj instanceof ParseFilter) {
                    customParseFilters.add((ParseFilter) obj);
                } else if (obj instanceof URI) {
                    startUri = (URI) obj;
                } else {
                    log.error("Unexpected contextSpecificObject: " + obj.getClass().getCanonicalName());
                }
            }
        }
        if (spiderParams.getMaxChildren() > 0) {
            // Add the filters to filter on maximum number of children
            MaxChildrenFetchFilter maxChildrenFetchFilter = new MaxChildrenFetchFilter();
            maxChildrenFetchFilter.setMaxChildren(spiderParams.getMaxChildren());
            maxChildrenFetchFilter.setModel(extension.getModel());
            MaxChildrenParseFilter maxChildrenParseFilter = new MaxChildrenParseFilter();
            maxChildrenParseFilter.setMaxChildren(spiderParams.getMaxChildren());
            maxChildrenParseFilter.setModel(extension.getModel());
            customFetchFilters.add(maxChildrenFetchFilter);
            customParseFilters.add(maxChildrenParseFilter);
        }
        SpiderScan scan = new SpiderScan(extension, spiderParams, target, startUri, user, id, name);
        scan.setCustomSpiderParsers(customSpiderParsers);
        scan.setCustomFetchFilters(customFetchFilters);
        scan.setCustomParseFilters(customParseFilters);
        this.spiderScanMap.put(id, scan);
        this.spiderScanList.add(scan);
        scan.start();
        return id;
    } finally {
        spiderScansLock.unlock();
    }
}