Examples with FetchFilter org.zaproxy.zap.spider.filters.FetchFilter used on opensource projects

Example 1 with FetchFilter

use of org.zaproxy.zap.spider.filters.FetchFilter in project zaproxy by zaproxy.

the class SpiderThread method startSpider.

/**
	 * Start spider.
	 */
private void startSpider() {
    spider = new Spider(id, extension, spiderParams, extension.getModel().getOptionsParam().getConnectionParam(), extension.getModel(), this.scanContext);
    // Register this thread as a Spider Listener, so it gets notified of events and is able
    // to manipulate the UI accordingly
    spider.addSpiderListener(this);
    // Add the pending listeners
    for (SpiderListener l : pendingSpiderListeners) {
        spider.addSpiderListener(l);
    }
    // Add the list of (regex) URIs that should be excluded
    List<String> excludeList = new ArrayList<>();
    excludeList.addAll(extension.getExcludeList());
    excludeList.addAll(extension.getModel().getSession().getExcludeFromSpiderRegexs());
    excludeList.addAll(extension.getModel().getSession().getGlobalExcludeURLRegexs());
    spider.setExcludeList(excludeList);
    // Add seeds accordingly
    addSeeds();
    spider.setScanAsUser(scanUser);
    // Add any custom parsers and filters specified
    if (this.customSpiderParsers != null) {
        for (SpiderParser sp : this.customSpiderParsers) {
            spider.addCustomParser(sp);
        }
    }
    if (this.customFetchFilters != null) {
        for (FetchFilter ff : this.customFetchFilters) {
            spider.addFetchFilter(ff);
        }
    }
    if (this.customParseFilters != null) {
        for (ParseFilter pf : this.customParseFilters) {
            spider.addParseFilter(pf);
        }
    }
    // Start the spider
    spider.start();
}

Example 2 with FetchFilter

use of org.zaproxy.zap.spider.filters.FetchFilter in project zaproxy by zaproxy.

the class SpiderController method resourceURIFound.

@Override
public void resourceURIFound(HttpMessage responseMessage, int depth, String uri, boolean shouldIgnore) {
    log.debug("New resource found: " + uri);
    if (uri == null) {
        return;
    }
    // Create the uri
    URI uriV = createURI(uri);
    if (uriV == null) {
        return;
    }
    // Check if the uri was processed already
    String visitedURI;
    try {
        visitedURI = URLCanonicalizer.buildCleanedParametersURIRepresentation(uriV, spider.getSpiderParam().getHandleParameters(), spider.getSpiderParam().isHandleODataParametersVisited());
    } catch (URIException e) {
        return;
    }
    synchronized (visitedGet) {
        if (visitedGet.contains(visitedURI)) {
            // log.debug("URI already visited: " + visitedURI);
            return;
        } else {
            visitedGet.add(visitedURI);
        }
    }
    // Check if any of the filters disallows this uri
    for (FetchFilter f : fetchFilters) {
        FetchStatus s = f.checkFilter(uriV);
        if (s != FetchStatus.VALID) {
            log.debug("URI: " + uriV + " was filtered by a filter with reason: " + s);
            spider.notifyListenersFoundURI(uri, HttpRequestHeader.GET, s);
            return;
        }
    }
    // Check if should be ignored and not fetched
    if (shouldIgnore) {
        log.debug("URI: " + uriV + " is valid, but will not be fetched, by parser reccommendation.");
        spider.notifyListenersFoundURI(uri, HttpRequestHeader.GET, FetchStatus.VALID);
        return;
    }
    spider.notifyListenersFoundURI(uri, HttpRequestHeader.GET, FetchStatus.VALID);
    // Submit the task
    SpiderTask task = new SpiderTask(spider, responseMessage.getRequestHeader().getURI(), uriV, depth, HttpRequestHeader.GET);
    spider.submitTask(task);
}

Example 3 with FetchFilter

use of org.zaproxy.zap.spider.filters.FetchFilter in project zaproxy by zaproxy.

the class Spider method init.

/**
	 * Initialize the spider.
	 */
private void init() {
    this.paused = false;
    this.stopped = true;
    this.tasksDoneCount = 0;
    this.tasksTotalCount = 0;
    this.initialized = false;
    // Add a default fetch filter and any custom ones
    defaultFetchFilter = new DefaultFetchFilter();
    this.addFetchFilter(defaultFetchFilter);
    for (FetchFilter filter : extension.getCustomFetchFilters()) {
        this.addFetchFilter(filter);
    }
    // Add a default parse filter and any custom ones
    this.addParseFilter(new DefaultParseFilter());
    for (ParseFilter filter : extension.getCustomParseFilters()) this.addParseFilter(filter);
    // Add the scan context, if any
    defaultFetchFilter.setScanContext(this.scanContext);
    defaultFetchFilter.setDomainsAlwaysInScope(spiderParam.getDomainsAlwaysInScopeEnabled());
}

Example 4 with FetchFilter

use of org.zaproxy.zap.spider.filters.FetchFilter in project zaproxy by zaproxy.

the class SpiderController method resourcePostURIFound.

@Override
public void resourcePostURIFound(HttpMessage responseMessage, int depth, String uri, String requestBody) {
    log.debug("New POST resource found: " + uri);
    // Check if the uri was processed already
    synchronized (visitedPost) {
        if (arrayKeyValueExists(uri, requestBody)) {
            log.debug("URI already visited: " + uri);
            return;
        } else {
            if (visitedPost.containsKey(uri)) {
                visitedPost.get(uri).add(requestBody);
            } else {
                ArrayList<String> l = new ArrayList<String>();
                l.add(requestBody);
                visitedPost.put(uri, l);
            }
        }
    }
    // Create the uri
    URI uriV = createURI(uri);
    if (uriV == null) {
        return;
    }
    // Check if any of the filters disallows this uri
    for (FetchFilter f : fetchFilters) {
        FetchStatus s = f.checkFilter(uriV);
        if (s != FetchStatus.VALID) {
            log.debug("URI: " + uriV + " was filtered by a filter with reason: " + s);
            spider.notifyListenersFoundURI(uri, HttpRequestHeader.POST, s);
            return;
        }
    }
    spider.notifyListenersFoundURI(uri, HttpRequestHeader.POST, FetchStatus.VALID);
    // Submit the task
    SpiderTask task = new SpiderTask(spider, responseMessage.getRequestHeader().getURI(), uriV, depth, HttpRequestHeader.POST, requestBody);
    spider.submitTask(task);
}

Example 5 with FetchFilter

use of org.zaproxy.zap.spider.filters.FetchFilter in project zaproxy by zaproxy.

the class SpiderScanController method startScan.

@Override
public int startScan(String name, Target target, User user, Object[] contextSpecificObjects) {
    spiderScansLock.lock();
    try {
        int id = this.scanIdCounter++;
        SpiderParam spiderParams = extension.getSpiderParam();
        List<SpiderParser> customSpiderParsers = new ArrayList<SpiderParser>();
        List<FetchFilter> customFetchFilters = new ArrayList<FetchFilter>();
        List<ParseFilter> customParseFilters = new ArrayList<ParseFilter>();
        URI startUri = null;
        if (contextSpecificObjects != null) {
            for (Object obj : contextSpecificObjects) {
                if (obj instanceof SpiderParam) {
                    log.debug("Setting custom spider params");
                    spiderParams = (SpiderParam) obj;
                } else if (obj instanceof SpiderParser) {
                    customSpiderParsers.add((SpiderParser) obj);
                } else if (obj instanceof FetchFilter) {
                    customFetchFilters.add((FetchFilter) obj);
                } else if (obj instanceof ParseFilter) {
                    customParseFilters.add((ParseFilter) obj);
                } else if (obj instanceof URI) {
                    startUri = (URI) obj;
                } else {
                    log.error("Unexpected contextSpecificObject: " + obj.getClass().getCanonicalName());
                }
            }
        }
        if (spiderParams.getMaxChildren() > 0) {
            // Add the filters to filter on maximum number of children
            MaxChildrenFetchFilter maxChildrenFetchFilter = new MaxChildrenFetchFilter();
            maxChildrenFetchFilter.setMaxChildren(spiderParams.getMaxChildren());
            maxChildrenFetchFilter.setModel(extension.getModel());
            MaxChildrenParseFilter maxChildrenParseFilter = new MaxChildrenParseFilter();
            maxChildrenParseFilter.setMaxChildren(spiderParams.getMaxChildren());
            maxChildrenParseFilter.setModel(extension.getModel());
            customFetchFilters.add(maxChildrenFetchFilter);
            customParseFilters.add(maxChildrenParseFilter);
        }
        SpiderScan scan = new SpiderScan(extension, spiderParams, target, startUri, user, id, name);
        scan.setCustomSpiderParsers(customSpiderParsers);
        scan.setCustomFetchFilters(customFetchFilters);
        scan.setCustomParseFilters(customParseFilters);
        this.spiderScanMap.put(id, scan);
        this.spiderScanList.add(scan);
        scan.start();
        return id;
    } finally {
        spiderScansLock.unlock();
    }
}