Examples with CurlResponse org.codelibs.elasticsearch.runner.net.CurlResponse used on opensource projects

Example 11 with CurlResponse

use of org.codelibs.elasticsearch.runner.net.CurlResponse in project fess by codelibs.

the class GitBucketDataStoreImpl method crawlFileContents.

protected void crawlFileContents(final String rootURL, final String authToken, final String owner, final String name, final String refStr, final String path, final int depth, final long readInterval, final Consumer<String> consumer) {
    if (MAX_DEPTH <= depth) {
        return;
    }
    final String url = encode(rootURL, "api/v3/repos/" + owner + "/" + name + "/contents/" + path, "ref=" + refStr);
    try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) {
        final InputStream iStream = curlResponse.getContentAsStream();
        final List<Object> fileList = parseList(iStream);
        for (int i = 0; i < fileList.size(); ++i) {
            @SuppressWarnings("unchecked") final Map<String, String> file = (Map<String, String>) fileList.get(i);
            final String newPath = path.isEmpty() ? file.get("name") : path + "/" + file.get("name");
            switch(file.get("type")) {
                case "file":
                    consumer.accept(newPath);
                    break;
                case "dir":
                    if (readInterval > 0) {
                        sleep(readInterval);
                    }
                    crawlFileContents(rootURL, authToken, owner, name, refStr, newPath, depth + 1, readInterval, consumer);
                    break;
            }
        }
    } catch (final Exception e) {
        logger.warn("Failed to access to " + url, e);
    }
}