Examples with CurlResponse org.codelibs.elasticsearch.runner.net.CurlResponse used on opensource projects

Example 1 with CurlResponse

use of org.codelibs.elasticsearch.runner.net.CurlResponse in project fess by codelibs.

the class AdminBackupAction method upload.

@Execute
public HtmlResponse upload(final UploadForm form) {
    validate(form, messages -> {
    }, () -> asListHtml());
    verifyToken(() -> asListHtml());
    asyncManager.async(() -> {
        final String fileName = form.bulkFile.getFileName();
        if (fileName.startsWith("system") && fileName.endsWith(".properties")) {
            try (final InputStream in = form.bulkFile.getInputStream()) {
                ComponentUtil.getSystemProperties().load(in);
            } catch (final IOException e) {
                logger.warn("Failed to process system.properties file: " + form.bulkFile.getFileName(), e);
            }
        } else {
            try (CurlResponse response = Curl.post(ResourceUtil.getElasticsearchHttpUrl() + "/_bulk").header("Content-Type", "application/json").onConnect((req, con) -> {
                con.setDoOutput(true);
                try (InputStream in = form.bulkFile.getInputStream();
                    OutputStream out = con.getOutputStream()) {
                    CopyUtil.copy(in, out);
                } catch (IOException e) {
                    throw new IORuntimeException(e);
                }
            }).execute()) {
                if (logger.isDebugEnabled()) {
                    logger.debug("Bulk Response:\n" + response.getContentAsString());
                }
                systemHelper.reloadConfiguration();
            } catch (final Exception e) {
                logger.warn("Failed to process bulk file: " + form.bulkFile.getFileName(), e);
            }
        }
    });
    saveInfo(messages -> messages.addSuccessBulkProcessStarted(GLOBAL));
    // no-op
    return redirect(getClass());
}

Example 2 with CurlResponse

use of org.codelibs.elasticsearch.runner.net.CurlResponse in project fess by codelibs.

the class GitBucketDataStoreImpl method getIssueComments.

private List<String> getIssueComments(final String issueUrl, final String authToken) {
    final String commentsUrl = issueUrl + "/comments";
    final List<String> commentList = new ArrayList<String>();
    try (CurlResponse curlResponse = Curl.get(commentsUrl).header("Authorization", "token " + authToken).execute()) {
        final String commentsJson = curlResponse.getContentAsString();
        List<Map<String, Object>> comments = new ObjectMapper().readValue(commentsJson, new TypeReference<List<Map<String, Object>>>() {
        });
        for (Map<String, Object> comment : comments) {
            if (comment.containsKey("body")) {
                commentList.add((String) comment.get("body"));
            }
        }
    } catch (final Exception e) {
        logger.warn("Failed to access to " + issueUrl, e);
    }
    return commentList;
}

Example 3 with CurlResponse

use of org.codelibs.elasticsearch.runner.net.CurlResponse in project fess by codelibs.

the class GitBucketDataStoreImpl method storeIssueById.

private void storeIssueById(final String rootURL, final String authToken, final String issueLabel, final String owner, final String name, final Integer issueId, final List<String> roleList, final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap, final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap) {
    final String issueUrl = rootURL + "api/v3/repos/" + owner + "/" + name + "/issues/" + issueId.toString();
    final String viewUrl = rootURL + owner + "/" + name + "/issues/" + issueId.toString();
    if (logger.isInfoEnabled()) {
        logger.info("Get a content from " + issueUrl);
    }
    final Map<String, Object> dataMap = new HashMap<>();
    String contentStr = "";
    dataMap.putAll(defaultDataMap);
    // FIXME: Use `ComponentUtil.getDocumentHelper().processRequest` instead of `Curl.get`
    try (CurlResponse curlResponse = Curl.get(issueUrl).header("Authorization", "token " + authToken).execute()) {
        final Map<String, Object> map = curlResponse.getContentAsMap();
        dataMap.put("title", map.getOrDefault("title", ""));
        contentStr = (String) map.getOrDefault("body", "");
    } catch (final Exception e) {
        logger.warn("Failed to access to " + issueUrl, e);
    }
    final String commentsStr = String.join("\n", getIssueComments(issueUrl, authToken));
    contentStr += "\n" + commentsStr;
    dataMap.put("content", contentStr);
    dataMap.put("url", viewUrl);
    dataMap.put("role", roleList);
    dataMap.put("label", Collections.singletonList(issueLabel));
    // TODO scriptMap
    callback.store(paramMap, dataMap);
    return;
}

Example 4 with CurlResponse

use of org.codelibs.elasticsearch.runner.net.CurlResponse in project fess by codelibs.

the class GitBucketDataStoreImpl method storeWikiContents.

@SuppressWarnings("unchecked")
private void storeWikiContents(final String rootURL, final String authToken, final String wikiLabel, final String owner, final String name, final List<String> roleList, final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap, final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap, final long readInterval) {
    final String wikiUrl = rootURL + "api/v3/fess/" + owner + "/" + name + "/wiki";
    List<String> pageList = Collections.emptyList();
    // Get list of pages
    try (CurlResponse curlResponse = Curl.get(wikiUrl).header("Authorization", "token " + authToken).execute()) {
        final Map<String, Object> map = curlResponse.getContentAsMap();
        pageList = (List<String>) map.get("pages");
    } catch (final Exception e) {
        logger.warn("Failed to access to " + wikiUrl, e);
    }
    for (final String page : pageList) {
        // FIXME: URL encoding (e.g. page name that contains spaces)
        final String pageUrl = wikiUrl + "/contents/" + page + ".md";
        final String viewUrl = rootURL + owner + "/" + name + "/wiki/" + page;
        if (logger.isInfoEnabled()) {
            logger.info("Get a content from " + pageUrl);
        }
        final Map<String, Object> dataMap = new HashMap<>();
        dataMap.putAll(defaultDataMap);
        dataMap.putAll(ComponentUtil.getDocumentHelper().processRequest(crawlingConfig, paramMap.get("crawlingInfoId"), pageUrl));
        dataMap.put("url", viewUrl);
        dataMap.put("role", roleList);
        dataMap.put("label", Collections.singletonList(wikiLabel));
        // TODO scriptMap
        callback.store(paramMap, dataMap);
        logger.info("Stored " + pageUrl);
        if (readInterval > 0) {
            sleep(readInterval);
        }
    }
}

Example 5 with CurlResponse

use of org.codelibs.elasticsearch.runner.net.CurlResponse in project fess by codelibs.

the class DictionaryManager method getDictionaryFiles.

public DictionaryFile<? extends DictionaryItem>[] getDictionaryFiles() {
    try (CurlResponse response = Curl.get(ResourceUtil.getElasticsearchHttpUrl() + "/_configsync/file").header("Content-Type", "application/json").param("fields", "path,@timestamp").param("size", ComponentUtil.getFessConfig().getPageDictionaryMaxFetchSize()).execute()) {
        final Map<String, Object> contentMap = response.getContentAsMap();
        @SuppressWarnings("unchecked") final List<Map<String, Object>> fileList = (List<Map<String, Object>>) contentMap.get("file");
        return fileList.stream().map(fileMap -> {
            try {
                final String path = fileMap.get("path").toString();
                final Date timestamp = new SimpleDateFormat(Constants.DATE_FORMAT_ISO_8601_EXTEND_UTC).parse(fileMap.get("@timestamp").toString());
                for (final DictionaryCreator creator : creatorList) {
                    final DictionaryFile<? extends DictionaryItem> file = creator.create(path, timestamp);
                    if (file != null) {
                        return file;
                    }
                }
            } catch (final Exception e) {
                logger.warn("Failed to load " + fileMap, e);
            }
            return null;
        }).filter(file -> file != null).toArray(n -> new DictionaryFile<?>[n]);
    } catch (final IOException e) {
        throw new DictionaryException("Failed to access dictionaries", e);
    }
}