Examples with URLModel com.kyj.fx.voeditor.visual.framework.URLModel used on opensource projects

Example 6 with URLModel

use of com.kyj.fx.voeditor.visual.framework.URLModel in project Gargoyle by callakrsos.

the class TF_IDF method getString.

public void getString(Collection<String> links) {
    URLModel[] array = links.parallelStream().map(link -> {
        URLModel model = URLModel.empty();
        try {
            ResponseHandler<URLModel> responseHandler = new ResponseHandler<URLModel>() {

                @Override
                public URLModel apply(InputStream is, Integer code) {
                    if (code == 200) {
                        return new URLModel(link, ValueUtil.toString(is));
                    }
                    return URLModel.empty();
                }
            };
            if (link.startsWith("https")) {
                model = RequestUtil.requestSSL(new URL(link), responseHandler);
            } else {
                model = RequestUtil.request(new URL(link), responseHandler);
            }
        } catch (Exception e) {
            return URLModel.empty();
        }
        return model;
    }).filter(v -> !v.isEmpty()).map(v -> {
        String content = v.getContent();
        ExtractorBase instance = ArticleExtractor.getInstance();
        InputSource source = new InputSource(new StringReader(content));
        source.setEncoding("UTF-8");
        try {
            content = ValueUtil.HTML.getNewsContent(instance, source);
            v.setContent(content);
        } catch (Exception e) {
            v = URLModel.empty();
            e.printStackTrace();
        }
        return v;
    }).filter(v -> !v.isEmpty()).toArray(URLModel[]::new);
    List<KeyValue> tf_IDF = ValueUtil.toTF_IDF(array);
    tf_IDF.forEach(v -> {
        System.out.println(v.toString());
    });
}