Examples with RawNode com.virjar.vscrawler.core.selector.combine.selectables.RawNode used on opensource projects

Example 1 with RawNode

use of com.virjar.vscrawler.core.selector.combine.selectables.RawNode in project vscrawler by virjar.

the class Converters method registerJson.

private static void registerJson() {
    register(JsonNode.class, JsonNode.class, new NodeConvert<JsonNode, JsonNode>() {

        @Override
        public JsonNode convert(JsonNode from) {
            return from;
        }
    });
    register(RawNode.class, JsonNode.class, new NodeConvert<RawNode, JsonNode>() {

        @Override
        public JsonNode convert(RawNode from) {
            return new JsonNode(from.getBaseUrl(), from.getRawText());
        }
    });
    register(StringNode.class, JsonNode.class, new NodeConvert<StringNode, JsonNode>() {

        @Override
        public JsonNode convert(StringNode from) {
            JsonNode jsonNode = new JsonNode(from.getBaseUrl(), (String) null);
            jsonNode.setModel(Lists.transform(from.createOrGetModel(), new Function<String, JSON>() {

                @Override
                public JSON apply(String input) {
                    return (JSON) JSON.parse(input);
                }
            }));
            return jsonNode;
        }
    });
    register(XpathNode.class, JsonNode.class, new NodeConvert<XpathNode, JsonNode>() {

        @Override
        public JsonNode convert(XpathNode from) {
            JsonNode ret = new JsonNode(from.getBaseUrl(), (String) null);
            ret.setModel(Lists.newLinkedList(Iterables.transform(Iterables.filter(from.createOrGetModel(), new Predicate<SIPNode>() {

                @Override
                public boolean apply(SIPNode input) {
                    return input.isText();
                }
            }), new Function<SIPNode, JSON>() {

                @Override
                public JSON apply(SIPNode input) {
                    return (JSON) JSON.parse(input.getTextVal());
                }
            })));
            return ret;
        }
    });
}

Example 2 with RawNode

use of com.virjar.vscrawler.core.selector.combine.selectables.RawNode in project vscrawler by virjar.

the class Converters method registerRaw.

private static void registerRaw() {
    register(RawNode.class, RawNode.class, new NodeConvert<RawNode, RawNode>() {

        @Override
        public RawNode convert(RawNode from) {
            return from;
        }
    });
    register(JsonNode.class, RawNode.class, new NodeConvert<JsonNode, RawNode>() {

        @Override
        public RawNode convert(JsonNode from) {
            List<JSON> jsons = from.createOrGetModel();
            if (jsons.size() == 1) {
                return new RawNode(from.getBaseUrl(), jsons.get(1).toJSONString());
            }
            StringBuilder sb = new StringBuilder();
            for (JSON json : jsons) {
                sb.append(json.toJSONString()).append(" ");
            }
            return new RawNode(from.getBaseUrl(), sb.toString());
        }
    });
    register(StringNode.class, RawNode.class, new NodeConvert<StringNode, RawNode>() {

        @Override
        public RawNode convert(StringNode from) {
            return new RawNode(from.getBaseUrl(), StringUtils.join(from.createOrGetModel(), " "));
        }
    });
    register(XpathNode.class, RawNode.class, new NodeConvert<XpathNode, RawNode>() {

        @Override
        public RawNode convert(XpathNode from) {
            return new RawNode(from.getBaseUrl(), StringUtils.join(Iterables.transform(from.createOrGetModel(), new Function<SIPNode, String>() {

                @Override
                public String apply(SIPNode input) {
                    return input.isText() ? input.getTextVal() : input.getElement().html();
                }
            }), " "));
        }
    });
}

Example 3 with RawNode

use of com.virjar.vscrawler.core.selector.combine.selectables.RawNode in project vscrawler by virjar.

the class Converters method registerString.

private static void registerString() {
    register(StringNode.class, StringNode.class, new NodeConvert<StringNode, StringNode>() {

        @Override
        public StringNode convert(StringNode from) {
            return from;
        }
    });
    register(JsonNode.class, StringNode.class, new NodeConvert<JsonNode, StringNode>() {

        private List<String> genDefault(List<JSON> fromModel) {
            return Lists.transform(fromModel, new Function<JSON, String>() {

                @Override
                public String apply(JSON input) {
                    return input.toJSONString();
                }
            });
        }

        @Override
        public StringNode convert(JsonNode from) {
            StringNode ret = new StringNode(from.getBaseUrl(), null);
            List<String> tempRet = Lists.newLinkedList();
            List<JSON> fromModel = from.createOrGetModel();
            for (JSON json : fromModel) {
                if (json instanceof JSONArray) {
                    JSONArray jsonArray = (JSONArray) json;
                    for (Object o : jsonArray) {
                        if (o instanceof CharSequence) {
                            tempRet.add(o.toString());
                        } else {
                            ret.setModel(genDefault(fromModel));
                            return ret;
                        }
                    }
                } else {
                    ret.setModel(genDefault(fromModel));
                    return ret;
                }
            }
            ret.setModel(tempRet);
            return ret;
        }
    });
    register(RawNode.class, StringNode.class, new NodeConvert<RawNode, StringNode>() {

        @Override
        public StringNode convert(RawNode from) {
            return new StringNode(from.getBaseUrl(), from.createOrGetModel());
        }
    });
    register(XpathNode.class, StringNode.class, new NodeConvert<XpathNode, StringNode>() {

        @Override
        public StringNode convert(XpathNode from) {
            StringNode ret = new StringNode(from.getBaseUrl(), null);
            ret.setModel(Lists.transform(from.createOrGetModel(), new Function<SIPNode, String>() {

                @Override
                public String apply(SIPNode input) {
                    return input.isText() ? input.getTextVal() : input.getElement().html();
                }
            }));
            return ret;
        }
    });
}

Example 4 with RawNode

use of com.virjar.vscrawler.core.selector.combine.selectables.RawNode in project vscrawler by virjar.

the class Converters method registerXpath.

private static void registerXpath() {
    register(XpathNode.class, XpathNode.class, new NodeConvert<XpathNode, XpathNode>() {

        @Override
        public XpathNode convert(XpathNode from) {
            return from;
        }
    });
    register(JsonNode.class, XpathNode.class, new NodeConvert<JsonNode, XpathNode>() {

        @Override
        public XpathNode convert(JsonNode from) {
            throw new UnsupportedOperationException("can not cover json to xpath");
        }
    });
    register(RawNode.class, XpathNode.class, new NodeConvert<RawNode, XpathNode>() {

        @Override
        public XpathNode convert(RawNode from) {
            return new XpathNode(from.getBaseUrl(), from.getRawText());
        }
    });
    register(StringNode.class, XpathNode.class, new NodeConvert<StringNode, XpathNode>() {

        @Override
        public XpathNode convert(final StringNode from) {
            XpathNode ret = new XpathNode(from.getBaseUrl(), (String) null);
            ret.setModel(new SipNodes(Lists.newLinkedList(Iterables.transform(from.createOrGetModel(), new Function<String, SIPNode>() {

                @Override
                public SIPNode apply(String input) {
                    try {
                        Document document = Jsoup.parse(input, from.getBaseUrl());
                        if (document != null) {
                            return SIPNode.e(document);
                        }
                    } catch (Exception e) {
                    // do nothing
                    }
                    return SIPNode.t(input);
                }
            }))));
            return ret;
        }
    });
}