use of com.virjar.vscrawler.core.selector.combine.selectables.XpathNode in project vscrawler by virjar.
the class AbstractSelectable method css.
public XpathNode css(String css) {
XpathNode xpathNode = new XpathNode(getBaseUrl(), (String) null);
SipNodes newModels = new SipNodes();
for (SIPNode sipNode : covert(XpathNode.class).createOrGetModel()) {
if (sipNode.isText()) {
continue;
}
for (Element el : sipNode.getElement().select(css)) {
newModels.add(SIPNode.e(el));
}
}
xpathNode.setModel(newModels);
return xpathNode;
}
use of com.virjar.vscrawler.core.selector.combine.selectables.XpathNode in project vscrawler by virjar.
the class AbstractSelectable method xpath.
/**
* xpath抽取
*
* @param xpathEvaluator xpath表达式模型
* @return xpath抽取结果
*/
public XpathNode xpath(XpathEvaluator xpathEvaluator) {
SipNodes sipNodes = xpathEvaluator.evaluate(covert(XpathNode.class).createOrGetModel());
// TODO rawText
XpathNode xpathNode = new XpathNode(getBaseUrl(), getRawText());
xpathNode.setModel(sipNodes);
return xpathNode;
}
use of com.virjar.vscrawler.core.selector.combine.selectables.XpathNode in project vscrawler by virjar.
the class Converters method registerJson.
private static void registerJson() {
register(JsonNode.class, JsonNode.class, new NodeConvert<JsonNode, JsonNode>() {
@Override
public JsonNode convert(JsonNode from) {
return from;
}
});
register(RawNode.class, JsonNode.class, new NodeConvert<RawNode, JsonNode>() {
@Override
public JsonNode convert(RawNode from) {
return new JsonNode(from.getBaseUrl(), from.getRawText());
}
});
register(StringNode.class, JsonNode.class, new NodeConvert<StringNode, JsonNode>() {
@Override
public JsonNode convert(StringNode from) {
JsonNode jsonNode = new JsonNode(from.getBaseUrl(), (String) null);
jsonNode.setModel(Lists.transform(from.createOrGetModel(), new Function<String, JSON>() {
@Override
public JSON apply(String input) {
return (JSON) JSON.parse(input);
}
}));
return jsonNode;
}
});
register(XpathNode.class, JsonNode.class, new NodeConvert<XpathNode, JsonNode>() {
@Override
public JsonNode convert(XpathNode from) {
JsonNode ret = new JsonNode(from.getBaseUrl(), (String) null);
ret.setModel(Lists.newLinkedList(Iterables.transform(Iterables.filter(from.createOrGetModel(), new Predicate<SIPNode>() {
@Override
public boolean apply(SIPNode input) {
return input.isText();
}
}), new Function<SIPNode, JSON>() {
@Override
public JSON apply(SIPNode input) {
return (JSON) JSON.parse(input.getTextVal());
}
})));
return ret;
}
});
}
use of com.virjar.vscrawler.core.selector.combine.selectables.XpathNode in project vscrawler by virjar.
the class Converters method registerRaw.
private static void registerRaw() {
register(RawNode.class, RawNode.class, new NodeConvert<RawNode, RawNode>() {
@Override
public RawNode convert(RawNode from) {
return from;
}
});
register(JsonNode.class, RawNode.class, new NodeConvert<JsonNode, RawNode>() {
@Override
public RawNode convert(JsonNode from) {
List<JSON> jsons = from.createOrGetModel();
if (jsons.size() == 1) {
return new RawNode(from.getBaseUrl(), jsons.get(1).toJSONString());
}
StringBuilder sb = new StringBuilder();
for (JSON json : jsons) {
sb.append(json.toJSONString()).append(" ");
}
return new RawNode(from.getBaseUrl(), sb.toString());
}
});
register(StringNode.class, RawNode.class, new NodeConvert<StringNode, RawNode>() {
@Override
public RawNode convert(StringNode from) {
return new RawNode(from.getBaseUrl(), StringUtils.join(from.createOrGetModel(), " "));
}
});
register(XpathNode.class, RawNode.class, new NodeConvert<XpathNode, RawNode>() {
@Override
public RawNode convert(XpathNode from) {
return new RawNode(from.getBaseUrl(), StringUtils.join(Iterables.transform(from.createOrGetModel(), new Function<SIPNode, String>() {
@Override
public String apply(SIPNode input) {
return input.isText() ? input.getTextVal() : input.getElement().html();
}
}), " "));
}
});
}
use of com.virjar.vscrawler.core.selector.combine.selectables.XpathNode in project vscrawler by virjar.
the class Converters method registerXpath.
private static void registerXpath() {
register(XpathNode.class, XpathNode.class, new NodeConvert<XpathNode, XpathNode>() {
@Override
public XpathNode convert(XpathNode from) {
return from;
}
});
register(JsonNode.class, XpathNode.class, new NodeConvert<JsonNode, XpathNode>() {
@Override
public XpathNode convert(JsonNode from) {
throw new UnsupportedOperationException("can not cover json to xpath");
}
});
register(RawNode.class, XpathNode.class, new NodeConvert<RawNode, XpathNode>() {
@Override
public XpathNode convert(RawNode from) {
return new XpathNode(from.getBaseUrl(), from.getRawText());
}
});
register(StringNode.class, XpathNode.class, new NodeConvert<StringNode, XpathNode>() {
@Override
public XpathNode convert(final StringNode from) {
XpathNode ret = new XpathNode(from.getBaseUrl(), (String) null);
ret.setModel(new SipNodes(Lists.newLinkedList(Iterables.transform(from.createOrGetModel(), new Function<String, SIPNode>() {
@Override
public SIPNode apply(String input) {
try {
Document document = Jsoup.parse(input, from.getBaseUrl());
if (document != null) {
return SIPNode.e(document);
}
} catch (Exception e) {
// do nothing
}
return SIPNode.t(input);
}
}))));
return ret;
}
});
}
Aggregations