use of com.virjar.vscrawler.core.selector.combine.selectables.JsonNode in project vscrawler by virjar.
the class Converters method registerJson.
private static void registerJson() {
register(JsonNode.class, JsonNode.class, new NodeConvert<JsonNode, JsonNode>() {
@Override
public JsonNode convert(JsonNode from) {
return from;
}
});
register(RawNode.class, JsonNode.class, new NodeConvert<RawNode, JsonNode>() {
@Override
public JsonNode convert(RawNode from) {
return new JsonNode(from.getBaseUrl(), from.getRawText());
}
});
register(StringNode.class, JsonNode.class, new NodeConvert<StringNode, JsonNode>() {
@Override
public JsonNode convert(StringNode from) {
JsonNode jsonNode = new JsonNode(from.getBaseUrl(), (String) null);
jsonNode.setModel(Lists.transform(from.createOrGetModel(), new Function<String, JSON>() {
@Override
public JSON apply(String input) {
return (JSON) JSON.parse(input);
}
}));
return jsonNode;
}
});
register(XpathNode.class, JsonNode.class, new NodeConvert<XpathNode, JsonNode>() {
@Override
public JsonNode convert(XpathNode from) {
JsonNode ret = new JsonNode(from.getBaseUrl(), (String) null);
ret.setModel(Lists.newLinkedList(Iterables.transform(Iterables.filter(from.createOrGetModel(), new Predicate<SIPNode>() {
@Override
public boolean apply(SIPNode input) {
return input.isText();
}
}), new Function<SIPNode, JSON>() {
@Override
public JSON apply(SIPNode input) {
return (JSON) JSON.parse(input.getTextVal());
}
})));
return ret;
}
});
}
use of com.virjar.vscrawler.core.selector.combine.selectables.JsonNode in project vscrawler by virjar.
the class AbstractSelectable method jsonPath.
public JsonNode jsonPath(final JSONPath jsonPath) {
// 转化为json
JsonNode fromNode = covert(JsonNode.class);
List<JSON> result = Lists.transform(fromNode.createOrGetModel(), new Function<JSON, JSON>() {
@Override
public JSON apply(JSON input) {
return (JSON) jsonPath.eval(input);
}
});
JsonNode jsonNode = new JsonNode(getBaseUrl());
jsonNode.setModel(result);
return jsonNode;
}
use of com.virjar.vscrawler.core.selector.combine.selectables.JsonNode in project vscrawler by virjar.
the class Converters method registerRaw.
private static void registerRaw() {
register(RawNode.class, RawNode.class, new NodeConvert<RawNode, RawNode>() {
@Override
public RawNode convert(RawNode from) {
return from;
}
});
register(JsonNode.class, RawNode.class, new NodeConvert<JsonNode, RawNode>() {
@Override
public RawNode convert(JsonNode from) {
List<JSON> jsons = from.createOrGetModel();
if (jsons.size() == 1) {
return new RawNode(from.getBaseUrl(), jsons.get(1).toJSONString());
}
StringBuilder sb = new StringBuilder();
for (JSON json : jsons) {
sb.append(json.toJSONString()).append(" ");
}
return new RawNode(from.getBaseUrl(), sb.toString());
}
});
register(StringNode.class, RawNode.class, new NodeConvert<StringNode, RawNode>() {
@Override
public RawNode convert(StringNode from) {
return new RawNode(from.getBaseUrl(), StringUtils.join(from.createOrGetModel(), " "));
}
});
register(XpathNode.class, RawNode.class, new NodeConvert<XpathNode, RawNode>() {
@Override
public RawNode convert(XpathNode from) {
return new RawNode(from.getBaseUrl(), StringUtils.join(Iterables.transform(from.createOrGetModel(), new Function<SIPNode, String>() {
@Override
public String apply(SIPNode input) {
return input.isText() ? input.getTextVal() : input.getElement().html();
}
}), " "));
}
});
}
use of com.virjar.vscrawler.core.selector.combine.selectables.JsonNode in project vscrawler by virjar.
the class Converters method registerXpath.
private static void registerXpath() {
register(XpathNode.class, XpathNode.class, new NodeConvert<XpathNode, XpathNode>() {
@Override
public XpathNode convert(XpathNode from) {
return from;
}
});
register(JsonNode.class, XpathNode.class, new NodeConvert<JsonNode, XpathNode>() {
@Override
public XpathNode convert(JsonNode from) {
throw new UnsupportedOperationException("can not cover json to xpath");
}
});
register(RawNode.class, XpathNode.class, new NodeConvert<RawNode, XpathNode>() {
@Override
public XpathNode convert(RawNode from) {
return new XpathNode(from.getBaseUrl(), from.getRawText());
}
});
register(StringNode.class, XpathNode.class, new NodeConvert<StringNode, XpathNode>() {
@Override
public XpathNode convert(final StringNode from) {
XpathNode ret = new XpathNode(from.getBaseUrl(), (String) null);
ret.setModel(new SipNodes(Lists.newLinkedList(Iterables.transform(from.createOrGetModel(), new Function<String, SIPNode>() {
@Override
public SIPNode apply(String input) {
try {
Document document = Jsoup.parse(input, from.getBaseUrl());
if (document != null) {
return SIPNode.e(document);
}
} catch (Exception e) {
// do nothing
}
return SIPNode.t(input);
}
}))));
return ret;
}
});
}
use of com.virjar.vscrawler.core.selector.combine.selectables.JsonNode in project vscrawler by virjar.
the class Converters method registerString.
private static void registerString() {
register(StringNode.class, StringNode.class, new NodeConvert<StringNode, StringNode>() {
@Override
public StringNode convert(StringNode from) {
return from;
}
});
register(JsonNode.class, StringNode.class, new NodeConvert<JsonNode, StringNode>() {
private List<String> genDefault(List<JSON> fromModel) {
return Lists.transform(fromModel, new Function<JSON, String>() {
@Override
public String apply(JSON input) {
return input.toJSONString();
}
});
}
@Override
public StringNode convert(JsonNode from) {
StringNode ret = new StringNode(from.getBaseUrl(), null);
List<String> tempRet = Lists.newLinkedList();
List<JSON> fromModel = from.createOrGetModel();
for (JSON json : fromModel) {
if (json instanceof JSONArray) {
JSONArray jsonArray = (JSONArray) json;
for (Object o : jsonArray) {
if (o instanceof CharSequence) {
tempRet.add(o.toString());
} else {
ret.setModel(genDefault(fromModel));
return ret;
}
}
} else {
ret.setModel(genDefault(fromModel));
return ret;
}
}
ret.setModel(tempRet);
return ret;
}
});
register(RawNode.class, StringNode.class, new NodeConvert<RawNode, StringNode>() {
@Override
public StringNode convert(RawNode from) {
return new StringNode(from.getBaseUrl(), from.createOrGetModel());
}
});
register(XpathNode.class, StringNode.class, new NodeConvert<XpathNode, StringNode>() {
@Override
public StringNode convert(XpathNode from) {
StringNode ret = new StringNode(from.getBaseUrl(), null);
ret.setModel(Lists.transform(from.createOrGetModel(), new Function<SIPNode, String>() {
@Override
public String apply(SIPNode input) {
return input.isText() ? input.getTextVal() : input.getElement().html();
}
}));
return ret;
}
});
}
Aggregations