Example 6 with ChildUrlsException
use of org.codelibs.fess.crawler.exception.ChildUrlsException in project fess by codelibs.
the class FessXpathTransformerTest method test_canonicalXpath.
public void test_canonicalXpath() throws Exception {
final FessXpathTransformer transformer = new FessXpathTransformer();
transformer.init();
final Map<String, Object> dataMap = new HashMap<String, Object>();
final ResponseData responseData = new ResponseData();
responseData.setUrl("http://example.com/");
String data = "<html><body>aaa</body></html>";
Document document = getDocument(data);
try {
transformer.putAdditionalData(dataMap, responseData, document);
fail();
} catch (final ComponentNotFoundException e) {
// ignore
}
data = "<html><head><link rel=\"canonical\" href=\"http://example.com/\"></head><body>aaa</body></html>";
document = getDocument(data);
try {
transformer.putAdditionalData(dataMap, responseData, document);
fail();
} catch (final ComponentNotFoundException e) {
// ignore
}
data = "<html><head><link rel=\"canonical\" href=\"http://example.com/foo\"></head><body>aaa</body></html>";
document = getDocument(data);
try {
transformer.putAdditionalData(dataMap, responseData, document);
fail();
} catch (final ChildUrlsException e) {
final Set<RequestData> childUrlList = e.getChildUrlList();
assertEquals(1, childUrlList.size());
assertEquals("http://example.com/foo", childUrlList.iterator().next().getUrl());
}
data = "<html><link rel=\"canonical\" href=\"http://example.com/foo\"><body>aaa</body></html>";
document = getDocument(data);
try {
transformer.putAdditionalData(dataMap, responseData, document);
fail();
} catch (final ChildUrlsException e) {
final Set<RequestData> childUrlList = e.getChildUrlList();
assertEquals(1, childUrlList.size());
assertEquals("http://example.com/foo", childUrlList.iterator().next().getUrl());
}
}
Also used :
ChildUrlsException(org.codelibs.fess.crawler.exception.ChildUrlsException)
Set(java.util.Set)
ComponentNotFoundException(org.lastaflute.di.core.exception.ComponentNotFoundException)
HashMap(java.util.HashMap)
ResponseData(org.codelibs.fess.crawler.entity.ResponseData)
Document(org.w3c.dom.Document)
Example 7 with ChildUrlsException
use of org.codelibs.fess.crawler.exception.ChildUrlsException in project fess by codelibs.
the class DocumentHelper method processRequest.
public Map<String, Object> processRequest(final CrawlingConfig crawlingConfig, final String crawlingInfoId, final String url) {
if (StringUtil.isBlank(crawlingInfoId)) {
throw new CrawlingAccessException("sessionId is null.");
}
final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getCrawlerClientFactory();
crawlingConfig.initializeClientFactory(crawlerClientFactory);
final CrawlerClient client = crawlerClientFactory.getClient(url);
if (client == null) {
throw new CrawlingAccessException("CrawlerClient is null for " + url);
}
final long startTime = System.currentTimeMillis();
try (final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(url).build())) {
if (responseData.getRedirectLocation() != null) {
final Set<RequestData> childUrlList = new HashSet<>();
childUrlList.add(RequestDataBuilder.newRequestData().get().url(responseData.getRedirectLocation()).build());
throw new ChildUrlsException(childUrlList, "Redirected from " + url);
}
responseData.setExecutionTime(System.currentTimeMillis() - startTime);
responseData.setSessionId(crawlingInfoId);
final RuleManager ruleManager = SingletonLaContainer.getComponent(RuleManager.class);
final Rule rule = ruleManager.getRule(responseData);
if (rule == null) {
throw new CrawlingAccessException("No url rule for " + url);
} else {
responseData.setRuleId(rule.getRuleId());
final ResponseProcessor responseProcessor = rule.getResponseProcessor();
if (responseProcessor instanceof DefaultResponseProcessor) {
final Transformer transformer = ((DefaultResponseProcessor) responseProcessor).getTransformer();
final ResultData resultData = transformer.transform(responseData);
final byte[] data = resultData.getData();
if (data != null) {
try {
@SuppressWarnings("unchecked") final Map<String, Object> result = (Map<String, Object>) SerializeUtil.fromBinaryToObject(data);
return result;
} catch (final Exception e) {
throw new CrawlerSystemException("Could not create an instance from bytes.", e);
}
}
} else {
throw new CrawlingAccessException("The response processor is not DefaultResponseProcessor. responseProcessor: " + responseProcessor + ", url: " + url);
}
}
return null;
} catch (final Exception e) {
throw new CrawlingAccessException("Failed to parse " + url, e);
}
}
Also used :
ChildUrlsException(org.codelibs.fess.crawler.exception.ChildUrlsException)
Transformer(org.codelibs.fess.crawler.transformer.Transformer)
CrawlingAccessException(org.codelibs.fess.crawler.exception.CrawlingAccessException)
CrawlerClientFactory(org.codelibs.fess.crawler.client.CrawlerClientFactory)
ResponseData(org.codelibs.fess.crawler.entity.ResponseData)
DefaultResponseProcessor(org.codelibs.fess.crawler.processor.impl.DefaultResponseProcessor)
ResponseProcessor(org.codelibs.fess.crawler.processor.ResponseProcessor)
CrawlingAccessException(org.codelibs.fess.crawler.exception.CrawlingAccessException)
CrawlerSystemException(org.codelibs.fess.crawler.exception.CrawlerSystemException)
IOException(java.io.IOException)
ChildUrlsException(org.codelibs.fess.crawler.exception.ChildUrlsException)
ResultData(org.codelibs.fess.crawler.entity.ResultData)
RequestData(org.codelibs.fess.crawler.entity.RequestData)
DefaultResponseProcessor(org.codelibs.fess.crawler.processor.impl.DefaultResponseProcessor)
CrawlerSystemException(org.codelibs.fess.crawler.exception.CrawlerSystemException)
RuleManager(org.codelibs.fess.crawler.rule.RuleManager)
CrawlerClient(org.codelibs.fess.crawler.client.CrawlerClient)
Rule(org.codelibs.fess.crawler.rule.Rule)
Map(java.util.Map)
HashSet(java.util.HashSet)
Example 8 with ChildUrlsException
use of org.codelibs.fess.crawler.exception.ChildUrlsException in project fess by codelibs.
the class FessXpathTransformerTest method test_processMetaRobots_none.
public void test_processMetaRobots_none() throws Exception {
final String data = "<meta name=\"robots\" content=\"none\" />";
final Document document = getDocument(data);
final FessXpathTransformer transformer = new FessXpathTransformer();
final ResponseData responseData = new ResponseData();
responseData.setUrl("http://example.com/");
try {
transformer.processMetaRobots(responseData, new ResultData(), document);
fail();
} catch (ChildUrlsException e) {
assertTrue(e.getChildUrlList().isEmpty());
} catch (Exception e) {
fail();
}
}
Also used :
ChildUrlsException(org.codelibs.fess.crawler.exception.ChildUrlsException)
ResultData(org.codelibs.fess.crawler.entity.ResultData)
ResponseData(org.codelibs.fess.crawler.entity.ResponseData)
Document(org.w3c.dom.Document)
ComponentNotFoundException(org.lastaflute.di.core.exception.ComponentNotFoundException)
ChildUrlsException(org.codelibs.fess.crawler.exception.ChildUrlsException)
Aggregations
ChildUrlsException (org.codelibs.fess.crawler.exception.ChildUrlsException)8
ResponseData (org.codelibs.fess.crawler.entity.ResponseData)7
ResultData (org.codelibs.fess.crawler.entity.ResultData)6
Document (org.w3c.dom.Document)5
ComponentNotFoundException (org.lastaflute.di.core.exception.ComponentNotFoundException)4
Map (java.util.Map)3
CrawlerSystemException (org.codelibs.fess.crawler.exception.CrawlerSystemException)3
ArrayList (java.util.ArrayList)2
HashMap (java.util.HashMap)2
HashSet (java.util.HashSet)2
List (java.util.List)2
Set (java.util.Set)2
TransformerException (javax.xml.transform.TransformerException)2
SerializeUtil (org.codelibs.core.io.SerializeUtil)2
StreamUtil.stream (org.codelibs.core.stream.StreamUtil.stream)2
Constants (org.codelibs.fess.Constants)2
RequestDataBuilder (org.codelibs.fess.crawler.builder.RequestDataBuilder)2
CrawlerClient (org.codelibs.fess.crawler.client.CrawlerClient)2
FessConfig (org.codelibs.fess.mylasta.direction.FessConfig)2
ComponentUtil (org.codelibs.fess.util.ComponentUtil)2
