Example 1 with LabelTypePattern
use of org.codelibs.fess.helper.LabelTypeHelper.LabelTypePattern in project fess by codelibs.
the class FessXpathTransformerTest method test_transform.
public void test_transform() throws Exception {
String data = "<html><head><title>Test</title></head><body><h1>Header1</h1><p>This is a pen.</p></body></html>";
final FessXpathTransformer fessXpathTransformer = new FessXpathTransformer();
fessXpathTransformer.init();
SingletonLaContainerFactory.getContainer().register(CrawlingInfoHelper.class, "crawlingInfoHelper");
SingletonLaContainerFactory.getContainer().register(PathMappingHelper.class, "pathMappingHelper");
SingletonLaContainerFactory.getContainer().register(CrawlingConfigHelper.class, "crawlingConfigHelper");
SingletonLaContainerFactory.getContainer().register(SystemHelper.class, "systemHelper");
SingletonLaContainerFactory.getContainer().register(FileTypeHelper.class, "fileTypeHelper");
SingletonLaContainerFactory.getContainer().register(DocumentHelper.class, "documentHelper");
SingletonLaContainerFactory.getContainer().register(LabelTypeHelper.class, "labelTypeHelper");
WebConfig webConfig = new WebConfig();
setValueToObject(webConfig, "labelTypeList", new ArrayList<LabelType>());
ComponentUtil.getCrawlingConfigHelper().store("test", webConfig);
setValueToObject(ComponentUtil.getLabelTypeHelper(), "labelTypePatternList", new ArrayList<LabelTypePattern>());
for (int i = 0; i < 10000; i++) {
if (i % 1000 == 0) {
logger.info(MemoryUtil.getMemoryUsageLog() + ":" + i);
System.gc();
}
ResponseData responseData = new ResponseData();
responseData.setCharSet("UTF-8");
responseData.setContentLength(data.length());
responseData.setExecutionTime(1000L);
responseData.setHttpStatusCode(200);
responseData.setLastModified(new Date());
responseData.setMethod("GET");
responseData.setMimeType("text/html");
responseData.setParentUrl("http://fess.codelibs.org/");
responseData.setResponseBody(data.getBytes());
responseData.setSessionId("test-1");
responseData.setStatus(0);
responseData.setUrl("http://fess.codelibs.org/test.html");
ResultData resultData = fessXpathTransformer.transform(responseData);
// System.out.println(resultData.toString());
}
System.gc();
Thread.sleep(1000L);
logger.info(MemoryUtil.getMemoryUsageLog());
assertTrue(MemoryUtil.getUsedMemory() < 100000000L);
}
Also used :
ResultData(org.codelibs.fess.crawler.entity.ResultData)
LabelType(org.codelibs.fess.es.config.exentity.LabelType)
LabelTypePattern(org.codelibs.fess.helper.LabelTypeHelper.LabelTypePattern)
ResponseData(org.codelibs.fess.crawler.entity.ResponseData)
WebConfig(org.codelibs.fess.es.config.exentity.WebConfig)
Date(java.util.Date)
Aggregations
Date (java.util.Date)1
ResponseData (org.codelibs.fess.crawler.entity.ResponseData)1
ResultData (org.codelibs.fess.crawler.entity.ResultData)1
LabelType (org.codelibs.fess.es.config.exentity.LabelType)1
WebConfig (org.codelibs.fess.es.config.exentity.WebConfig)1
LabelTypePattern (org.codelibs.fess.helper.LabelTypeHelper.LabelTypePattern)1
