use of org.apache.nutch.parse.ParseImpl in project nutch by apache.
the class TestLinksIndexingFilter method testIndexHostsOnlyAndFilterInlinks.
@Test
public void testIndexHostsOnlyAndFilterInlinks() throws Exception {
conf = NutchConfiguration.create();
conf.set(LinksIndexingFilter.LINKS_ONLY_HOSTS, "true");
conf.set(LinksIndexingFilter.LINKS_INLINKS_HOST, "true");
filter.setConf(conf);
Inlinks inlinks = new Inlinks();
inlinks.add(new Inlink("http://www.test.com", "test"));
inlinks.add(new Inlink("http://www.example.com", "example"));
NutchDocument doc = filter.filter(new NutchDocument(), new ParseImpl("text", new ParseData(new ParseStatus(), "title", new Outlink[0], metadata)), new Text("http://www.example.com/"), new CrawlDatum(), inlinks);
Assert.assertEquals(1, doc.getField("inlinks").getValues().size());
Assert.assertEquals("Index only the host portion of the inlinks after filtering", new URL("http://www.test.com").getHost(), doc.getFieldValue("inlinks"));
}
use of org.apache.nutch.parse.ParseImpl in project nutch by apache.
the class TestLinksIndexingFilter method testIndexHostsOnlyAndFilterOutlinks.
@Test
public void testIndexHostsOnlyAndFilterOutlinks() throws Exception {
conf = NutchConfiguration.create();
conf.set(LinksIndexingFilter.LINKS_ONLY_HOSTS, "true");
conf.set(LinksIndexingFilter.LINKS_OUTLINKS_HOST, "true");
Outlink[] outlinks = generateOutlinks(true);
filter.setConf(conf);
NutchDocument doc = filter.filter(new NutchDocument(), new ParseImpl("text", new ParseData(new ParseStatus(), "title", outlinks, metadata)), new Text("http://www.example.com/"), new CrawlDatum(), new Inlinks());
Assert.assertEquals(1, doc.getField("outlinks").getValues().size());
Assert.assertEquals("Index only the host portion of the outlinks after filtering", new URL("http://www.test.com").getHost(), doc.getFieldValue("outlinks"));
}
use of org.apache.nutch.parse.ParseImpl in project nutch by apache.
the class TestLinksIndexingFilter method testFilterInlinks.
@Test
public void testFilterInlinks() throws Exception {
conf.set(LinksIndexingFilter.LINKS_INLINKS_HOST, "true");
filter.setConf(conf);
Inlinks inlinks = new Inlinks();
inlinks.add(new Inlink("http://www.test.com", "test"));
inlinks.add(new Inlink("http://www.example.com", "example"));
NutchDocument doc = filter.filter(new NutchDocument(), new ParseImpl("text", new ParseData(new ParseStatus(), "title", new Outlink[0], metadata)), new Text("http://www.example.com/"), new CrawlDatum(), inlinks);
Assert.assertEquals(1, doc.getField("inlinks").getValues().size());
Assert.assertEquals("Filter inlinks, allow only those from a different host", "http://www.test.com", doc.getFieldValue("inlinks"));
}
use of org.apache.nutch.parse.ParseImpl in project nutch by apache.
the class TestMoreIndexingFilter method assertContentType.
private void assertContentType(Configuration conf, String source, String expected) throws IndexingException {
Metadata metadata = new Metadata();
metadata.add(Response.CONTENT_TYPE, source);
MoreIndexingFilter filter = new MoreIndexingFilter();
filter.setConf(conf);
NutchDocument doc = filter.filter(new NutchDocument(), new ParseImpl("text", new ParseData(new ParseStatus(), "title", new Outlink[0], metadata)), new Text("http://www.example.com/"), new CrawlDatum(), new Inlinks());
Assert.assertEquals("mime type not detected", expected, doc.getFieldValue("type"));
}
use of org.apache.nutch.parse.ParseImpl in project nutch by apache.
the class TestMoreIndexingFilter method testContentDispositionTitle.
@Test
public void testContentDispositionTitle() throws IndexingException {
Configuration conf = NutchConfiguration.create();
Metadata metadata = new Metadata();
metadata.add(Response.CONTENT_DISPOSITION, "filename=filename.ext");
MoreIndexingFilter filter = new MoreIndexingFilter();
filter.setConf(conf);
Text url = new Text("http://www.example.com/");
ParseImpl parseImpl = new ParseImpl("text", new ParseData(new ParseStatus(), "title", new Outlink[0], metadata));
NutchDocument doc = new NutchDocument();
doc = filter.filter(doc, parseImpl, url, new CrawlDatum(), new Inlinks());
Assert.assertEquals("content-disposition not detected", "filename.ext", doc.getFieldValue("title"));
/* NUTCH-1140: do not add second title to avoid a multi-valued title field */
doc = new NutchDocument();
doc.add("title", "title");
doc = filter.filter(doc, parseImpl, url, new CrawlDatum(), new Inlinks());
Assert.assertEquals("do not add second title by content-disposition", "title", doc.getFieldValue("title"));
}
Aggregations