Example 21 with Tika
use of org.apache.tika.Tika in project tika by apache.
the class RTFParserTest method testMaxLength.
@Test
public void testMaxLength() throws Exception {
File file = getResourceAsFile("/test-documents/testRTFJapanese.rtf");
Metadata metadata = new Metadata();
InputStream stream = TikaInputStream.get(file, metadata);
// Test w/ default limit:
Tika localTika = new Tika();
String content = localTika.parseToString(stream, metadata);
// parseToString closes for convenience:
//stream.close();
assertTrue(content.length() > 500);
// Test setting max length on the instance:
localTika.setMaxStringLength(200);
stream = TikaInputStream.get(file, metadata);
content = localTika.parseToString(stream, metadata);
// parseToString closes for convenience:
//stream.close();
assertTrue(content.length() <= 200);
// Test setting max length per-call:
stream = TikaInputStream.get(file, metadata);
content = localTika.parseToString(stream, metadata, 100);
// parseToString closes for convenience:
//stream.close();
assertTrue(content.length() <= 100);
}
Also used :
TikaInputStream(org.apache.tika.io.TikaInputStream)
FileInputStream(java.io.FileInputStream)
InputStream(java.io.InputStream)
Metadata(org.apache.tika.metadata.Metadata)
RTFMetadata(org.apache.tika.metadata.RTFMetadata)
Tika(org.apache.tika.Tika)
File(java.io.File)
Test(org.junit.Test)
TikaTest(org.apache.tika.TikaTest)
Example 22 with Tika
use of org.apache.tika.Tika in project tika by apache.
the class ZipParserTest method testUnsupportedZipCompressionMethod.
/**
* Test case for the ability of the ZIP parser to extract the name of
* a ZIP entry even if the content of the entry is unreadable due to an
* unsupported compression method.
*
* @see <a href="https://issues.apache.org/jira/browse/TIKA-346">TIKA-346</a>
*/
@Test
public void testUnsupportedZipCompressionMethod() throws Exception {
String content = new Tika().parseToString(ZipParserTest.class.getResourceAsStream("/test-documents/moby.zip"));
assertContains("README", content);
}Example 23 with Tika
use of org.apache.tika.Tika in project tika by apache.
the class DL4JInceptionV3NetTest method recognise.
@Test
@Ignore("until we can make this more robust across platforms")
public void recognise() throws Exception {
TikaConfig config;
try {
config = new TikaConfig(getClass().getResourceAsStream("dl4j-inception3-config.xml"));
} catch (TikaConfigException e) {
if (e.getMessage() != null && e.getMessage().contains("Connection refused")) {
return;
}
throw e;
}
Tika tika = new Tika(config);
Metadata md = new Metadata();
tika.parse(getClass().getResourceAsStream("cat.jpg"), md);
String[] objects = md.getValues("OBJECT");
boolean found = false;
for (String object : objects) {
if (object.contains("_cat")) {
found = true;
}
}
assertTrue(found);
}
Also used :
TikaConfig(org.apache.tika.config.TikaConfig)
TikaConfigException(org.apache.tika.exception.TikaConfigException)
Metadata(org.apache.tika.metadata.Metadata)
Tika(org.apache.tika.Tika)
Ignore(org.junit.Ignore)
Test(org.junit.Test)
Example 24 with Tika
use of org.apache.tika.Tika in project tika by apache.
the class HtmlParserTest method testWhitespaceBetweenTableCells.
/**
* Test case for TIKA-268
*
* @see <a href="https://issues.apache.org/jira/browse/TIKA-268">TIKA-268</a>
*/
@Test
public void testWhitespaceBetweenTableCells() throws Exception {
String test = "<html><body><table><tr><td>a</td><td>b</td></table></body></html>";
String content = new Tika().parseToString(new ByteArrayInputStream(test.getBytes(UTF_8)));
assertContains("a", content);
assertContains("b", content);
assertFalse(content.contains("ab"));
}
Also used :
ByteArrayInputStream(java.io.ByteArrayInputStream)
Tika(org.apache.tika.Tika)
Test(org.junit.Test)
TikaTest(org.apache.tika.TikaTest)
Example 25 with Tika
use of org.apache.tika.Tika in project nutch by apache.
the class CommonCrawlDataDumper method dump.
/**
* Dumps the reverse engineered CBOR content from the provided segment
* directories if a parent directory contains more than one segment,
* otherwise a single segment can be passed as an argument. If the boolean
* argument is provided then the CBOR is also zipped.
*
* @param outputDir the directory you wish to dump the raw content to. This
* directory will be created.
* @param segmentRootDir a directory containing one or more segments.
* @param linkdb Path to linkdb.
* @param gzip a boolean flag indicating whether the CBOR content should also
* be gzipped.
* @param epochFilename if {@code true}, output files will be names using the epoch time (in milliseconds).
* @param extension a file extension to use with output documents.
* @throws Exception if any exception occurs.
*/
public void dump(File outputDir, File segmentRootDir, File linkdb, boolean gzip, String[] mimeTypes, boolean epochFilename, String extension, boolean warc) throws Exception {
if (gzip) {
LOG.info("Gzipping CBOR data has been skipped");
}
// total file counts
Map<String, Integer> typeCounts = new HashMap<>();
// filtered file counters
Map<String, Integer> filteredCounts = new HashMap<>();
Configuration nutchConfig = NutchConfiguration.create();
Path segmentRootPath = new Path(segmentRootDir.toString());
FileSystem fs = segmentRootPath.getFileSystem(nutchConfig);
// get all paths
List<Path> parts = new ArrayList<>();
RemoteIterator<LocatedFileStatus> files = fs.listFiles(segmentRootPath, true);
String partPattern = ".*" + File.separator + Content.DIR_NAME + File.separator + "part-[0-9]{5}" + File.separator + "data";
while (files.hasNext()) {
LocatedFileStatus next = files.next();
if (next.isFile()) {
Path path = next.getPath();
if (path.toString().matches(partPattern)) {
parts.add(path);
}
}
}
LinkDbReader linkDbReader = null;
if (linkdb != null) {
linkDbReader = new LinkDbReader(nutchConfig, new Path(linkdb.toString()));
}
if (parts == null || parts.size() == 0) {
LOG.error("No segment directories found in {} ", segmentRootDir.getAbsolutePath());
System.exit(1);
}
LOG.info("Found {} segment parts", parts.size());
if (gzip && !warc) {
fileList = new ArrayList<>();
constructNewStream(outputDir);
}
for (Path segmentPart : parts) {
LOG.info("Processing segment Part : [ {} ]", segmentPart);
try {
SequenceFile.Reader reader = new SequenceFile.Reader(nutchConfig, SequenceFile.Reader.file(segmentPart));
Writable key = (Writable) reader.getKeyClass().newInstance();
Content content = null;
while (reader.next(key)) {
content = new Content();
reader.getCurrentValue(content);
Metadata metadata = content.getMetadata();
String url = key.toString();
String baseName = FilenameUtils.getBaseName(url);
String extensionName = FilenameUtils.getExtension(url);
if (!extension.isEmpty()) {
extensionName = extension;
} else if ((extensionName == null) || extensionName.isEmpty()) {
extensionName = "html";
}
String outputFullPath = null;
String outputRelativePath = null;
String filename = null;
String timestamp = null;
String reverseKey = null;
if (epochFilename || config.getReverseKey()) {
try {
long epoch = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z").parse(getDate(metadata.get("Date"))).getTime();
timestamp = String.valueOf(epoch);
} catch (ParseException pe) {
LOG.warn(pe.getMessage());
}
reverseKey = reverseUrl(url);
config.setReverseKeyValue(reverseKey.replace("/", "_") + "_" + DigestUtils.sha1Hex(url) + "_" + timestamp);
}
if (!warc) {
if (epochFilename) {
outputFullPath = DumpFileUtil.createFileNameFromUrl(outputDir.getAbsolutePath(), reverseKey, url, timestamp, extensionName, !gzip);
outputRelativePath = outputFullPath.substring(0, outputFullPath.lastIndexOf(File.separator) - 1);
filename = content.getMetadata().get(Metadata.DATE) + "." + extensionName;
} else {
String md5Ofurl = DumpFileUtil.getUrlMD5(url);
String fullDir = DumpFileUtil.createTwoLevelsDirectory(outputDir.getAbsolutePath(), md5Ofurl, !gzip);
filename = DumpFileUtil.createFileName(md5Ofurl, baseName, extensionName);
outputFullPath = String.format("%s/%s", fullDir, filename);
String[] fullPathLevels = fullDir.split(Pattern.quote(File.separator));
String firstLevelDirName = fullPathLevels[fullPathLevels.length - 2];
String secondLevelDirName = fullPathLevels[fullPathLevels.length - 1];
outputRelativePath = firstLevelDirName + secondLevelDirName;
}
}
// Encode all filetypes if no mimetypes have been given
Boolean filter = (mimeTypes == null);
String jsonData = "";
try {
String mimeType = new Tika().detect(content.getContent());
// Maps file to JSON-based structure
// there may be duplicates, so using set
Set<String> inUrls = null;
if (linkDbReader != null) {
Inlinks inlinks = linkDbReader.getInlinks((Text) key);
if (inlinks != null) {
Iterator<Inlink> iterator = inlinks.iterator();
inUrls = new LinkedHashSet<>();
while (inUrls.size() <= MAX_INLINKS && iterator.hasNext()) {
inUrls.add(iterator.next().getFromUrl());
}
}
}
// TODO: Make this Jackson Format implementation reusable
try (CommonCrawlFormat format = CommonCrawlFormatFactory.getCommonCrawlFormat(warc ? "WARC" : "JACKSON", nutchConfig, config)) {
if (inUrls != null) {
format.setInLinks(new ArrayList<>(inUrls));
}
jsonData = format.getJsonData(url, content, metadata);
}
collectStats(typeCounts, mimeType);
// collects statistics for the given mimetypes
if ((mimeType != null) && (mimeTypes != null) && Arrays.asList(mimeTypes).contains(mimeType)) {
collectStats(filteredCounts, mimeType);
filter = true;
}
} catch (IOException ioe) {
LOG.error("Fatal error in creating JSON data: " + ioe.getMessage());
return;
}
if (!warc) {
if (filter) {
byte[] byteData = serializeCBORData(jsonData);
if (!gzip) {
File outputFile = new File(outputFullPath);
if (outputFile.exists()) {
LOG.info("Skipping writing: [" + outputFullPath + "]: file already exists");
} else {
LOG.info("Writing: [" + outputFullPath + "]");
IOUtils.copy(new ByteArrayInputStream(byteData), new FileOutputStream(outputFile));
}
} else {
if (fileList.contains(outputFullPath)) {
LOG.info("Skipping compressing: [" + outputFullPath + "]: file already exists");
} else {
fileList.add(outputFullPath);
LOG.info("Compressing: [" + outputFullPath + "]");
// TarArchiveEntry tarEntry = new TarArchiveEntry(firstLevelDirName + File.separator + secondLevelDirName + File.separator + filename);
TarArchiveEntry tarEntry = new TarArchiveEntry(outputRelativePath + File.separator + filename);
tarEntry.setSize(byteData.length);
tarOutput.putArchiveEntry(tarEntry);
tarOutput.write(byteData);
tarOutput.closeArchiveEntry();
}
}
}
}
}
reader.close();
} catch (Exception e) {
LOG.warn("SKIPPED: {} Because : {}", segmentPart, e.getMessage());
} finally {
fs.close();
}
}
if (gzip && !warc) {
closeStream();
}
if (!typeCounts.isEmpty()) {
LOG.info("CommonsCrawlDataDumper File Stats: " + DumpFileUtil.displayFileTypes(typeCounts, filteredCounts));
}
}
Also used :
Configuration(org.apache.hadoop.conf.Configuration)
NutchConfiguration(org.apache.nutch.util.NutchConfiguration)
HashMap(java.util.HashMap)
ArrayList(java.util.ArrayList)
Metadata(org.apache.nutch.metadata.Metadata)
LinkDbReader(org.apache.nutch.crawl.LinkDbReader)
Writable(org.apache.hadoop.io.Writable)
LinkDbReader(org.apache.nutch.crawl.LinkDbReader)
Tika(org.apache.tika.Tika)
SequenceFile(org.apache.hadoop.io.SequenceFile)
FileSystem(org.apache.hadoop.fs.FileSystem)
Path(org.apache.hadoop.fs.Path)
LocatedFileStatus(org.apache.hadoop.fs.LocatedFileStatus)
Inlinks(org.apache.nutch.crawl.Inlinks)
IOException(java.io.IOException)
Inlink(org.apache.nutch.crawl.Inlink)
TarArchiveEntry(org.apache.commons.compress.archivers.tar.TarArchiveEntry)
ParseException(java.text.ParseException)
MalformedURLException(java.net.MalformedURLException)
IOException(java.io.IOException)
ByteArrayInputStream(java.io.ByteArrayInputStream)
Content(org.apache.nutch.protocol.Content)
FileOutputStream(java.io.FileOutputStream)
ParseException(java.text.ParseException)
SimpleDateFormat(com.ibm.icu.text.SimpleDateFormat)
SequenceFile(org.apache.hadoop.io.SequenceFile)
File(java.io.File)
Aggregations
Tika (org.apache.tika.Tika)54
Test (org.junit.Test)32
Metadata (org.apache.tika.metadata.Metadata)29
ByteArrayInputStream (java.io.ByteArrayInputStream)14
TikaTest (org.apache.tika.TikaTest)12
TikaConfig (org.apache.tika.config.TikaConfig)12
File (java.io.File)8
InputStream (java.io.InputStream)7
URL (java.net.URL)6
TikaInputStream (org.apache.tika.io.TikaInputStream)5
IOException (java.io.IOException)4
HashSet (java.util.HashSet)4
Ignore (org.junit.Ignore)4
FileInputStream (java.io.FileInputStream)3
ArrayList (java.util.ArrayList)3
HashMap (java.util.HashMap)3
Content (org.apache.nutch.protocol.Content)3
Before (org.junit.Before)3
FileOutputStream (java.io.FileOutputStream)2
UnsupportedEncodingException (java.io.UnsupportedEncodingException)2
