Examples with LinkDatum org.apache.nutch.scoring.webgraph.LinkDatum used on opensource projects

Example 1 with LinkDatum

use of org.apache.nutch.scoring.webgraph.LinkDatum in project nutch by apache.

the class LinkReader method read.

@Override
public List read(String path) throws FileNotFoundException {
    List<HashMap> rows = new ArrayList<>();
    Path file = new Path(path);
    SequenceFile.Reader reader;
    try {
        reader = new SequenceFile.Reader(conf, Reader.file(file));
        Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
        LinkDatum value = new LinkDatum();
        while (reader.next(key, value)) {
            try {
                HashMap<String, String> t_row = getLinksRow(key, value);
                rows.add(t_row);
            } catch (Exception e) {
            }
        }
        reader.close();
    } catch (FileNotFoundException fne) {
        throw new FileNotFoundException();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        LOG.error("Error occurred while reading file {} : ", file, StringUtils.stringifyException(e));
        throw new WebApplicationException();
    }
    return rows;
}

Example 2 with LinkDatum

use of org.apache.nutch.scoring.webgraph.LinkDatum in project nutch by apache.

the class LinkReader method head.

@Override
public List head(String path, int nrows) throws FileNotFoundException {
    List<HashMap> rows = new ArrayList<>();
    Path file = new Path(path);
    SequenceFile.Reader reader;
    try {
        reader = new SequenceFile.Reader(conf, Reader.file(file));
        Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
        LinkDatum value = new LinkDatum();
        int i = 0;
        while (reader.next(key, value) && i < nrows) {
            HashMap<String, String> t_row = getLinksRow(key, value);
            rows.add(t_row);
            i++;
        }
        reader.close();
    } catch (FileNotFoundException fne) {
        throw new FileNotFoundException();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        LOG.error("Error occurred while reading file {} : ", file, StringUtils.stringifyException(e));
        throw new WebApplicationException();
    }
    return rows;
}

Example 3 with LinkDatum

use of org.apache.nutch.scoring.webgraph.LinkDatum in project nutch by apache.

the class LinkReader method slice.

@Override
public List slice(String path, int start, int end) throws FileNotFoundException {
    List<HashMap> rows = new ArrayList<>();
    Path file = new Path(path);
    SequenceFile.Reader reader;
    try {
        reader = new SequenceFile.Reader(conf, Reader.file(file));
        Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
        LinkDatum value = new LinkDatum();
        int i = 0;
        // increment to read start position
        for (; i < start && reader.next(key, value); i++) {
        }
        while (reader.next(key, value) && i < end) {
            HashMap<String, String> t_row = getLinksRow(key, value);
            rows.add(t_row);
            i++;
        }
        reader.close();
    } catch (FileNotFoundException fne) {
        throw new FileNotFoundException();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        LOG.error("Error occurred while reading file {} : ", file, StringUtils.stringifyException(e));
        throw new WebApplicationException();
    }
    return rows;
}