Examples with Reader org.apache.hadoop.io.SequenceFile.Reader used on opensource projects

Example 11 with Reader

use of org.apache.hadoop.io.SequenceFile.Reader in project nutch by apache.

the class NodeReader method read.

@Override
public List read(String path) throws FileNotFoundException {
    List<HashMap> rows = new ArrayList<>();
    Path file = new Path(path);
    SequenceFile.Reader reader;
    try {
        reader = new SequenceFile.Reader(conf, Reader.file(file));
        Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
        Node value = new Node();
        while (reader.next(key, value)) {
            try {
                HashMap<String, String> t_row = getNodeRow(key, value);
                rows.add(t_row);
            } catch (Exception e) {
            }
        }
        reader.close();
    } catch (FileNotFoundException fne) {
        throw new FileNotFoundException();
    } catch (IOException e) {
        e.printStackTrace();
        LOG.error("Error occurred while reading file {} : {}", file, StringUtils.stringifyException(e));
        throw new WebApplicationException();
    }
    return rows;
}

Example 12 with Reader

use of org.apache.hadoop.io.SequenceFile.Reader in project kylin by apache.

the class HiveToBaseCuboidMapperPerformanceTest method test.

@Ignore("convenient trial tool for dev")
@Test
public void test() throws IOException, InterruptedException {
    Configuration hconf = HadoopUtil.getCurrentConfiguration();
    HiveToBaseCuboidMapper mapper = new HiveToBaseCuboidMapper();
    Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null);
    mapper.doSetup(context);
    Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
    Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
    Text value = new Text();
    while (reader.next(key, value)) {
        mapper.map(key, value, context);
    }
    reader.close();
}

Example 13 with Reader

use of org.apache.hadoop.io.SequenceFile.Reader in project geowave by locationtech.

the class GeoWaveNNIT method readFile.

private int readFile() throws IllegalArgumentException, IOException {
    int count = 0;
    final FileSystem fs = FileSystem.get(MapReduceTestUtils.getConfiguration());
    final FileStatus[] fss = fs.listStatus(new Path(TestUtils.TEMP_DIR + File.separator + MapReduceTestEnvironment.HDFS_BASE_DIRECTORY + "/t1/pairs"));
    for (final FileStatus ifs : fss) {
        if (ifs.isFile() && ifs.getPath().toString().matches(".*part-r-0000[0-9]")) {
            try (SequenceFile.Reader reader = new SequenceFile.Reader(MapReduceTestUtils.getConfiguration(), Reader.file(ifs.getPath()))) {
                final Text key = new Text();
                final Text val = new Text();
                while (reader.next(key, val)) {
                    count++;
                }
            }
        }
    }
    return count;
}

Example 14 with Reader

use of org.apache.hadoop.io.SequenceFile.Reader in project circus-train by ExpediaGroup.

the class CircusTrainCopyListingTest method typical.

@Test
public void typical() throws IOException {
    File input = temp.newFolder("input");
    File inputSub2 = new File(input, "sub1/sub2");
    inputSub2.mkdirs();
    Files.asCharSink(new File(inputSub2, "data"), UTF_8).write("test1");
    File listFile = temp.newFile("listFile");
    Path pathToListFile = new Path(listFile.toURI());
    List<Path> sourceDataLocations = new ArrayList<>();
    sourceDataLocations.add(new Path(inputSub2.toURI()));
    DistCpOptions options = new DistCpOptions(sourceDataLocations, new Path("dummy"));
    CircusTrainCopyListing.setRootPath(conf, new Path(input.toURI()));
    CircusTrainCopyListing copyListing = new CircusTrainCopyListing(conf, null);
    copyListing.doBuildListing(pathToListFile, options);
    try (Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(pathToListFile))) {
        Text key = new Text();
        CopyListingFileStatus value = new CopyListingFileStatus();
        assertTrue(reader.next(key, value));
        assertThat(key.toString(), is("/sub1/sub2"));
        assertThat(value.getPath().toUri().toString(), endsWith("/input/sub1/sub2"));
        assertTrue(reader.next(key, value));
        assertThat(key.toString(), is("/sub1/sub2/data"));
        assertThat(value.getPath().toUri().toString(), endsWith("/input/sub1/sub2/data"));
        assertFalse(reader.next(key, value));
    }
}

Example 15 with Reader

use of org.apache.hadoop.io.SequenceFile.Reader in project nutch by apache.

the class LinkReader method slice.

@Override
public List slice(String path, int start, int end) throws FileNotFoundException {
    List<HashMap> rows = new ArrayList<>();
    Path file = new Path(path);
    SequenceFile.Reader reader;
    try {
        reader = new SequenceFile.Reader(conf, Reader.file(file));
        Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
        LinkDatum value = new LinkDatum();
        int i = 0;
        // increment to read start position
        for (; i < start && reader.next(key, value); i++) {
        }
        while (reader.next(key, value) && i < end) {
            HashMap<String, String> t_row = getLinksRow(key, value);
            rows.add(t_row);
            i++;
        }
        reader.close();
    } catch (FileNotFoundException fne) {
        throw new FileNotFoundException();
    } catch (IOException e) {
        e.printStackTrace();
        LOG.error("Error occurred while reading file {} : {}", file, StringUtils.stringifyException(e));
        throw new WebApplicationException();
    }
    return rows;
}