Examples with Writer org.apache.hadoop.io.file.tfile.TFile.Writer used on opensource projects

Example 11 with Writer

use of org.apache.hadoop.io.file.tfile.TFile.Writer in project hadoop by apache.

the class TestTFileComparator2 method testSortedLongWritable.

@Test
public void testSortedLongWritable() throws IOException {
    Configuration conf = new Configuration();
    Path path = new Path(ROOT, name);
    FileSystem fs = path.getFileSystem(conf);
    FSDataOutputStream out = fs.create(path);
    try {
        TFile.Writer writer = new Writer(out, BLOCK_SIZE, "gz", jClassLongWritableComparator, conf);
        try {
            LongWritable key = new LongWritable(0);
            for (long i = 0; i < NENTRY; ++i) {
                key.set(cube(i - NENTRY / 2));
                DataOutputStream dos = writer.prepareAppendKey(-1);
                try {
                    key.write(dos);
                } finally {
                    dos.close();
                }
                dos = writer.prepareAppendValue(-1);
                try {
                    dos.write(buildValue(i).getBytes());
                } finally {
                    dos.close();
                }
            }
        } finally {
            writer.close();
        }
    } finally {
        out.close();
    }
    FSDataInputStream in = fs.open(path);
    try {
        TFile.Reader reader = new TFile.Reader(in, fs.getFileStatus(path).getLen(), conf);
        try {
            TFile.Reader.Scanner scanner = reader.createScanner();
            long i = 0;
            BytesWritable value = new BytesWritable();
            for (; !scanner.atEnd(); scanner.advance()) {
                scanner.entry().getValue(value);
                assertEquals(buildValue(i), new String(value.getBytes(), 0, value.getLength()));
                ++i;
            }
        } finally {
            reader.close();
        }
    } finally {
        in.close();
    }
}

Example 12 with Writer

use of org.apache.hadoop.io.file.tfile.TFile.Writer in project hadoop by apache.

the class TestTFileComparators method testFailureBadComparatorNames.

// bad comparator format
@Test
public void testFailureBadComparatorNames() throws IOException {
    try {
        writer = new Writer(out, BLOCK_SIZE, compression, "badcmp", conf);
        Assert.fail("Failed to catch unsupported comparator names");
    } catch (Exception e) {
        // noop, expecting exceptions
        e.printStackTrace();
    }
}

Example 13 with Writer

use of org.apache.hadoop.io.file.tfile.TFile.Writer in project hadoop by apache.

the class TestTFileComparators method testFailureBadJClassNames.

// jclass that doesn't exist
@Test
public void testFailureBadJClassNames() throws IOException {
    try {
        writer = new Writer(out, BLOCK_SIZE, compression, "jclass: some.non.existence.clazz", conf);
        Assert.fail("Failed to catch unsupported comparator names");
    } catch (Exception e) {
        // noop, expecting exceptions
        e.printStackTrace();
    }
}

Example 14 with Writer

use of org.apache.hadoop.io.file.tfile.TFile.Writer in project hadoop by apache.

the class TestTFileStreams method setUp.

@Before
public void setUp() throws IOException {
    conf = new Configuration();
    path = new Path(ROOT, outputFile);
    fs = path.getFileSystem(conf);
    out = fs.create(path);
    writer = new Writer(out, BLOCK_SIZE, compression, comparator, conf);
}