Examples with TypedBytesOutput org.apache.hadoop.typedbytes.TypedBytesOutput used on opensource projects

Example 1 with TypedBytesOutput

use of org.apache.hadoop.typedbytes.TypedBytesOutput in project hadoop by apache.

the class TypedBytesMapApp method go.

public void go() throws IOException {
    TypedBytesInput tbinput = new TypedBytesInput(new DataInputStream(System.in));
    TypedBytesOutput tboutput = new TypedBytesOutput(new DataOutputStream(System.out));
    Object key = tbinput.readRaw();
    while (key != null) {
        Object value = tbinput.read();
        for (String part : value.toString().split(find)) {
            // write key
            tboutput.write(part);
            // write value
            tboutput.write(1);
        }
        System.err.println("reporter:counter:UserCounters,InputLines,1");
        key = tbinput.readRaw();
    }
    System.out.flush();
}

Example 2 with TypedBytesOutput

use of org.apache.hadoop.typedbytes.TypedBytesOutput in project hadoop by apache.

the class TypedBytesReduceApp method go.

public void go() throws IOException {
    TypedBytesInput tbinput = new TypedBytesInput(new DataInputStream(System.in));
    TypedBytesOutput tboutput = new TypedBytesOutput(new DataOutputStream(System.out));
    Object prevKey = null;
    int sum = 0;
    Object key = tbinput.read();
    while (key != null) {
        if (prevKey != null && !key.equals(prevKey)) {
            // write key
            tboutput.write(prevKey);
            // write value
            tboutput.write(sum);
            sum = 0;
        }
        sum += (Integer) tbinput.read();
        prevKey = key;
        key = tbinput.read();
    }
    tboutput.write(prevKey);
    tboutput.write(sum);
    System.out.flush();
}

Example 3 with TypedBytesOutput

use of org.apache.hadoop.typedbytes.TypedBytesOutput in project hadoop by apache.

the class TestLoadTypedBytes method testLoading.

@Test
public void testLoading() throws Exception {
    Configuration conf = new Configuration();
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    FileSystem fs = cluster.getFileSystem();
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    TypedBytesOutput tboutput = new TypedBytesOutput(new DataOutputStream(out));
    for (int i = 0; i < 100; i++) {
        // key
        tboutput.write(new Long(i));
        // value
        tboutput.write("" + (10 * i));
    }
    InputStream isBackup = System.in;
    ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
    System.setIn(in);
    LoadTypedBytes loadtb = new LoadTypedBytes(conf);
    try {
        Path root = new Path("/typedbytestest");
        assertTrue(fs.mkdirs(root));
        assertTrue(fs.exists(root));
        String[] args = new String[1];
        args[0] = "/typedbytestest/test.seq";
        int ret = loadtb.run(args);
        assertEquals("Return value != 0.", 0, ret);
        Path file = new Path(root, "test.seq");
        assertTrue(fs.exists(file));
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);
        int counter = 0;
        TypedBytesWritable key = new TypedBytesWritable();
        TypedBytesWritable value = new TypedBytesWritable();
        while (reader.next(key, value)) {
            assertEquals(Long.class, key.getValue().getClass());
            assertEquals(String.class, value.getValue().getClass());
            assertTrue("Invalid record.", Integer.parseInt(value.toString()) % 10 == 0);
            counter++;
        }
        assertEquals("Wrong number of records.", 100, counter);
    } finally {
        try {
            fs.close();
        } catch (Exception e) {
        }
        System.setIn(isBackup);
        cluster.shutdown();
    }
}

Example 4 with TypedBytesOutput

use of org.apache.hadoop.typedbytes.TypedBytesOutput in project hadoop by apache.

the class TypedBytesInputWriter method initialize.

@Override
public void initialize(PipeMapRed pipeMapRed) throws IOException {
    super.initialize(pipeMapRed);
    DataOutput clientOut = pipeMapRed.getClientOutput();
    tbOut = new TypedBytesOutput(clientOut);
    tbwOut = new TypedBytesWritableOutput(clientOut);
}