Examples with DataOutputStream java.io.DataOutputStream used on opensource projects

Example 36 with DataOutputStream

use of java.io.DataOutputStream in project hadoop by apache.

the class TestJoinTupleWritable method testWideWritable2.

@Test
public void testWideWritable2() throws Exception {
    Writable[] manyWrits = makeRandomWritables(71);
    TupleWritable sTuple = new TupleWritable(manyWrits);
    for (int i = 0; i < manyWrits.length; i++) {
        sTuple.setWritten(i);
    }
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    sTuple.write(new DataOutputStream(out));
    ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
    TupleWritable dTuple = new TupleWritable();
    dTuple.readFields(new DataInputStream(in));
    assertTrue("Failed to write/read tuple", sTuple.equals(dTuple));
    assertEquals("All tuple data has not been read from the stream", -1, in.read());
}

Example 37 with DataOutputStream

use of java.io.DataOutputStream in project hadoop by apache.

the class TestJoinTupleWritable method testWritable.

@Test
public void testWritable() throws Exception {
    Random r = new Random();
    Writable[] writs = { new BooleanWritable(r.nextBoolean()), new FloatWritable(r.nextFloat()), new FloatWritable(r.nextFloat()), new IntWritable(r.nextInt()), new LongWritable(r.nextLong()), new BytesWritable("dingo".getBytes()), new LongWritable(r.nextLong()), new IntWritable(r.nextInt()), new BytesWritable("yak".getBytes()), new IntWritable(r.nextInt()) };
    TupleWritable sTuple = makeTuple(writs);
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    sTuple.write(new DataOutputStream(out));
    ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
    TupleWritable dTuple = new TupleWritable();
    dTuple.readFields(new DataInputStream(in));
    assertTrue("Failed to write/read tuple", sTuple.equals(dTuple));
}

Example 38 with DataOutputStream

use of java.io.DataOutputStream in project hadoop by apache.

the class TestDelegatingInputFormat method getPath.

static Path getPath(final String location, final FileSystem fs) throws IOException {
    Path path = new Path(location);
    // create a multi-block file on hdfs
    DataOutputStream out = fs.create(path, true, 4096, (short) 2, 512, null);
    for (int i = 0; i < 1000; ++i) {
        out.writeChars("Hello\n");
    }
    out.close();
    return path;
}

Example 39 with DataOutputStream

use of java.io.DataOutputStream in project hadoop by apache.

the class TestMRAppWithCombiner method createInputOutPutFolder.

static void createInputOutPutFolder(Path inDir, Path outDir, int numMaps) throws Exception {
    FileSystem fs = FileSystem.get(conf);
    if (fs.exists(outDir)) {
        fs.delete(outDir, true);
    }
    if (!fs.exists(inDir)) {
        fs.mkdirs(inDir);
    }
    String input = "The quick brown fox\n" + "has many silly\n" + "red fox sox\n";
    for (int i = 0; i < numMaps; ++i) {
        DataOutputStream file = fs.create(new Path(inDir, "part-" + i));
        file.writeBytes(input);
        file.close();
    }
}

Example 40 with DataOutputStream

use of java.io.DataOutputStream in project hadoop by apache.

the class TestMROldApiJobs method runJob.

static boolean runJob(JobConf conf, Path inDir, Path outDir, int numMaps, int numReds) throws IOException, InterruptedException {
    FileSystem fs = FileSystem.get(conf);
    if (fs.exists(outDir)) {
        fs.delete(outDir, true);
    }
    if (!fs.exists(inDir)) {
        fs.mkdirs(inDir);
    }
    String input = "The quick brown fox\n" + "has many silly\n" + "red fox sox\n";
    for (int i = 0; i < numMaps; ++i) {
        DataOutputStream file = fs.create(new Path(inDir, "part-" + i));
        file.writeBytes(input);
        file.close();
    }
    DistributedCache.addFileToClassPath(TestMRJobs.APP_JAR, conf, fs);
    conf.setOutputCommitter(CustomOutputCommitter.class);
    conf.setInputFormat(TextInputFormat.class);
    conf.setOutputKeyClass(LongWritable.class);
    conf.setOutputValueClass(Text.class);
    FileInputFormat.setInputPaths(conf, inDir);
    FileOutputFormat.setOutputPath(conf, outDir);
    conf.setNumMapTasks(numMaps);
    conf.setNumReduceTasks(numReds);
    JobClient jobClient = new JobClient(conf);
    RunningJob job = jobClient.submitJob(conf);
    return jobClient.monitorAndPrintJob(conf, job);
}