Examples with TeraValidate org.apache.hadoop.examples.terasort.TeraValidate used on opensource projects

Example 1 with TeraValidate

use of org.apache.hadoop.examples.terasort.TeraValidate in project ignite by apache.

the class HadoopTeraSortTest method teraValidate.

/**
     * Implements validation phase of the sample.
     * @throws Exception
     */
private void teraValidate() throws Exception {
    System.out.println("TeraValidate ===============================================================");
    getFileSystem().delete(new Path(validateOutDir), true);
    // Generate input data:
    int res = ToolRunner.run(new Configuration(), new TeraValidate(), new String[] { "-Dmapreduce.framework.name=local", sortOutDir, validateOutDir });
    assertEquals(0, res);
    FileStatus[] fileStatuses = getFileSystem().listStatus(new Path(validateOutDir), new PathFilter() {

        @Override
        public boolean accept(Path path) {
            // Typically name is "part-r-00000":
            return path.getName().startsWith("part-r-");
        }
    });
    // TeraValidate has only 1 reduce, so should be only 1 result file:
    assertEquals(1, fileStatuses.length);
    // The result file must contain only 1 line with the checksum, like this:
    // "checksum        7a27e2d0d55de",
    // typically it has length of 23 bytes.
    // If sorting was not correct, the result contains list of K-V pairs that are not ordered correctly.
    // In such case the size of the output will be much larger.
    long len = fileStatuses[0].getLen();
    assertTrue("TeraValidate length: " + len, len >= 16 && len <= 32);
}