Examples with MD5MD5CRC32FileChecksum org.apache.hadoop.fs.MD5MD5CRC32FileChecksum used on opensource projects

Example 6 with MD5MD5CRC32FileChecksum

use of org.apache.hadoop.fs.MD5MD5CRC32FileChecksum in project kylo by Teradata.

the class ComputeHDFSChecksumsTest method testMultipleFilesWithDirectoryDefined.

@Test
public void testMultipleFilesWithDirectoryDefined() throws Exception {
    String fileName = "000000_0";
    String fileName2 = "000000_1";
    String fileName3 = "000000_2";
    String directory = "/dropzone";
    Mockito.when(fileSystem.getFileChecksum(any(Path.class))).thenReturn(new MD5MD5CRC32FileChecksum(0, 512, new MD5Hash("112233445566778899aabbccddeeff00"))).thenReturn(new MD5MD5CRC32FileChecksum(0, 512, new MD5Hash("112233445566778899aabbccddeeff01"))).thenReturn(new MD5MD5CRC32FileChecksum(0, 512, new MD5Hash("112233445566778899aabbccddeeff02")));
    runner.setProperty(ComputeHDFSChecksums.DIRECTORY, directory);
    runner.setProperty(ComputeHDFSChecksums.FAIL_IF_INCORRECT_CHECKSUM, "True");
    runner.setProperty(ComputeHDFSChecksums.FILES, String.format("[" + fileEntry + "," + fileEntry + "," + fileEntry + "]", fileName, "AAAAAAAAAAAAAAIAESIzRFVmd4iZqrvM3e7/AAAAAAA=", fileName2, "AAAAAAAAAAAAAAIAESIzRFVmd4iZqrvM3e7/AQAAAAA=", fileName3, "AAAAAAAAAAAAAAIAESIzRFVmd4iZqrvM3e7/AgAAAAA="));
    runner.enqueue(new byte[0]);
    runner.run();
    // Check relationships
    Assert.assertEquals(0, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_FAILURE).size());
    Assert.assertEquals(1, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_SUCCESS).size());
    // Check file system calls
    InOrder inOrder = Mockito.inOrder(fileSystem);
    inOrder.verify(fileSystem).getFileChecksum(new Path(directory, fileName));
    inOrder.verify(fileSystem).getFileChecksum(new Path(directory, fileName2));
    inOrder.verify(fileSystem).getFileChecksum(new Path(directory, fileName3));
    inOrder.verifyNoMoreInteractions();
}

Example 7 with MD5MD5CRC32FileChecksum

use of org.apache.hadoop.fs.MD5MD5CRC32FileChecksum in project hadoop by apache.

the class WebHdfsFileSystem method getFileChecksum.

@Override
public MD5MD5CRC32FileChecksum getFileChecksum(final Path p) throws IOException {
    statistics.incrementReadOps(1);
    storageStatistics.incrementOpCounter(OpType.GET_FILE_CHECKSUM);
    final HttpOpParam.Op op = GetOpParam.Op.GETFILECHECKSUM;
    return new FsPathResponseRunner<MD5MD5CRC32FileChecksum>(op, p) {

        @Override
        MD5MD5CRC32FileChecksum decodeResponse(Map<?, ?> json) throws IOException {
            return JsonUtilClient.toMD5MD5CRC32FileChecksum(json);
        }
    }.run();
}

Example 8 with MD5MD5CRC32FileChecksum

use of org.apache.hadoop.fs.MD5MD5CRC32FileChecksum in project kylo by Teradata.

the class ComputeHDFSChecksumsTest method testSingleFileInListFailOnWrongChecksum.

@Test
public void testSingleFileInListFailOnWrongChecksum() throws Exception {
    String fileName = "000000_0";
    Mockito.doReturn(new MD5MD5CRC32FileChecksum(0, 512, new MD5Hash("112233445566778899aabbccddeeff00"))).when(fileSystem).getFileChecksum(any(Path.class));
    runner.setProperty(ComputeHDFSChecksums.FAIL_IF_INCORRECT_CHECKSUM, "True");
    runner.setProperty(ComputeHDFSChecksums.FILES, String.format("[" + fileEntry + "]", fileName, "AAACAAAAAAAAAAAArRnBpxcZ9ze14XqfLMB4yA=="));
    runner.enqueue(new byte[0]);
    runner.run();
    // Check relationships
    Assert.assertEquals(1, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_FAILURE).size());
    Assert.assertEquals(0, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_SUCCESS).size());
    // Check file system calls
    verifyGetFileChecksumCall(fileName);
}

Example 9 with MD5MD5CRC32FileChecksum

use of org.apache.hadoop.fs.MD5MD5CRC32FileChecksum in project kylo by Teradata.

the class ComputeHDFSChecksumsTest method testMultipleFilesFailOnSingleWrongChecksum.

@Test
public void testMultipleFilesFailOnSingleWrongChecksum() throws Exception {
    String fileName = "000000_0";
    String fileName2 = "000000_1";
    String fileName3 = "000000_2";
    Mockito.when(fileSystem.getFileChecksum(any(Path.class))).thenReturn(new MD5MD5CRC32FileChecksum(0, 512, new MD5Hash("112233445566778899aabbccddeeff00"))).thenReturn(new MD5MD5CRC32FileChecksum(0, 512, new MD5Hash("112233445566778899aabbccddeeff01"))).thenReturn(new MD5MD5CRC32FileChecksum(0, 512, new MD5Hash("112233445566778899aabbccddeeff02")));
    runner.setProperty(ComputeHDFSChecksums.FAIL_IF_INCORRECT_CHECKSUM, "True");
    runner.setProperty(ComputeHDFSChecksums.FILES, String.format("[" + fileEntry + "," + fileEntry + "," + fileEntry + "]", fileName, "AAAAAAAAAAAAAAIAESIzRFVmd4iZqrvM3e7/AAAAAAA=", fileName2, "AAAAAAAAAAAAAAIAESIzRFVmd4iZqrvM3e7/AAAAAAA=", fileName3, "AAAAAAAAAAAAAAIAESIzRFVmd4iZqrvM3e7/AgAAAAA="));
    runner.enqueue(new byte[0]);
    runner.run();
    // Check relationships
    Assert.assertEquals(1, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_FAILURE).size());
    Assert.assertEquals(0, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_SUCCESS).size());
    // Check file system calls
    InOrder inOrder = Mockito.inOrder(fileSystem);
    inOrder.verify(fileSystem).getFileChecksum(new Path(fileName));
    inOrder.verify(fileSystem).getFileChecksum(new Path(fileName2));
    inOrder.verifyNoMoreInteractions();
}

Example 10 with MD5MD5CRC32FileChecksum

use of org.apache.hadoop.fs.MD5MD5CRC32FileChecksum in project SSM by Intel-bigdata.

the class SmartDFSClient method getFileChecksum.

@Override
public MD5MD5CRC32FileChecksum getFileChecksum(String src, long length) throws IOException {
    MD5MD5CRC32FileChecksum ret = super.getFileChecksum(src, length);
    if (ret.getChecksumOpt().getBytesPerChecksum() == 0) {
        FileState fileState = getFileState(src);
        if (fileState instanceof CompactFileState) {
            try {
                // Get original checksum for small file.
                byte[] bytes = getXAttr(src, SmartConstants.SMART_FILE_CHECKSUM_XATTR_NAME);
                ret = new MD5MD5CRC32FileChecksum();
                ret.readFields(new DataInputStream(new ByteArrayInputStream(bytes)));
            } catch (IOException e) {
                throw new IOException("Failed to get checksum for SSM Small File: " + e.getMessage());
            }
        }
    }
    return ret;
}