Examples with ChecksumOpt org.apache.hadoop.fs.Options.ChecksumOpt used on opensource projects

Example 1 with ChecksumOpt

use of org.apache.hadoop.fs.Options.ChecksumOpt in project hadoop by apache.

the class TestCopyMapper method createSourceDataWithDifferentChecksumType.

private static void createSourceDataWithDifferentChecksumType() throws Exception {
    mkdirs(SOURCE_PATH + "/1");
    mkdirs(SOURCE_PATH + "/2");
    mkdirs(SOURCE_PATH + "/2/3/4");
    mkdirs(SOURCE_PATH + "/2/3");
    mkdirs(SOURCE_PATH + "/5");
    touchFile(SOURCE_PATH + "/5/6", new ChecksumOpt(DataChecksum.Type.CRC32, 512));
    mkdirs(SOURCE_PATH + "/7");
    mkdirs(SOURCE_PATH + "/7/8");
    touchFile(SOURCE_PATH + "/7/8/9", new ChecksumOpt(DataChecksum.Type.CRC32C, 512));
}

Example 2 with ChecksumOpt

use of org.apache.hadoop.fs.Options.ChecksumOpt in project hadoop by apache.

the class TestDistributedFileSystem method testCreateWithCustomChecksum.

@Test
public void testCreateWithCustomChecksum() throws Exception {
    Configuration conf = getTestConfiguration();
    MiniDFSCluster cluster = null;
    Path testBasePath = new Path("/test/csum");
    // create args 
    Path path1 = new Path(testBasePath, "file_wtih_crc1");
    Path path2 = new Path(testBasePath, "file_with_crc2");
    ChecksumOpt opt1 = new ChecksumOpt(DataChecksum.Type.CRC32C, 512);
    ChecksumOpt opt2 = new ChecksumOpt(DataChecksum.Type.CRC32, 512);
    // common args
    FsPermission perm = FsPermission.getDefault().applyUMask(FsPermission.getUMask(conf));
    EnumSet<CreateFlag> flags = EnumSet.of(CreateFlag.OVERWRITE, CreateFlag.CREATE);
    short repl = 1;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
        FileSystem dfs = cluster.getFileSystem();
        dfs.mkdirs(testBasePath);
        // create two files with different checksum types
        FSDataOutputStream out1 = dfs.create(path1, perm, flags, 4096, repl, 131072L, null, opt1);
        FSDataOutputStream out2 = dfs.create(path2, perm, flags, 4096, repl, 131072L, null, opt2);
        for (int i = 0; i < 1024; i++) {
            out1.write(i);
            out2.write(i);
        }
        out1.close();
        out2.close();
        // the two checksums must be different.
        MD5MD5CRC32FileChecksum sum1 = (MD5MD5CRC32FileChecksum) dfs.getFileChecksum(path1);
        MD5MD5CRC32FileChecksum sum2 = (MD5MD5CRC32FileChecksum) dfs.getFileChecksum(path2);
        assertFalse(sum1.equals(sum2));
        // check the individual params
        assertEquals(DataChecksum.Type.CRC32C, sum1.getCrcType());
        assertEquals(DataChecksum.Type.CRC32, sum2.getCrcType());
    } finally {
        if (cluster != null) {
            cluster.getFileSystem().delete(testBasePath, true);
            cluster.shutdown();
        }
    }
}

Example 3 with ChecksumOpt

use of org.apache.hadoop.fs.Options.ChecksumOpt in project hadoop by apache.

the class DfsClientConf method createChecksum.

/** create a DataChecksum with the given option. */
public DataChecksum createChecksum(ChecksumOpt userOpt) {
    // Fill in any missing field with the default.
    ChecksumOpt opt = ChecksumOpt.processChecksumOpt(defaultChecksumOpt, userOpt);
    DataChecksum dataChecksum = DataChecksum.newDataChecksum(opt.getChecksumType(), opt.getBytesPerChecksum());
    if (dataChecksum == null) {
        throw new HadoopIllegalArgumentException("Invalid checksum type: userOpt=" + userOpt + ", default=" + defaultChecksumOpt + ", effective=null");
    }
    return dataChecksum;
}

Example 4 with ChecksumOpt

use of org.apache.hadoop.fs.Options.ChecksumOpt in project hadoop by apache.

the class TestFsOptions method testProcessChecksumOpt.

@Test
public void testProcessChecksumOpt() {
    ChecksumOpt defaultOpt = new ChecksumOpt(DataChecksum.Type.CRC32, 512);
    ChecksumOpt finalOpt;
    // Give a null 
    finalOpt = ChecksumOpt.processChecksumOpt(defaultOpt, null);
    checkParams(defaultOpt, finalOpt);
    // null with bpc
    finalOpt = ChecksumOpt.processChecksumOpt(defaultOpt, null, 1024);
    checkParams(DataChecksum.Type.CRC32, 1024, finalOpt);
    ChecksumOpt myOpt = new ChecksumOpt();
    // custom with unspecified parameters
    finalOpt = ChecksumOpt.processChecksumOpt(defaultOpt, myOpt);
    checkParams(defaultOpt, finalOpt);
    myOpt = new ChecksumOpt(DataChecksum.Type.CRC32C, 2048);
    // custom config
    finalOpt = ChecksumOpt.processChecksumOpt(defaultOpt, myOpt);
    checkParams(DataChecksum.Type.CRC32C, 2048, finalOpt);
    // custom config + bpc
    finalOpt = ChecksumOpt.processChecksumOpt(defaultOpt, myOpt, 4096);
    checkParams(DataChecksum.Type.CRC32C, 4096, finalOpt);
}

Example 5 with ChecksumOpt

use of org.apache.hadoop.fs.Options.ChecksumOpt in project hadoop by apache.

the class AbstractFileSystem method create.

/**
   * The specification of this method matches that of
   * {@link FileContext#create(Path, EnumSet, Options.CreateOpts...)} except
   * that the Path f must be fully qualified and the permission is absolute
   * (i.e. umask has been applied).
   */
public final FSDataOutputStream create(final Path f, final EnumSet<CreateFlag> createFlag, Options.CreateOpts... opts) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, UnresolvedLinkException, IOException {
    checkPath(f);
    int bufferSize = -1;
    short replication = -1;
    long blockSize = -1;
    int bytesPerChecksum = -1;
    ChecksumOpt checksumOpt = null;
    FsPermission permission = null;
    Progressable progress = null;
    Boolean createParent = null;
    for (CreateOpts iOpt : opts) {
        if (CreateOpts.BlockSize.class.isInstance(iOpt)) {
            if (blockSize != -1) {
                throw new HadoopIllegalArgumentException("BlockSize option is set multiple times");
            }
            blockSize = ((CreateOpts.BlockSize) iOpt).getValue();
        } else if (CreateOpts.BufferSize.class.isInstance(iOpt)) {
            if (bufferSize != -1) {
                throw new HadoopIllegalArgumentException("BufferSize option is set multiple times");
            }
            bufferSize = ((CreateOpts.BufferSize) iOpt).getValue();
        } else if (CreateOpts.ReplicationFactor.class.isInstance(iOpt)) {
            if (replication != -1) {
                throw new HadoopIllegalArgumentException("ReplicationFactor option is set multiple times");
            }
            replication = ((CreateOpts.ReplicationFactor) iOpt).getValue();
        } else if (CreateOpts.BytesPerChecksum.class.isInstance(iOpt)) {
            if (bytesPerChecksum != -1) {
                throw new HadoopIllegalArgumentException("BytesPerChecksum option is set multiple times");
            }
            bytesPerChecksum = ((CreateOpts.BytesPerChecksum) iOpt).getValue();
        } else if (CreateOpts.ChecksumParam.class.isInstance(iOpt)) {
            if (checksumOpt != null) {
                throw new HadoopIllegalArgumentException("CreateChecksumType option is set multiple times");
            }
            checksumOpt = ((CreateOpts.ChecksumParam) iOpt).getValue();
        } else if (CreateOpts.Perms.class.isInstance(iOpt)) {
            if (permission != null) {
                throw new HadoopIllegalArgumentException("Perms option is set multiple times");
            }
            permission = ((CreateOpts.Perms) iOpt).getValue();
        } else if (CreateOpts.Progress.class.isInstance(iOpt)) {
            if (progress != null) {
                throw new HadoopIllegalArgumentException("Progress option is set multiple times");
            }
            progress = ((CreateOpts.Progress) iOpt).getValue();
        } else if (CreateOpts.CreateParent.class.isInstance(iOpt)) {
            if (createParent != null) {
                throw new HadoopIllegalArgumentException("CreateParent option is set multiple times");
            }
            createParent = ((CreateOpts.CreateParent) iOpt).getValue();
        } else {
            throw new HadoopIllegalArgumentException("Unkown CreateOpts of type " + iOpt.getClass().getName());
        }
    }
    if (permission == null) {
        throw new HadoopIllegalArgumentException("no permission supplied");
    }
    FsServerDefaults ssDef = getServerDefaults();
    if (ssDef.getBlockSize() % ssDef.getBytesPerChecksum() != 0) {
        throw new IOException("Internal error: default blockSize is" + " not a multiple of default bytesPerChecksum ");
    }
    if (blockSize == -1) {
        blockSize = ssDef.getBlockSize();
    }
    // Create a checksum option honoring user input as much as possible.
    // If bytesPerChecksum is specified, it will override the one set in
    // checksumOpt. Any missing value will be filled in using the default.
    ChecksumOpt defaultOpt = new ChecksumOpt(ssDef.getChecksumType(), ssDef.getBytesPerChecksum());
    checksumOpt = ChecksumOpt.processChecksumOpt(defaultOpt, checksumOpt, bytesPerChecksum);
    if (bufferSize == -1) {
        bufferSize = ssDef.getFileBufferSize();
    }
    if (replication == -1) {
        replication = ssDef.getReplication();
    }
    if (createParent == null) {
        createParent = false;
    }
    if (blockSize % bytesPerChecksum != 0) {
        throw new HadoopIllegalArgumentException("blockSize should be a multiple of checksumsize");
    }
    return this.createInternal(f, createFlag, permission, bufferSize, replication, blockSize, progress, checksumOpt, createParent);
}