Examples with ErasureCodingPolicy org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy used on opensource projects

Example 21 with ErasureCodingPolicy

use of org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy in project SSM by Intel-bigdata.

the class UnErasureCodingAction method execute.

@Override
protected void execute() throws Exception {
    final String MATCH_RESULT = "The current EC policy is replication already.";
    final String DIR_RESULT = "The replication EC policy is set successfully for the given directory.";
    final String CONVERT_RESULT = "The file is converted successfully with replication EC policy.";
    this.setDfsClient(HadoopUtil.getDFSClient(HadoopUtil.getNameNodeUri(conf), conf));
    HdfsFileStatus fileStatus = dfsClient.getFileInfo(srcPath);
    if (fileStatus == null) {
        throw new ActionException("File doesn't exist!");
    }
    ErasureCodingPolicy srcEcPolicy = fileStatus.getErasureCodingPolicy();
    // if ecPolicy is null, it means replication.
    if (srcEcPolicy == null) {
        this.progress = 1.0F;
        appendLog(MATCH_RESULT);
        return;
    }
    if (fileStatus.isDir()) {
        dfsClient.setErasureCodingPolicy(srcPath, ecPolicyName);
        progress = 1.0F;
        appendLog(DIR_RESULT);
        return;
    }
    try {
        convert(fileStatus);
        setAttributes(srcPath, fileStatus, ecTmpPath);
        dfsClient.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE);
        appendLog(CONVERT_RESULT);
        appendLog(String.format("The previous EC policy is %s.", srcEcPolicy.getName()));
        appendLog(String.format("The current EC policy is %s.", REPLICATION_POLICY_NAME));
    } catch (ActionException ex) {
        try {
            if (dfsClient.getFileInfo(ecTmpPath) != null) {
                dfsClient.delete(ecTmpPath, false);
            }
        } catch (IOException e) {
            LOG.error("Failed to delete tmp file created during the conversion!");
        }
        throw new ActionException(ex);
    }
}

Example 22 with ErasureCodingPolicy

use of org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy in project SSM by Intel-bigdata.

the class ErasureCodingAction method execute.

@Override
protected void execute() throws Exception {
    final String MATCH_RESULT = "The current EC policy is already matched with the target one.";
    final String DIR_RESULT = "The EC policy is set successfully for the given directory.";
    final String CONVERT_RESULT = "The file is converted successfully with the given or default EC policy.";
    // Make sure DFSClient is used instead of SmartDFSClient.
    this.setDfsClient(HadoopUtil.getDFSClient(HadoopUtil.getNameNodeUri(conf), conf));
    // keep attribute consistent
    HdfsFileStatus fileStatus = dfsClient.getFileInfo(srcPath);
    if (fileStatus == null) {
        throw new ActionException("File doesn't exist!");
    }
    validateEcPolicy(ecPolicyName);
    ErasureCodingPolicy srcEcPolicy = fileStatus.getErasureCodingPolicy();
    // if the current ecPolicy is already the target one, no need to convert
    if (srcEcPolicy != null) {
        if (srcEcPolicy.getName().equals(ecPolicyName)) {
            appendLog(MATCH_RESULT);
            this.progress = 1.0F;
            return;
        }
    } else {
        // if ecPolicy is null, it means replication.
        if (ecPolicyName.equals(REPLICATION_POLICY_NAME)) {
            appendLog(MATCH_RESULT);
            this.progress = 1.0F;
            return;
        }
    }
    if (fileStatus.isDir()) {
        dfsClient.setErasureCodingPolicy(srcPath, ecPolicyName);
        this.progress = 1.0F;
        appendLog(DIR_RESULT);
        return;
    }
    HdfsDataOutputStream outputStream = null;
    try {
        // a file only with replication policy can be appended.
        if (srcEcPolicy == null) {
            // append the file to acquire the lock to avoid modifying, real appending wouldn't occur.
            outputStream = dfsClient.append(srcPath, bufferSize, EnumSet.of(CreateFlag.APPEND), null, null);
        }
        convert(fileStatus);
        /**
         * The append operation will change the modification time accordingly,
         * so we use the FileStatus obtained before append to set ecTmp file's most attributes
         */
        setAttributes(srcPath, fileStatus, ecTmpPath);
        dfsClient.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE);
        appendLog(CONVERT_RESULT);
        if (srcEcPolicy == null) {
            appendLog("The previous EC policy is replication.");
        } else {
            appendLog(String.format("The previous EC policy is %s.", srcEcPolicy.getName()));
        }
        appendLog(String.format("The current EC policy is %s.", ecPolicyName));
    } catch (ActionException ex) {
        try {
            if (dfsClient.getFileInfo(ecTmpPath) != null) {
                dfsClient.delete(ecTmpPath, false);
            }
        } catch (IOException e) {
            LOG.error("Failed to delete tmp file created during the conversion!");
        }
        throw new ActionException(ex);
    } finally {
        if (outputStream != null) {
            try {
                outputStream.close();
            } catch (IOException ex) {
            // Hide the expected exception that the original file is missing.
            }
        }
    }
}

Example 23 with ErasureCodingPolicy

use of org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy in project hadoop by apache.

the class TestFSImage method testSupportBlockGroup.

/**
   * Ensure that FSImage supports BlockGroup.
   */
@Test(timeout = 60000)
public void testSupportBlockGroup() throws Exception {
    final short GROUP_SIZE = (short) (testECPolicy.getNumDataUnits() + testECPolicy.getNumParityUnits());
    final int BLOCK_SIZE = 8 * 1024 * 1024;
    Configuration conf = new HdfsConfiguration();
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
    DFSTestUtil.enableAllECPolicies(conf);
    MiniDFSCluster cluster = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE).build();
        cluster.waitActive();
        DistributedFileSystem fs = cluster.getFileSystem();
        Path parentDir = new Path("/ec-10-4");
        Path childDir = new Path(parentDir, "ec-3-2");
        ErasureCodingPolicy ec32Policy = ErasureCodingPolicyManager.getPolicyByID(HdfsConstants.RS_3_2_POLICY_ID);
        // Create directories and files
        fs.mkdirs(parentDir);
        fs.mkdirs(childDir);
        fs.setErasureCodingPolicy(parentDir, testECPolicy.getName());
        fs.setErasureCodingPolicy(childDir, ec32Policy.getName());
        Path file_10_4 = new Path(parentDir, "striped_file_10_4");
        Path file_3_2 = new Path(childDir, "striped_file_3_2");
        // Write content to files
        byte[] bytes = StripedFileTestUtil.generateBytes(BLOCK_SIZE);
        DFSTestUtil.writeFile(fs, file_10_4, new String(bytes));
        DFSTestUtil.writeFile(fs, file_3_2, new String(bytes));
        // Save namespace and restart NameNode
        fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
        fs.saveNamespace();
        fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
        cluster.restartNameNodes();
        fs = cluster.getFileSystem();
        assertTrue(fs.exists(file_10_4));
        assertTrue(fs.exists(file_3_2));
        // check the information of file_10_4
        FSNamesystem fsn = cluster.getNamesystem();
        INodeFile inode = fsn.dir.getINode(file_10_4.toString()).asFile();
        assertTrue(inode.isStriped());
        assertEquals(testECPolicy.getId(), inode.getErasureCodingPolicyID());
        BlockInfo[] blks = inode.getBlocks();
        assertEquals(1, blks.length);
        assertTrue(blks[0].isStriped());
        assertEquals(testECPolicy.getId(), fs.getErasureCodingPolicy(file_10_4).getId());
        assertEquals(testECPolicy.getId(), ((BlockInfoStriped) blks[0]).getErasureCodingPolicy().getId());
        assertEquals(testECPolicy.getNumDataUnits(), ((BlockInfoStriped) blks[0]).getDataBlockNum());
        assertEquals(testECPolicy.getNumParityUnits(), ((BlockInfoStriped) blks[0]).getParityBlockNum());
        byte[] content = DFSTestUtil.readFileAsBytes(fs, file_10_4);
        assertArrayEquals(bytes, content);
        // check the information of file_3_2
        inode = fsn.dir.getINode(file_3_2.toString()).asFile();
        assertTrue(inode.isStriped());
        assertEquals(ErasureCodingPolicyManager.getPolicyByID(HdfsConstants.RS_3_2_POLICY_ID).getId(), inode.getErasureCodingPolicyID());
        blks = inode.getBlocks();
        assertEquals(1, blks.length);
        assertTrue(blks[0].isStriped());
        assertEquals(ec32Policy.getId(), fs.getErasureCodingPolicy(file_3_2).getId());
        assertEquals(ec32Policy.getNumDataUnits(), ((BlockInfoStriped) blks[0]).getDataBlockNum());
        assertEquals(ec32Policy.getNumParityUnits(), ((BlockInfoStriped) blks[0]).getParityBlockNum());
        content = DFSTestUtil.readFileAsBytes(fs, file_3_2);
        assertArrayEquals(bytes, content);
        // check the EC policy on parent Dir
        ErasureCodingPolicy ecPolicy = fsn.getErasureCodingPolicy(parentDir.toString());
        assertNotNull(ecPolicy);
        assertEquals(testECPolicy.getId(), ecPolicy.getId());
        // check the EC policy on child Dir
        ecPolicy = fsn.getErasureCodingPolicy(childDir.toString());
        assertNotNull(ecPolicy);
        assertEquals(ec32Policy.getId(), ecPolicy.getId());
        // check the EC policy on root directory
        ecPolicy = fsn.getErasureCodingPolicy("/");
        assertNull(ecPolicy);
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 24 with ErasureCodingPolicy

use of org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy in project hadoop by apache.

the class TestEnabledECPolicies method testGetPolicies.

private void testGetPolicies(ErasureCodingPolicy[] enabledPolicies) throws Exception {
    HdfsConfiguration conf = new HdfsConfiguration();
    conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, Arrays.asList(enabledPolicies).stream().map(ErasureCodingPolicy::getName).collect(Collectors.joining(", ")));
    ErasureCodingPolicyManager manager = new ErasureCodingPolicyManager(conf);
    // Check that returned values are unique
    Set<String> found = new HashSet<>();
    for (ErasureCodingPolicy p : manager.getEnabledPolicies()) {
        Assert.assertFalse("Duplicate policy name found: " + p.getName(), found.contains(p.getName()));
        found.add(p.getName());
    }
    // Check that the policies specified in conf are found
    for (ErasureCodingPolicy p : enabledPolicies) {
        Assert.assertTrue("Did not find specified EC policy " + p.getName(), found.contains(p.getName()));
    }
    Assert.assertEquals(enabledPolicies.length, found.size());
    // Check that getEnabledPolicyByName only returns enabled policies
    for (ErasureCodingPolicy p : SYSTEM_POLICIES) {
        if (found.contains(p.getName())) {
            // Enabled policy should be present
            Assert.assertNotNull("getEnabledPolicyByName did not find enabled policy" + p.getName(), manager.getEnabledPolicyByName(p.getName()));
        } else {
            // Disabled policy should not be present
            Assert.assertNull("getEnabledPolicyByName found disabled policy " + p.getName(), manager.getEnabledPolicyByName(p.getName()));
        }
    }
}

Example 25 with ErasureCodingPolicy

use of org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy in project hadoop by apache.

the class TestFsck method testFsckOpenECFiles.

@Test
public void testFsckOpenECFiles() throws Exception {
    DFSTestUtil util = new DFSTestUtil.Builder().setName("TestFsckECFile").setNumFiles(4).build();
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
    ErasureCodingPolicy ecPolicy = StripedFileTestUtil.getDefaultECPolicy();
    final int dataBlocks = ecPolicy.getNumDataUnits();
    final int cellSize = ecPolicy.getCellSize();
    final int numAllUnits = dataBlocks + ecPolicy.getNumParityUnits();
    int blockSize = 2 * cellSize;
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
    conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, ecPolicy.getName());
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numAllUnits + 1).build();
    String topDir = "/myDir";
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    util.createFiles(fs, topDir);
    // set topDir to EC when it has replicated files
    cluster.getFileSystem().getClient().setErasureCodingPolicy(topDir, ecPolicy.getName());
    // create a new file under topDir
    DFSTestUtil.createFile(fs, new Path(topDir, "ecFile"), 1024, (short) 1, 0L);
    // Open a EC file for writing and do not close for now
    Path openFile = new Path(topDir + "/openECFile");
    FSDataOutputStream out = fs.create(openFile);
    int blockGroupSize = dataBlocks * blockSize;
    // data size is more than 1 block group and less than 2 block groups
    byte[] randomBytes = new byte[2 * blockGroupSize - cellSize];
    int seed = 42;
    new Random(seed).nextBytes(randomBytes);
    out.write(randomBytes);
    // make sure the fsck can correctly handle mixed ec/replicated files
    runFsck(conf, 0, true, topDir, "-files", "-blocks", "-openforwrite");
    // We expect the filesystem to be HEALTHY and show one open file
    String outStr = runFsck(conf, 0, true, openFile.toString(), "-files", "-blocks", "-openforwrite");
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    assertTrue(outStr.contains("OPENFORWRITE"));
    assertTrue(outStr.contains("Live_repl=" + numAllUnits));
    assertTrue(outStr.contains("Expected_repl=" + numAllUnits));
    // Use -openforwrite option to list open files
    outStr = runFsck(conf, 0, true, openFile.toString(), "-files", "-blocks", "-locations", "-openforwrite", "-replicaDetails");
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    assertTrue(outStr.contains("OPENFORWRITE"));
    assertTrue(outStr.contains("Live_repl=" + numAllUnits));
    assertTrue(outStr.contains("Expected_repl=" + numAllUnits));
    assertTrue(outStr.contains("Under Construction Block:"));
    // check reported blockIDs of internal blocks
    LocatedStripedBlock lsb = (LocatedStripedBlock) fs.getClient().getLocatedBlocks(openFile.toString(), 0, cellSize * dataBlocks).get(0);
    long groupId = lsb.getBlock().getBlockId();
    byte[] indices = lsb.getBlockIndices();
    DatanodeInfo[] locs = lsb.getLocations();
    long blockId;
    for (int i = 0; i < indices.length; i++) {
        blockId = groupId + indices[i];
        String str = "blk_" + blockId + ":" + locs[i];
        assertTrue(outStr.contains(str));
    }
    // check the output of under-constructed blocks doesn't include the blockIDs
    String regex = ".*Expected_repl=" + numAllUnits + "(.*)\nStatus:.*";
    Pattern p = Pattern.compile(regex, Pattern.DOTALL);
    Matcher m = p.matcher(outStr);
    assertTrue(m.find());
    String ucBlockOutput = m.group(1);
    assertFalse(ucBlockOutput.contains("blk_"));
    // Close the file
    out.close();
    // Now, fsck should show HEALTHY fs and should not show any open files
    outStr = runFsck(conf, 0, true, openFile.toString(), "-files", "-blocks", "-locations", "-racks", "-replicaDetails");
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    assertFalse(outStr.contains("OPENFORWRITE"));
    assertFalse(outStr.contains("Under Construction Block:"));
    assertFalse(outStr.contains("Expected_repl=" + numAllUnits));
    assertTrue(outStr.contains("Live_repl=" + numAllUnits));
    util.cleanup(fs, topDir);
}