Examples with ErasureCodingPolicy org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy used on opensource projects

Example 41 with ErasureCodingPolicy

use of org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy in project hadoop by apache.

the class TestFileStatusWithECPolicy method testFileStatusWithECPolicy.

@Test
public void testFileStatusWithECPolicy() throws Exception {
    // test directory doesn't have an EC policy
    final Path dir = new Path("/foo");
    assertTrue(fs.mkdir(dir, FsPermission.getDirDefault()));
    assertNull(client.getFileInfo(dir.toString()).getErasureCodingPolicy());
    // test file doesn't have an EC policy
    final Path file = new Path(dir, "foo");
    fs.create(file).close();
    assertNull(client.getFileInfo(file.toString()).getErasureCodingPolicy());
    fs.delete(file, true);
    final ErasureCodingPolicy ecPolicy1 = StripedFileTestUtil.getDefaultECPolicy();
    // set EC policy on dir
    fs.setErasureCodingPolicy(dir, ecPolicy1.getName());
    final ErasureCodingPolicy ecPolicy2 = client.getFileInfo(dir.toUri().getPath()).getErasureCodingPolicy();
    assertNotNull(ecPolicy2);
    assertTrue(ecPolicy1.equals(ecPolicy2));
    // test file doesn't have an EC policy
    fs.create(file).close();
    final ErasureCodingPolicy ecPolicy3 = fs.getClient().getFileInfo(file.toUri().getPath()).getErasureCodingPolicy();
    assertNotNull(ecPolicy3);
    assertTrue(ecPolicy1.equals(ecPolicy3));
}

Example 42 with ErasureCodingPolicy

use of org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy in project hadoop by apache.

the class TestUnsetAndChangeDirectoryEcPolicy method testUnsetRootDirEcPolicy.

/*
   * Test unset EC policy on root directory.
   */
@Test
public void testUnsetRootDirEcPolicy() throws Exception {
    final int numBlocks = 1;
    final int fileLen = blockGroupSize * numBlocks;
    final Path rootPath = new Path("/");
    final Path ecFilePath = new Path(rootPath, "ec_file");
    final Path replicateFilePath = new Path(rootPath, "rep_file");
    // Test unset root path which has no EC policy
    fs.unsetErasureCodingPolicy(rootPath);
    // Set EC policy on root path
    fs.setErasureCodingPolicy(rootPath, ecPolicy.getName());
    DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L);
    fs.unsetErasureCodingPolicy(rootPath);
    DFSTestUtil.createFile(fs, replicateFilePath, fileLen, (short) 1, 0L);
    // ec_file should has EC policy set
    ErasureCodingPolicy tempEcPolicy = fs.getErasureCodingPolicy(ecFilePath);
    Assert.assertTrue("Erasure coding policy mismatch!", tempEcPolicy.getName().equals(ecPolicy.getName()));
    // rep_file should not have EC policy set
    tempEcPolicy = fs.getErasureCodingPolicy(replicateFilePath);
    Assert.assertNull("Replicate file should not have erasure coding policy!", tempEcPolicy);
    // Directory should not return erasure coding policy
    tempEcPolicy = fs.getErasureCodingPolicy(rootPath);
    Assert.assertNull("Directory should not have erasure coding policy set!", tempEcPolicy);
    fs.delete(rootPath, true);
}

Example 43 with ErasureCodingPolicy

use of org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy in project hadoop by apache.

the class TestUnsetAndChangeDirectoryEcPolicy method testNestedEcPolicy.

/*
  * Test nested directory with different EC policy.
  */
@Test
public void testNestedEcPolicy() throws Exception {
    final int numBlocks = 1;
    final int fileLen = blockGroupSize * numBlocks;
    final Path parentDir = new Path("/ec-6-3");
    final Path childDir = new Path("/ec-6-3/ec-3-2");
    final Path ec63FilePath = new Path(childDir, "ec_6_3_file");
    final Path ec32FilePath = new Path(childDir, "ec_3_2_file");
    final Path ec63FilePath2 = new Path(childDir, "ec_6_3_file_2");
    final ErasureCodingPolicy ec32Policy = ErasureCodingPolicyManager.getPolicyByID(HdfsConstants.RS_3_2_POLICY_ID);
    fs.mkdirs(parentDir);
    fs.setErasureCodingPolicy(parentDir, ecPolicy.getName());
    fs.mkdirs(childDir);
    // Create RS(6,3) EC policy file
    DFSTestUtil.createFile(fs, ec63FilePath, fileLen, (short) 1, 0L);
    // Set RS(3,2) EC policy on child directory
    fs.setErasureCodingPolicy(childDir, ec32Policy.getName());
    // Create RS(3,2) EC policy file
    DFSTestUtil.createFile(fs, ec32FilePath, fileLen, (short) 1, 0L);
    // Start to check
    // ec_6_3_file should has RS-6-3 EC policy
    ErasureCodingPolicy tempEcPolicy = fs.getErasureCodingPolicy(ec63FilePath);
    Assert.assertTrue("Erasure coding policy mismatch!", tempEcPolicy.getName().equals(ecPolicy.getName()));
    // ec_3_2_file should have RS-3-2 policy
    tempEcPolicy = fs.getErasureCodingPolicy(ec32FilePath);
    Assert.assertTrue("Erasure coding policy mismatch!", tempEcPolicy.getName().equals(ec32Policy.getName()));
    // Child directory should have RS-3-2 policy
    tempEcPolicy = fs.getErasureCodingPolicy(childDir);
    Assert.assertTrue("Directory should have erasure coding policy set!", tempEcPolicy.getName().equals(ec32Policy.getName()));
    // Unset EC policy on child directory
    fs.unsetErasureCodingPolicy(childDir);
    DFSTestUtil.createFile(fs, ec63FilePath2, fileLen, (short) 1, 0L);
    // ec_6_3_file_2 should have RS-6-3 policy
    tempEcPolicy = fs.getErasureCodingPolicy(ec63FilePath2);
    Assert.assertTrue("Erasure coding policy mismatch!", tempEcPolicy.getName().equals(ecPolicy.getName()));
    // Child directory should have RS-6-3 policy now
    tempEcPolicy = fs.getErasureCodingPolicy(childDir);
    Assert.assertTrue("Directory should have erasure coding policy set!", tempEcPolicy.getName().equals(ecPolicy.getName()));
    fs.delete(parentDir, true);
}

Example 44 with ErasureCodingPolicy

use of org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy in project hadoop by apache.

the class TestPBHelper method assertBlockECRecoveryInfoEquals.

private void assertBlockECRecoveryInfoEquals(BlockECReconstructionInfo blkECRecoveryInfo1, BlockECReconstructionInfo blkECRecoveryInfo2) {
    assertEquals(blkECRecoveryInfo1.getExtendedBlock(), blkECRecoveryInfo2.getExtendedBlock());
    DatanodeInfo[] sourceDnInfos1 = blkECRecoveryInfo1.getSourceDnInfos();
    DatanodeInfo[] sourceDnInfos2 = blkECRecoveryInfo2.getSourceDnInfos();
    assertDnInfosEqual(sourceDnInfos1, sourceDnInfos2);
    DatanodeInfo[] targetDnInfos1 = blkECRecoveryInfo1.getTargetDnInfos();
    DatanodeInfo[] targetDnInfos2 = blkECRecoveryInfo2.getTargetDnInfos();
    assertDnInfosEqual(targetDnInfos1, targetDnInfos2);
    String[] targetStorageIDs1 = blkECRecoveryInfo1.getTargetStorageIDs();
    String[] targetStorageIDs2 = blkECRecoveryInfo2.getTargetStorageIDs();
    assertEquals(targetStorageIDs1.length, targetStorageIDs2.length);
    for (int i = 0; i < targetStorageIDs1.length; i++) {
        assertEquals(targetStorageIDs1[i], targetStorageIDs2[i]);
    }
    byte[] liveBlockIndices1 = blkECRecoveryInfo1.getLiveBlockIndices();
    byte[] liveBlockIndices2 = blkECRecoveryInfo2.getLiveBlockIndices();
    for (int i = 0; i < liveBlockIndices1.length; i++) {
        assertEquals(liveBlockIndices1[i], liveBlockIndices2[i]);
    }
    ErasureCodingPolicy ecPolicy1 = blkECRecoveryInfo1.getErasureCodingPolicy();
    ErasureCodingPolicy ecPolicy2 = blkECRecoveryInfo2.getErasureCodingPolicy();
    // Compare ECPolicies same as default ECPolicy as we used system default
    // ECPolicy used in this test
    compareECPolicies(StripedFileTestUtil.getDefaultECPolicy(), ecPolicy1);
    compareECPolicies(StripedFileTestUtil.getDefaultECPolicy(), ecPolicy2);
}

Example 45 with ErasureCodingPolicy

use of org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy in project SSM by Intel-bigdata.

the class CheckErasureCodingPolicy method execute.

@Override
public void execute() throws Exception {
    this.setDfsClient(HadoopUtil.getDFSClient(HadoopUtil.getNameNodeUri(conf), conf));
    ErasureCodingPolicy srcEcPolicy = dfsClient.getErasureCodingPolicy(srcPath);
    if (srcEcPolicy == null) {
        appendLog(RESULT_OF_NULL_EC_POLICY);
    } else {
        appendLog(srcEcPolicy.toString());
    }
}