Examples with ErasureCodingPolicy org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy used on opensource projects

Example 1 with ErasureCodingPolicy

use of org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy in project hadoop by apache.

the class TestErasureCodingPolicies method verifyErasureCodingInfo.

private void verifyErasureCodingInfo(String src, ErasureCodingPolicy usingECPolicy) throws IOException {
    HdfsFileStatus hdfsFileStatus = fs.getClient().getFileInfo(src);
    ErasureCodingPolicy ecPolicy = hdfsFileStatus.getErasureCodingPolicy();
    assertNotNull(ecPolicy);
    assertEquals("Actually used ecPolicy should be equal with target ecPolicy", usingECPolicy, ecPolicy);
}

Example 2 with ErasureCodingPolicy

use of org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy in project hadoop by apache.

the class TestErasureCodingPolicies method testGetErasureCodingPolicyWithSystemDefaultECPolicy.

@Test
public void testGetErasureCodingPolicyWithSystemDefaultECPolicy() throws Exception {
    String src = "/ec";
    final Path ecDir = new Path(src);
    fs.mkdir(ecDir, FsPermission.getDirDefault());
    // dir EC policy should be null
    assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy());
    // dir EC policy after setting
    ErasureCodingPolicy sysDefaultECPolicy = StripedFileTestUtil.getDefaultECPolicy();
    fs.getClient().setErasureCodingPolicy(src, sysDefaultECPolicy.getName());
    verifyErasureCodingInfo(src, sysDefaultECPolicy);
    fs.create(new Path(ecDir, "child1")).close();
    // verify for the files in ec dir
    verifyErasureCodingInfo(src + "/child1", sysDefaultECPolicy);
}

Example 3 with ErasureCodingPolicy

use of org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy in project hadoop by apache.

the class TestErasureCodingPolicies method testGetErasureCodingPolicy.

@Test
public void testGetErasureCodingPolicy() throws Exception {
    ErasureCodingPolicy[] sysECPolicies = ErasureCodingPolicyManager.getSystemPolicies();
    assertTrue("System ecPolicies should exist", sysECPolicies.length > 0);
    ErasureCodingPolicy usingECPolicy = sysECPolicies[0];
    String src = "/ec2";
    final Path ecDir = new Path(src);
    fs.mkdir(ecDir, FsPermission.getDirDefault());
    // dir ECInfo before being set
    assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy());
    // dir ECInfo after set
    fs.getClient().setErasureCodingPolicy(src, usingECPolicy.getName());
    verifyErasureCodingInfo(src, usingECPolicy);
    fs.create(new Path(ecDir, "child1")).close();
    // verify for the files in ec dir
    verifyErasureCodingInfo(src + "/child1", usingECPolicy);
}

Example 4 with ErasureCodingPolicy

use of org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy in project hadoop by apache.

the class TestErasureCodingPolicies method testMultiplePoliciesCoExist.

@Test
public void testMultiplePoliciesCoExist() throws Exception {
    ErasureCodingPolicy[] sysPolicies = ErasureCodingPolicyManager.getSystemPolicies();
    if (sysPolicies.length > 1) {
        for (ErasureCodingPolicy policy : sysPolicies) {
            Path dir = new Path("/policy_" + policy.getId());
            fs.mkdir(dir, FsPermission.getDefault());
            fs.setErasureCodingPolicy(dir, policy.getName());
            Path file = new Path(dir, "child");
            fs.create(file).close();
            assertEquals(policy, fs.getErasureCodingPolicy(file));
            assertEquals(policy, fs.getErasureCodingPolicy(dir));
            INode iNode = namesystem.getFSDirectory().getINode(file.toString());
            assertEquals(policy.getId(), iNode.asFile().getErasureCodingPolicyID());
            assertEquals(INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS, iNode.asFile().getFileReplication());
        }
    }
}

Example 5 with ErasureCodingPolicy

use of org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy in project hadoop by apache.

the class TestErasureCodingPolicyWithSnapshot method testSnapshotsOnErasureCodingDirsParentDir.

/**
   * Test correctness of successive snapshot creation and deletion with erasure
   * coding policies. Create snapshot of ecDir's parent directory.
   */
@Test(timeout = 120000)
public void testSnapshotsOnErasureCodingDirsParentDir() throws Exception {
    final int len = 1024;
    final Path ecDirParent = new Path("/parent");
    final Path ecDir = new Path(ecDirParent, "ecdir");
    final Path ecFile = new Path(ecDir, "ecfile");
    fs.mkdirs(ecDir);
    fs.allowSnapshot(ecDirParent);
    // set erasure coding policy
    fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
    DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
    String contents = DFSTestUtil.readFile(fs, ecFile);
    final Path snap1 = fs.createSnapshot(ecDirParent, "snap1");
    final Path snap1ECDir = new Path(snap1, ecDir.getName());
    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, fs.getErasureCodingPolicy(snap1ECDir));
    // Now delete the dir which has erasure coding policy. Re-create the dir again, and
    // take another snapshot
    fs.delete(ecDir, true);
    fs.mkdir(ecDir, FsPermission.getDirDefault());
    final Path snap2 = fs.createSnapshot(ecDirParent, "snap2");
    final Path snap2ECDir = new Path(snap2, ecDir.getName());
    assertNull("Expected null erasure coding policy", fs.getErasureCodingPolicy(snap2ECDir));
    // Make dir again with system default ec policy
    fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
    final Path snap3 = fs.createSnapshot(ecDirParent, "snap3");
    final Path snap3ECDir = new Path(snap3, ecDir.getName());
    // Check that snap3's ECPolicy has the correct settings
    ErasureCodingPolicy ezSnap3 = fs.getErasureCodingPolicy(snap3ECDir);
    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, ezSnap3);
    // Check that older snapshots still have the old ECPolicy settings
    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, fs.getErasureCodingPolicy(snap1ECDir));
    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, fs.getErasureCodingPolicy(snap2ECDir));
    // Verify contents of the snapshotted file
    final Path snapshottedECFile = new Path(snap1.toString() + "/" + ecDir.getName() + "/" + ecFile.getName());
    assertEquals("Contents of snapshotted file have changed unexpectedly", contents, DFSTestUtil.readFile(fs, snapshottedECFile));
    // Now delete the snapshots out of order and verify the EC policy
    // correctness
    fs.deleteSnapshot(ecDirParent, snap2.getName());
    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, fs.getErasureCodingPolicy(snap1ECDir));
    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, fs.getErasureCodingPolicy(snap3ECDir));
    fs.deleteSnapshot(ecDirParent, snap1.getName());
    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, fs.getErasureCodingPolicy(snap3ECDir));
}