Examples with BlockStoragePolicySuite org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite used on opensource projects

Example 1 with BlockStoragePolicySuite

use of org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite in project hadoop by apache.

the class FSDirRenameOp method verifyQuotaForRename.

/**
   * Verify quota for rename operation where srcInodes[srcInodes.length-1] moves
   * dstInodes[dstInodes.length-1]
   */
private static void verifyQuotaForRename(FSDirectory fsd, INodesInPath src, INodesInPath dst) throws QuotaExceededException {
    if (!fsd.getFSNamesystem().isImageLoaded() || fsd.shouldSkipQuotaChecks()) {
        // Do not check quota if edits log is still being processed
        return;
    }
    int i = 0;
    while (src.getINode(i) == dst.getINode(i)) {
        i++;
    }
    // src[i - 1] is the last common ancestor.
    BlockStoragePolicySuite bsps = fsd.getBlockStoragePolicySuite();
    final QuotaCounts delta = src.getLastINode().computeQuotaUsage(bsps);
    // Reduce the required quota by dst that is being removed
    final INode dstINode = dst.getLastINode();
    if (dstINode != null) {
        delta.subtract(dstINode.computeQuotaUsage(bsps));
    }
    FSDirectory.verifyQuota(dst, dst.length() - 1, delta, src.getINode(i - 1));
}

Example 2 with BlockStoragePolicySuite

use of org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite in project hadoop by apache.

the class TestHdfsAdmin method testHdfsAdminStoragePolicies.

/**
   * Test that we can set, get, unset storage policies via {@link HdfsAdmin}.
   */
@Test
public void testHdfsAdminStoragePolicies() throws Exception {
    HdfsAdmin hdfsAdmin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
    FileSystem fs = FileSystem.get(conf);
    final Path foo = new Path("/foo");
    final Path bar = new Path(foo, "bar");
    final Path wow = new Path(bar, "wow");
    DFSTestUtil.createFile(fs, wow, SIZE, REPL, 0);
    final BlockStoragePolicySuite suite = BlockStoragePolicySuite.createDefaultSuite();
    final BlockStoragePolicy warm = suite.getPolicy("WARM");
    final BlockStoragePolicy cold = suite.getPolicy("COLD");
    final BlockStoragePolicy hot = suite.getPolicy("HOT");
    /*
     * test: set storage policy
     */
    hdfsAdmin.setStoragePolicy(foo, warm.getName());
    hdfsAdmin.setStoragePolicy(bar, cold.getName());
    hdfsAdmin.setStoragePolicy(wow, hot.getName());
    /*
     * test: get storage policy after set
     */
    assertEquals(hdfsAdmin.getStoragePolicy(foo), warm);
    assertEquals(hdfsAdmin.getStoragePolicy(bar), cold);
    assertEquals(hdfsAdmin.getStoragePolicy(wow), hot);
    /*
     * test: unset storage policy
     */
    hdfsAdmin.unsetStoragePolicy(foo);
    hdfsAdmin.unsetStoragePolicy(bar);
    hdfsAdmin.unsetStoragePolicy(wow);
    /*
     * test: get storage policy after unset. HOT by default.
     */
    assertEquals(hdfsAdmin.getStoragePolicy(foo), hot);
    assertEquals(hdfsAdmin.getStoragePolicy(bar), hot);
    assertEquals(hdfsAdmin.getStoragePolicy(wow), hot);
    /*
     * test: get all storage policies
     */
    // Get policies via HdfsAdmin
    Set<String> policyNamesSet1 = new HashSet<>();
    for (BlockStoragePolicySpi policy : hdfsAdmin.getAllStoragePolicies()) {
        policyNamesSet1.add(policy.getName());
    }
    // Get policies via BlockStoragePolicySuite
    Set<String> policyNamesSet2 = new HashSet<>();
    for (BlockStoragePolicy policy : suite.getAllPolicies()) {
        policyNamesSet2.add(policy.getName());
    }
    // Ensure that we got the same set of policies in both cases.
    Assert.assertTrue(Sets.difference(policyNamesSet1, policyNamesSet2).isEmpty());
    Assert.assertTrue(Sets.difference(policyNamesSet2, policyNamesSet1).isEmpty());
}

Example 3 with BlockStoragePolicySuite

use of org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite in project hadoop by apache.

the class TestApplyingStoragePolicy method testStoragePolicyByDefault.

@Test
public void testStoragePolicyByDefault() throws Exception {
    final Path foo = new Path("/foo");
    final Path bar = new Path(foo, "bar");
    final Path wow = new Path(bar, "wow");
    final Path fooz = new Path(bar, "/fooz");
    DFSTestUtil.createFile(fs, wow, SIZE, REPL, 0);
    final BlockStoragePolicySuite suite = BlockStoragePolicySuite.createDefaultSuite();
    final BlockStoragePolicy hot = suite.getPolicy("HOT");
    /*
     * test: storage policy is HOT by default or inherited from nearest
     * ancestor, if not explicitly specified for newly created dir/file.
     */
    assertEquals(fs.getStoragePolicy(foo), hot);
    assertEquals(fs.getStoragePolicy(bar), hot);
    assertEquals(fs.getStoragePolicy(wow), hot);
    try {
        fs.getStoragePolicy(fooz);
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
}

Example 4 with BlockStoragePolicySuite

use of org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite in project hadoop by apache.

the class TestApplyingStoragePolicy method testNestedStoragePolicy.

@Test
public void testNestedStoragePolicy() throws Exception {
    final Path foo = new Path("/foo");
    final Path bar = new Path(foo, "bar");
    final Path wow = new Path(bar, "wow");
    final Path fooz = new Path("/foos");
    DFSTestUtil.createFile(fs, wow, SIZE, REPL, 0);
    final BlockStoragePolicySuite suite = BlockStoragePolicySuite.createDefaultSuite();
    final BlockStoragePolicy warm = suite.getPolicy("WARM");
    final BlockStoragePolicy cold = suite.getPolicy("COLD");
    final BlockStoragePolicy hot = suite.getPolicy("HOT");
    /*
     * test: set storage policy
     */
    fs.setStoragePolicy(foo, warm.getName());
    fs.setStoragePolicy(bar, cold.getName());
    fs.setStoragePolicy(wow, hot.getName());
    try {
        fs.setStoragePolicy(fooz, warm.getName());
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
    /*
     * test: get storage policy after set
     */
    assertEquals(fs.getStoragePolicy(foo), warm);
    assertEquals(fs.getStoragePolicy(bar), cold);
    assertEquals(fs.getStoragePolicy(wow), hot);
    try {
        fs.getStoragePolicy(fooz);
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
    /*
     * test: unset storage policy in the case of being nested
     */
    // unset wow
    fs.unsetStoragePolicy(wow);
    // inherit storage policy from wow's nearest ancestor
    assertEquals(fs.getStoragePolicy(wow), cold);
    // unset bar
    fs.unsetStoragePolicy(bar);
    // inherit storage policy from bar's nearest ancestor
    assertEquals(fs.getStoragePolicy(bar), warm);
    // unset foo
    fs.unsetStoragePolicy(foo);
    // default storage policy is applied, since no more available ancestors
    assertEquals(fs.getStoragePolicy(foo), hot);
    // unset fooz
    try {
        fs.unsetStoragePolicy(fooz);
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
    /*
     * test: default storage policy is applied, since no explicit policies from
     * ancestors are available
     */
    assertEquals(fs.getStoragePolicy(foo), hot);
    assertEquals(fs.getStoragePolicy(bar), hot);
    assertEquals(fs.getStoragePolicy(wow), hot);
    try {
        fs.getStoragePolicy(fooz);
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
}

Example 5 with BlockStoragePolicySuite

use of org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite in project hadoop by apache.

the class TestApplyingStoragePolicy method testSetAndUnsetStoragePolicy.

@Test
public void testSetAndUnsetStoragePolicy() throws Exception {
    final Path foo = new Path("/foo");
    final Path bar = new Path(foo, "bar");
    final Path wow = new Path(bar, "wow");
    final Path fooz = new Path(bar, "/fooz");
    DFSTestUtil.createFile(fs, wow, SIZE, REPL, 0);
    final BlockStoragePolicySuite suite = BlockStoragePolicySuite.createDefaultSuite();
    final BlockStoragePolicy warm = suite.getPolicy("WARM");
    final BlockStoragePolicy cold = suite.getPolicy("COLD");
    final BlockStoragePolicy hot = suite.getPolicy("HOT");
    /*
     * test: set storage policy
     */
    fs.setStoragePolicy(foo, warm.getName());
    fs.setStoragePolicy(bar, cold.getName());
    fs.setStoragePolicy(wow, hot.getName());
    try {
        fs.setStoragePolicy(fooz, warm.getName());
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
    /*
     * test: get storage policy after set
     */
    assertEquals(fs.getStoragePolicy(foo), warm);
    assertEquals(fs.getStoragePolicy(bar), cold);
    assertEquals(fs.getStoragePolicy(wow), hot);
    try {
        fs.getStoragePolicy(fooz);
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
    /*
     * test: unset storage policy
     */
    fs.unsetStoragePolicy(foo);
    fs.unsetStoragePolicy(bar);
    fs.unsetStoragePolicy(wow);
    try {
        fs.unsetStoragePolicy(fooz);
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
    /*
     * test: get storage policy after unset
     */
    assertEquals(fs.getStoragePolicy(foo), hot);
    assertEquals(fs.getStoragePolicy(bar), hot);
    assertEquals(fs.getStoragePolicy(wow), hot);
    try {
        fs.getStoragePolicy(fooz);
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
}