Examples with StorageType org.apache.hadoop.fs.StorageType used on opensource projects

Example 1 with StorageType

use of org.apache.hadoop.fs.StorageType in project hadoop by apache.

the class TestDistributedFileSystem method testLocatedFileStatusStorageIdsTypes.

@Test(timeout = 120000)
public void testLocatedFileStatusStorageIdsTypes() throws Exception {
    final Configuration conf = getTestConfiguration();
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    try {
        final DistributedFileSystem fs = cluster.getFileSystem();
        final Path testFile = new Path("/testListLocatedStatus");
        final int blockSize = 4096;
        final int numBlocks = 10;
        // Create a test file
        final int repl = 2;
        DFSTestUtil.createFile(fs, testFile, blockSize, numBlocks * blockSize, blockSize, (short) repl, 0xADDED);
        DFSTestUtil.waitForReplication(fs, testFile, (short) repl, 30000);
        // Get the listing
        RemoteIterator<LocatedFileStatus> it = fs.listLocatedStatus(testFile);
        assertTrue("Expected file to be present", it.hasNext());
        LocatedFileStatus stat = it.next();
        BlockLocation[] locs = stat.getBlockLocations();
        assertEquals("Unexpected number of locations", numBlocks, locs.length);
        Set<String> dnStorageIds = new HashSet<>();
        for (DataNode d : cluster.getDataNodes()) {
            try (FsDatasetSpi.FsVolumeReferences volumes = d.getFSDataset().getFsVolumeReferences()) {
                for (FsVolumeSpi vol : volumes) {
                    dnStorageIds.add(vol.getStorageID());
                }
            }
        }
        for (BlockLocation loc : locs) {
            String[] ids = loc.getStorageIds();
            // Run it through a set to deduplicate, since there should be no dupes
            Set<String> storageIds = new HashSet<>();
            Collections.addAll(storageIds, ids);
            assertEquals("Unexpected num storage ids", repl, storageIds.size());
            // Make sure these are all valid storage IDs
            assertTrue("Unknown storage IDs found!", dnStorageIds.containsAll(storageIds));
            // Check storage types are the default, since we didn't set any
            StorageType[] types = loc.getStorageTypes();
            assertEquals("Unexpected num storage types", repl, types.length);
            for (StorageType t : types) {
                assertEquals("Unexpected storage type", StorageType.DEFAULT, t);
            }
        }
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 2 with StorageType

use of org.apache.hadoop.fs.StorageType in project hadoop by apache.

the class TestBlockReplacement method testBlockMoveAcrossStorageInSameNode.

@Test
public void testBlockMoveAcrossStorageInSameNode() throws Exception {
    final Configuration conf = new HdfsConfiguration();
    // create only one datanode in the cluster to verify movement within
    // datanode.
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).storageTypes(new StorageType[] { StorageType.DISK, StorageType.ARCHIVE }).build();
    try {
        cluster.waitActive();
        final DistributedFileSystem dfs = cluster.getFileSystem();
        final Path file = new Path("/testBlockMoveAcrossStorageInSameNode/file");
        DFSTestUtil.createFile(dfs, file, 1024, (short) 1, 1024);
        LocatedBlocks locatedBlocks = dfs.getClient().getLocatedBlocks(file.toString(), 0);
        // get the current 
        LocatedBlock locatedBlock = locatedBlocks.get(0);
        ExtendedBlock block = locatedBlock.getBlock();
        DatanodeInfo[] locations = locatedBlock.getLocations();
        assertEquals(1, locations.length);
        StorageType[] storageTypes = locatedBlock.getStorageTypes();
        // current block should be written to DISK
        assertTrue(storageTypes[0] == StorageType.DISK);
        DatanodeInfo source = locations[0];
        // move block to ARCHIVE by using same DataNodeInfo for source, proxy and
        // destination so that movement happens within datanode 
        assertTrue(replaceBlock(block, source, source, source, StorageType.ARCHIVE, Status.SUCCESS));
        // wait till namenode notified
        Thread.sleep(3000);
        locatedBlocks = dfs.getClient().getLocatedBlocks(file.toString(), 0);
        // get the current 
        locatedBlock = locatedBlocks.get(0);
        assertEquals("Storage should be only one", 1, locatedBlock.getLocations().length);
        assertTrue("Block should be moved to ARCHIVE", locatedBlock.getStorageTypes()[0] == StorageType.ARCHIVE);
    } finally {
        cluster.shutdown();
    }
}

Example 3 with StorageType

use of org.apache.hadoop.fs.StorageType in project hadoop by apache.

the class TestReplicationPolicy method testChooseReplicasToDelete.

@Test
public void testChooseReplicasToDelete() throws Exception {
    Collection<DatanodeStorageInfo> nonExcess = new ArrayList<>();
    nonExcess.add(storages[0]);
    nonExcess.add(storages[1]);
    nonExcess.add(storages[2]);
    nonExcess.add(storages[3]);
    List<DatanodeStorageInfo> excessReplicas;
    BlockStoragePolicySuite POLICY_SUITE = BlockStoragePolicySuite.createDefaultSuite();
    BlockStoragePolicy storagePolicy = POLICY_SUITE.getDefaultPolicy();
    DatanodeStorageInfo excessSSD = DFSTestUtil.createDatanodeStorageInfo("Storage-excess-SSD-ID", "localhost", storages[0].getDatanodeDescriptor().getNetworkLocation(), "foo.com", StorageType.SSD, null);
    updateHeartbeatWithUsage(excessSSD.getDatanodeDescriptor(), 2 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, 2 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
    // use delete hint case.
    DatanodeDescriptor delHintNode = storages[0].getDatanodeDescriptor();
    List<StorageType> excessTypes = storagePolicy.chooseExcess((short) 3, DatanodeStorageInfo.toStorageTypes(nonExcess));
    excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 3, excessTypes, storages[3].getDatanodeDescriptor(), delHintNode);
    assertTrue(excessReplicas.size() == 1);
    assertTrue(excessReplicas.contains(storages[0]));
    // Excess type deletion
    DatanodeStorageInfo excessStorage = DFSTestUtil.createDatanodeStorageInfo("Storage-excess-ID", "localhost", delHintNode.getNetworkLocation(), "foo.com", StorageType.ARCHIVE, null);
    nonExcess.add(excessStorage);
    excessTypes = storagePolicy.chooseExcess((short) 3, DatanodeStorageInfo.toStorageTypes(nonExcess));
    excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 3, excessTypes, storages[3].getDatanodeDescriptor(), null);
    assertTrue(excessReplicas.contains(excessStorage));
    // The block was initially created on excessSSD(rack r1),
    // storages[4](rack r3) and storages[5](rack r3) with
    // ONESSD_STORAGE_POLICY_NAME storage policy. Replication factor = 3.
    // Right after balancer moves the block from storages[5] to
    // storages[3](rack r2), the application changes the storage policy from
    // ONESSD_STORAGE_POLICY_NAME to HOT_STORAGE_POLICY_ID. In this case,
    // we should be able to delete excessSSD since the remaining
    // storages ({storages[3]}, {storages[4], storages[5]})
    // are on different racks (r2, r3).
    nonExcess.clear();
    nonExcess.add(excessSSD);
    nonExcess.add(storages[3]);
    nonExcess.add(storages[4]);
    nonExcess.add(storages[5]);
    excessTypes = storagePolicy.chooseExcess((short) 3, DatanodeStorageInfo.toStorageTypes(nonExcess));
    excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 3, excessTypes, storages[3].getDatanodeDescriptor(), storages[5].getDatanodeDescriptor());
    assertEquals(1, excessReplicas.size());
    assertTrue(excessReplicas.contains(excessSSD));
    // Similar to above, but after policy change and before deletion,
    // the replicas are located on excessSSD(rack r1), storages[1](rack r1),
    // storages[2](rack r2) and storages[3](rack r2). Replication factor = 3.
    // In this case, we should be able to delete excessSSD since the remaining
    // storages ({storages[1]} , {storages[2], storages[3]})
    // are on different racks (r1, r2).
    nonExcess.clear();
    nonExcess.add(excessSSD);
    nonExcess.add(storages[1]);
    nonExcess.add(storages[2]);
    nonExcess.add(storages[3]);
    excessTypes = storagePolicy.chooseExcess((short) 3, DatanodeStorageInfo.toStorageTypes(nonExcess));
    excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 3, excessTypes, storages[1].getDatanodeDescriptor(), storages[3].getDatanodeDescriptor());
    assertEquals(1, excessReplicas.size());
    assertTrue(excessReplicas.contains(excessSSD));
    // Similar to above, but after policy change and before deletion,
    // the replicas are located on excessSSD(rack r1), storages[2](rack r2)
    // Replication factor = 1. We should be able to delete excessSSD.
    nonExcess.clear();
    nonExcess.add(excessSSD);
    nonExcess.add(storages[2]);
    excessTypes = storagePolicy.chooseExcess((short) 1, DatanodeStorageInfo.toStorageTypes(nonExcess));
    excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 1, excessTypes, storages[2].getDatanodeDescriptor(), null);
    assertEquals(1, excessReplicas.size());
    assertTrue(excessReplicas.contains(excessSSD));
    // The block was initially created on excessSSD(rack r1),
    // storages[4](rack r3) and storages[5](rack r3) with
    // ONESSD_STORAGE_POLICY_NAME storage policy. Replication factor = 2.
    // In this case, no replica can be chosen as the excessive replica by
    // chooseReplicasToDelete because if the SSD storage is deleted,
    // the remaining storages[4] and storages[5] are the same rack (r3),
    // violating block placement policy (i.e. the number of racks >= 2).
    // TODO BlockPlacementPolicyDefault should be able to rebalance the replicas
    // and then delete excessSSD.
    nonExcess.clear();
    nonExcess.add(excessSSD);
    nonExcess.add(storages[4]);
    nonExcess.add(storages[5]);
    excessTypes = storagePolicy.chooseExcess((short) 2, DatanodeStorageInfo.toStorageTypes(nonExcess));
    excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 2, excessTypes, null, null);
    assertEquals(0, excessReplicas.size());
}

Example 4 with StorageType

use of org.apache.hadoop.fs.StorageType in project hadoop by apache.

the class PBHelperClient method convertLocatedBlock.

public static LocatedBlockProto convertLocatedBlock(LocatedBlock b) {
    if (b == null)
        return null;
    Builder builder = LocatedBlockProto.newBuilder();
    DatanodeInfo[] locs = b.getLocations();
    List<DatanodeInfo> cachedLocs = Lists.newLinkedList(Arrays.asList(b.getCachedLocations()));
    for (int i = 0; i < locs.length; i++) {
        DatanodeInfo loc = locs[i];
        builder.addLocs(i, PBHelperClient.convert(loc));
        boolean locIsCached = cachedLocs.contains(loc);
        builder.addIsCached(locIsCached);
        if (locIsCached) {
            cachedLocs.remove(loc);
        }
    }
    Preconditions.checkArgument(cachedLocs.size() == 0, "Found additional cached replica locations that are not in the set of" + " storage-backed locations!");
    StorageType[] storageTypes = b.getStorageTypes();
    if (storageTypes != null) {
        for (StorageType storageType : storageTypes) {
            builder.addStorageTypes(convertStorageType(storageType));
        }
    }
    final String[] storageIDs = b.getStorageIDs();
    if (storageIDs != null) {
        builder.addAllStorageIDs(Arrays.asList(storageIDs));
    }
    if (b instanceof LocatedStripedBlock) {
        LocatedStripedBlock sb = (LocatedStripedBlock) b;
        byte[] indices = sb.getBlockIndices();
        builder.setBlockIndices(PBHelperClient.getByteString(indices));
        Token<BlockTokenIdentifier>[] blockTokens = sb.getBlockTokens();
        builder.addAllBlockTokens(convert(blockTokens));
    }
    return builder.setB(PBHelperClient.convert(b.getBlock())).setBlockToken(PBHelperClient.convert(b.getBlockToken())).setCorrupt(b.isCorrupt()).setOffset(b.getStartOffset()).build();
}

Example 5 with StorageType

use of org.apache.hadoop.fs.StorageType in project hadoop by apache.

the class BlockManager method addBlock.

/**
   * Get all valid locations of the block & add the block to results
   * @return the length of the added block; 0 if the block is not added. If the
   * added block is a block group, return its approximate internal block size
   */
private long addBlock(BlockInfo block, List<BlockWithLocations> results) {
    final List<DatanodeStorageInfo> locations = getValidLocations(block);
    if (locations.size() == 0) {
        return 0;
    } else {
        final String[] datanodeUuids = new String[locations.size()];
        final String[] storageIDs = new String[datanodeUuids.length];
        final StorageType[] storageTypes = new StorageType[datanodeUuids.length];
        for (int i = 0; i < locations.size(); i++) {
            final DatanodeStorageInfo s = locations.get(i);
            datanodeUuids[i] = s.getDatanodeDescriptor().getDatanodeUuid();
            storageIDs[i] = s.getStorageID();
            storageTypes[i] = s.getStorageType();
        }
        BlockWithLocations blkWithLocs = new BlockWithLocations(block, datanodeUuids, storageIDs, storageTypes);
        if (block.isStriped()) {
            BlockInfoStriped blockStriped = (BlockInfoStriped) block;
            byte[] indices = new byte[locations.size()];
            for (int i = 0; i < locations.size(); i++) {
                indices[i] = (byte) blockStriped.getStorageBlockIndex(locations.get(i));
            }
            results.add(new StripedBlockWithLocations(blkWithLocs, indices, blockStriped.getDataBlockNum(), blockStriped.getCellSize()));
            // approximate size
            return block.getNumBytes() / blockStriped.getDataBlockNum();
        } else {
            results.add(blkWithLocs);
            return block.getNumBytes();
        }
    }
}