Examples with StorageType org.apache.hadoop.fs.StorageType used on opensource projects

Example 66 with StorageType

use of org.apache.hadoop.fs.StorageType in project hadoop by apache.

the class BlockStoragePolicy method chooseStorageTypes.

/**
   * Choose the storage types for storing the remaining replicas, given the
   * replication number, the storage types of the chosen replicas and
   * the unavailable storage types. It uses fallback storage in case that
   * the desired storage type is unavailable.
   *
   * @param replication the replication number.
   * @param chosen the storage types of the chosen replicas.
   * @param unavailables the unavailable storage types.
   * @param isNewBlock Is it for new block creation?
   * @return a list of {@link StorageType}s for storing the replicas of a block.
   */
public List<StorageType> chooseStorageTypes(final short replication, final Iterable<StorageType> chosen, final EnumSet<StorageType> unavailables, final boolean isNewBlock) {
    final List<StorageType> excess = new LinkedList<>();
    final List<StorageType> storageTypes = chooseStorageTypes(replication, chosen, excess);
    final int expectedSize = storageTypes.size() - excess.size();
    final List<StorageType> removed = new LinkedList<>();
    for (int i = storageTypes.size() - 1; i >= 0; i--) {
        // replace/remove unavailable storage types.
        final StorageType t = storageTypes.get(i);
        if (unavailables.contains(t)) {
            final StorageType fallback = isNewBlock ? getCreationFallback(unavailables) : getReplicationFallback(unavailables);
            if (fallback == null) {
                removed.add(storageTypes.remove(i));
            } else {
                storageTypes.set(i, fallback);
            }
        }
    }
    // remove excess storage types after fallback replacement.
    diff(storageTypes, excess, null);
    if (storageTypes.size() < expectedSize) {
        LOG.warn("Failed to place enough replicas: expected size is {}" + " but only {} storage types can be selected (replication={}," + " selected={}, unavailable={}" + ", removed={}" + ", policy={}" + ")", expectedSize, storageTypes.size(), replication, storageTypes, unavailables, removed, this);
    }
    return storageTypes;
}

Example 67 with StorageType

use of org.apache.hadoop.fs.StorageType in project hadoop by apache.

the class BlockStoragePolicy method chooseStorageTypes.

/**
   * @return a list of {@link StorageType}s for storing the replicas of a block.
   */
public List<StorageType> chooseStorageTypes(final short replication) {
    final List<StorageType> types = new LinkedList<>();
    int i = 0, j = 0;
    // usage information for transient types.
    for (; i < replication && j < storageTypes.length; ++j) {
        if (!storageTypes[j].isTransient()) {
            types.add(storageTypes[j]);
            ++i;
        }
    }
    final StorageType last = storageTypes[storageTypes.length - 1];
    if (!last.isTransient()) {
        for (; i < replication; i++) {
            types.add(last);
        }
    }
    return types;
}

Example 68 with StorageType

use of org.apache.hadoop.fs.StorageType in project hadoop by apache.

the class BlockStoragePolicySuite method createDefaultSuite.

@VisibleForTesting
public static BlockStoragePolicySuite createDefaultSuite() {
    final BlockStoragePolicy[] policies = new BlockStoragePolicy[1 << ID_BIT_LENGTH];
    final byte lazyPersistId = HdfsConstants.MEMORY_STORAGE_POLICY_ID;
    policies[lazyPersistId] = new BlockStoragePolicy(lazyPersistId, HdfsConstants.MEMORY_STORAGE_POLICY_NAME, new StorageType[] { StorageType.RAM_DISK, StorageType.DISK }, new StorageType[] { StorageType.DISK }, new StorageType[] { StorageType.DISK }, // Cannot be changed on regular files, but inherited.
    true);
    final byte allssdId = HdfsConstants.ALLSSD_STORAGE_POLICY_ID;
    policies[allssdId] = new BlockStoragePolicy(allssdId, HdfsConstants.ALLSSD_STORAGE_POLICY_NAME, new StorageType[] { StorageType.SSD }, new StorageType[] { StorageType.DISK }, new StorageType[] { StorageType.DISK });
    final byte onessdId = HdfsConstants.ONESSD_STORAGE_POLICY_ID;
    policies[onessdId] = new BlockStoragePolicy(onessdId, HdfsConstants.ONESSD_STORAGE_POLICY_NAME, new StorageType[] { StorageType.SSD, StorageType.DISK }, new StorageType[] { StorageType.SSD, StorageType.DISK }, new StorageType[] { StorageType.SSD, StorageType.DISK });
    final byte hotId = HdfsConstants.HOT_STORAGE_POLICY_ID;
    policies[hotId] = new BlockStoragePolicy(hotId, HdfsConstants.HOT_STORAGE_POLICY_NAME, new StorageType[] { StorageType.DISK }, StorageType.EMPTY_ARRAY, new StorageType[] { StorageType.ARCHIVE });
    final byte warmId = HdfsConstants.WARM_STORAGE_POLICY_ID;
    policies[warmId] = new BlockStoragePolicy(warmId, HdfsConstants.WARM_STORAGE_POLICY_NAME, new StorageType[] { StorageType.DISK, StorageType.ARCHIVE }, new StorageType[] { StorageType.DISK, StorageType.ARCHIVE }, new StorageType[] { StorageType.DISK, StorageType.ARCHIVE });
    final byte coldId = HdfsConstants.COLD_STORAGE_POLICY_ID;
    policies[coldId] = new BlockStoragePolicy(coldId, HdfsConstants.COLD_STORAGE_POLICY_NAME, new StorageType[] { StorageType.ARCHIVE }, StorageType.EMPTY_ARRAY, StorageType.EMPTY_ARRAY);
    return new BlockStoragePolicySuite(hotId, policies);
}

Example 69 with StorageType

use of org.apache.hadoop.fs.StorageType in project hadoop by apache.

the class BlockManager method chooseExcessRedundancyStriped.

/**
   * We want block group has every internal block, but we have redundant
   * internal blocks (which have the same index).
   * In this method, we delete the redundant internal blocks until only one
   * left for each index.
   *
   * The block placement policy will make sure that the left internal blocks are
   * spread across racks and also try hard to pick one with least free space.
   */
private void chooseExcessRedundancyStriped(BlockCollection bc, final Collection<DatanodeStorageInfo> nonExcess, BlockInfo storedBlock, DatanodeDescriptor delNodeHint) {
    assert storedBlock instanceof BlockInfoStriped;
    BlockInfoStriped sblk = (BlockInfoStriped) storedBlock;
    short groupSize = sblk.getTotalBlockNum();
    // find all duplicated indices
    //indices found
    BitSet found = new BitSet(groupSize);
    //indices found more than once
    BitSet duplicated = new BitSet(groupSize);
    HashMap<DatanodeStorageInfo, Integer> storage2index = new HashMap<>();
    for (DatanodeStorageInfo storage : nonExcess) {
        int index = sblk.getStorageBlockIndex(storage);
        assert index >= 0;
        if (found.get(index)) {
            duplicated.set(index);
        }
        found.set(index);
        storage2index.put(storage, index);
    }
    // use delHint only if delHint is duplicated
    final DatanodeStorageInfo delStorageHint = DatanodeStorageInfo.getDatanodeStorageInfo(nonExcess, delNodeHint);
    if (delStorageHint != null) {
        Integer index = storage2index.get(delStorageHint);
        if (index != null && duplicated.get(index)) {
            processChosenExcessRedundancy(nonExcess, delStorageHint, storedBlock);
        }
    }
    // cardinality of found indicates the expected number of internal blocks
    final int numOfTarget = found.cardinality();
    final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(bc.getStoragePolicyID());
    final List<StorageType> excessTypes = storagePolicy.chooseExcess((short) numOfTarget, DatanodeStorageInfo.toStorageTypes(nonExcess));
    if (excessTypes.isEmpty()) {
        LOG.warn("excess types chosen for block {} among storages {} is empty", storedBlock, nonExcess);
        return;
    }
    BlockPlacementPolicy placementPolicy = placementPolicies.getPolicy(STRIPED);
    // for each duplicated index, delete some replicas until only one left
    for (int targetIndex = duplicated.nextSetBit(0); targetIndex >= 0; targetIndex = duplicated.nextSetBit(targetIndex + 1)) {
        List<DatanodeStorageInfo> candidates = new ArrayList<>();
        for (DatanodeStorageInfo storage : nonExcess) {
            int index = storage2index.get(storage);
            if (index == targetIndex) {
                candidates.add(storage);
            }
        }
        if (candidates.size() > 1) {
            List<DatanodeStorageInfo> replicasToDelete = placementPolicy.chooseReplicasToDelete(nonExcess, candidates, (short) 1, excessTypes, null, null);
            for (DatanodeStorageInfo chosen : replicasToDelete) {
                processChosenExcessRedundancy(nonExcess, chosen, storedBlock);
                candidates.remove(chosen);
            }
        }
        duplicated.clear(targetIndex);
    }
}

Example 70 with StorageType

use of org.apache.hadoop.fs.StorageType in project hadoop by apache.

the class BalancingPolicy method initAvgUtilization.

void initAvgUtilization() {
    for (StorageType t : StorageType.asList()) {
        final long capacity = totalCapacities.get(t);
        if (capacity > 0L) {
            final double avg = totalUsedSpaces.get(t) * 100.0 / capacity;
            avgUtilizations.set(t, avg);
        }
    }
}