Examples with LocatedStripedBlock org.apache.hadoop.hdfs.protocol.LocatedStripedBlock used on opensource projects

Example 21 with LocatedStripedBlock

use of org.apache.hadoop.hdfs.protocol.LocatedStripedBlock in project hadoop by apache.

the class TestReadStripedFileWithDecoding method testInvalidateBlock.

@Test
public void testInvalidateBlock() throws IOException {
    final Path file = new Path("/invalidate");
    final int length = 10;
    final byte[] bytes = StripedFileTestUtil.generateBytes(length);
    DFSTestUtil.writeFile(fs, file, bytes);
    int dnIndex = findFirstDataNode(file, cellSize * dataBlocks);
    Assert.assertNotEquals(-1, dnIndex);
    LocatedStripedBlock slb = (LocatedStripedBlock) fs.getClient().getLocatedBlocks(file.toString(), 0, cellSize * dataBlocks).get(0);
    final LocatedBlock[] blks = StripedBlockUtil.parseStripedBlockGroup(slb, cellSize, dataBlocks, parityBlocks);
    final Block b = blks[0].getBlock().getLocalBlock();
    DataNode dn = cluster.getDataNodes().get(dnIndex);
    // disable the heartbeat from DN so that the invalidated block record is kept
    // in NameNode until heartbeat expires and NN mark the dn as dead
    DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
    try {
        // delete the file
        fs.delete(file, true);
        // check the block is added to invalidateBlocks
        final FSNamesystem fsn = cluster.getNamesystem();
        final BlockManager bm = fsn.getBlockManager();
        DatanodeDescriptor dnd = NameNodeAdapter.getDatanode(fsn, dn.getDatanodeId());
        Assert.assertTrue(bm.containsInvalidateBlock(blks[0].getLocations()[0], b) || dnd.containsInvalidateBlock(b));
    } finally {
        DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, false);
    }
}

Example 22 with LocatedStripedBlock

use of org.apache.hadoop.hdfs.protocol.LocatedStripedBlock in project hadoop by apache.

the class TestReconstructStripedFile method assertFileBlocksReconstruction.

/**
   * Test the file blocks reconstruction.
   * 1. Check the replica is reconstructed in the target datanode,
   *    and verify the block replica length, generationStamp and content.
   * 2. Read the file and verify content.
   */
private void assertFileBlocksReconstruction(String fileName, int fileLen, ReconstructionType type, int toRecoverBlockNum) throws Exception {
    if (toRecoverBlockNum < 1 || toRecoverBlockNum > parityBlkNum) {
        Assert.fail("toRecoverBlockNum should be between 1 ~ " + parityBlkNum);
    }
    assertTrue("File length must be positive.", fileLen > 0);
    Path file = new Path(fileName);
    final byte[] data = new byte[fileLen];
    Arrays.fill(data, (byte) 1);
    DFSTestUtil.writeFile(fs, file, data);
    StripedFileTestUtil.waitBlockGroupsReported(fs, fileName);
    LocatedBlocks locatedBlocks = StripedFileTestUtil.getLocatedBlocks(file, fs);
    assertEquals(locatedBlocks.getFileLength(), fileLen);
    LocatedStripedBlock lastBlock = (LocatedStripedBlock) locatedBlocks.getLastLocatedBlock();
    DatanodeInfo[] storageInfos = lastBlock.getLocations();
    byte[] indices = lastBlock.getBlockIndices();
    BitSet bitset = new BitSet(dnNum);
    for (DatanodeInfo storageInfo : storageInfos) {
        bitset.set(dnMap.get(storageInfo));
    }
    int[] dead = generateDeadDnIndices(type, toRecoverBlockNum, indices);
    LOG.info("Note: indices == " + Arrays.toString(indices) + ". Generate errors on datanodes: " + Arrays.toString(dead));
    DatanodeInfo[] dataDNs = new DatanodeInfo[toRecoverBlockNum];
    int[] deadDnIndices = new int[toRecoverBlockNum];
    ExtendedBlock[] blocks = new ExtendedBlock[toRecoverBlockNum];
    File[] replicas = new File[toRecoverBlockNum];
    long[] replicaLengths = new long[toRecoverBlockNum];
    File[] metadatas = new File[toRecoverBlockNum];
    byte[][] replicaContents = new byte[toRecoverBlockNum][];
    Map<ExtendedBlock, DataNode> errorMap = new HashMap<>(dead.length);
    for (int i = 0; i < toRecoverBlockNum; i++) {
        dataDNs[i] = storageInfos[dead[i]];
        deadDnIndices[i] = dnMap.get(dataDNs[i]);
        // Check the block replica file on deadDn before it dead.
        blocks[i] = StripedBlockUtil.constructInternalBlock(lastBlock.getBlock(), cellSize, dataBlkNum, indices[dead[i]]);
        errorMap.put(blocks[i], cluster.getDataNodes().get(deadDnIndices[i]));
        replicas[i] = cluster.getBlockFile(deadDnIndices[i], blocks[i]);
        replicaLengths[i] = replicas[i].length();
        metadatas[i] = cluster.getBlockMetadataFile(deadDnIndices[i], blocks[i]);
        // the block replica on the datanode should be the same as expected
        assertEquals(replicaLengths[i], StripedBlockUtil.getInternalBlockLength(lastBlock.getBlockSize(), cellSize, dataBlkNum, indices[dead[i]]));
        assertTrue(metadatas[i].getName().endsWith(blocks[i].getGenerationStamp() + ".meta"));
        LOG.info("replica " + i + " locates in file: " + replicas[i]);
        replicaContents[i] = DFSTestUtil.readFileAsBytes(replicas[i]);
    }
    int lastGroupDataLen = fileLen % (dataBlkNum * blockSize);
    int lastGroupNumBlk = lastGroupDataLen == 0 ? dataBlkNum : Math.min(dataBlkNum, ((lastGroupDataLen - 1) / cellSize + 1));
    int groupSize = lastGroupNumBlk + parityBlkNum;
    // shutdown datanodes or generate corruption
    int stoppedDN = generateErrors(errorMap, type);
    // Check the locatedBlocks of the file again
    locatedBlocks = StripedFileTestUtil.getLocatedBlocks(file, fs);
    lastBlock = (LocatedStripedBlock) locatedBlocks.getLastLocatedBlock();
    storageInfos = lastBlock.getLocations();
    assertEquals(storageInfos.length, groupSize - stoppedDN);
    int[] targetDNs = new int[dnNum - groupSize];
    int n = 0;
    for (int i = 0; i < dnNum; i++) {
        if (!bitset.get(i)) {
            // not contain replica of the block.
            targetDNs[n++] = i;
        }
    }
    StripedFileTestUtil.waitForReconstructionFinished(file, fs, groupSize);
    targetDNs = sortTargetsByReplicas(blocks, targetDNs);
    // Check the replica on the new target node.
    for (int i = 0; i < toRecoverBlockNum; i++) {
        File replicaAfterReconstruction = cluster.getBlockFile(targetDNs[i], blocks[i]);
        LOG.info("replica after reconstruction " + replicaAfterReconstruction);
        File metadataAfterReconstruction = cluster.getBlockMetadataFile(targetDNs[i], blocks[i]);
        assertEquals(replicaLengths[i], replicaAfterReconstruction.length());
        LOG.info("replica before " + replicas[i]);
        assertTrue(metadataAfterReconstruction.getName().endsWith(blocks[i].getGenerationStamp() + ".meta"));
        byte[] replicaContentAfterReconstruction = DFSTestUtil.readFileAsBytes(replicaAfterReconstruction);
        Assert.assertArrayEquals(replicaContents[i], replicaContentAfterReconstruction);
    }
}

Example 23 with LocatedStripedBlock

use of org.apache.hadoop.hdfs.protocol.LocatedStripedBlock in project hadoop by apache.

the class TestReconstructStripedBlocksWithRackAwareness method testChooseExcessReplicasToDelete.

@Test
public void testChooseExcessReplicasToDelete() throws Exception {
    conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, StripedFileTestUtil.getDefaultECPolicy().getName());
    cluster = new MiniDFSCluster.Builder(conf).racks(racks).hosts(hosts).numDataNodes(hosts.length).build();
    cluster.waitActive();
    fs = cluster.getFileSystem();
    fs.setErasureCodingPolicy(new Path("/"), StripedFileTestUtil.getDefaultECPolicy().getName());
    MiniDFSCluster.DataNodeProperties lastHost = stopDataNode(hosts[hosts.length - 1]);
    final Path file = new Path("/foo");
    DFSTestUtil.createFile(fs, file, cellSize * dataBlocks * 2, (short) 1, 0L);
    // stop host1
    MiniDFSCluster.DataNodeProperties host1 = stopDataNode("host1");
    // bring last host back
    cluster.restartDataNode(lastHost);
    cluster.waitActive();
    // wait for reconstruction to finish
    final short blockNum = (short) (dataBlocks + parityBlocks);
    DFSTestUtil.waitForReplication(fs, file, blockNum, 15 * 1000);
    // restart host1
    cluster.restartDataNode(host1);
    cluster.waitActive();
    for (DataNode dn : cluster.getDataNodes()) {
        if (dn.getDatanodeId().getHostName().equals("host1")) {
            DataNodeTestUtils.triggerBlockReport(dn);
            break;
        }
    }
    // make sure the excess replica is detected, and we delete host1's replica
    // so that we have 6 racks
    DFSTestUtil.waitForReplication(fs, file, blockNum, 15 * 1000);
    LocatedBlocks blks = fs.getClient().getLocatedBlocks(file.toString(), 0);
    LocatedStripedBlock block = (LocatedStripedBlock) blks.getLastLocatedBlock();
    for (DatanodeInfo dn : block.getLocations()) {
        Assert.assertFalse(dn.getHostName().equals("host1"));
    }
}

Example 24 with LocatedStripedBlock

use of org.apache.hadoop.hdfs.protocol.LocatedStripedBlock in project hadoop by apache.

the class TestSortLocatedStripedBlock method createEachLocatedBlock.

private LocatedStripedBlock createEachLocatedBlock(int numDataBlk, int numParityBlk, List<Integer> decommnNodeIndices, List<Integer> targetNodeIndices, ArrayList<String> decommNodeInfo) {
    final long blockGroupID = Long.MIN_VALUE;
    int totalDns = numDataBlk + numParityBlk + targetNodeIndices.size();
    DatanodeInfo[] locs = new DatanodeInfo[totalDns];
    String[] storageIDs = new String[totalDns];
    StorageType[] storageTypes = new StorageType[totalDns];
    byte[] blkIndices = new byte[totalDns];
    // Adding data blocks
    int index = 0;
    for (; index < numDataBlk; index++) {
        blkIndices[index] = (byte) index;
        // Location port always equal to logical index of a block,
        // for easier verification
        locs[index] = DFSTestUtil.getLocalDatanodeInfo(blkIndices[index]);
        locs[index].setLastUpdateMonotonic(Time.monotonicNow());
        storageIDs[index] = locs[index].getDatanodeUuid();
        storageTypes[index] = StorageType.DISK;
        // set decommissioned state
        if (decommnNodeIndices.contains(index)) {
            locs[index].setDecommissioned();
            decommNodeInfo.add(locs[index].toString());
            // Removing it from the list to ensure that all the given nodes are
            // successfully marked as decomissioned.
            decommnNodeIndices.remove(new Integer(index));
        }
    }
    // Adding parity blocks after data blocks
    index = dataBlocks;
    for (int j = numDataBlk; j < numDataBlk + numParityBlk; j++, index++) {
        blkIndices[j] = (byte) index;
        // Location port always equal to logical index of a block,
        // for easier verification
        locs[j] = DFSTestUtil.getLocalDatanodeInfo(blkIndices[j]);
        locs[j].setLastUpdateMonotonic(Time.monotonicNow());
        storageIDs[j] = locs[j].getDatanodeUuid();
        storageTypes[j] = StorageType.DISK;
        // set decommissioned state
        if (decommnNodeIndices.contains(index)) {
            locs[j].setDecommissioned();
            decommNodeInfo.add(locs[j].toString());
            // Removing it from the list to ensure that all the given nodes are
            // successfully marked as decomissioned.
            decommnNodeIndices.remove(new Integer(index));
        }
    }
    // Add extra target nodes to storage list after the parity blocks
    int basePortValue = dataBlocks + parityBlocks;
    index = numDataBlk + numParityBlk;
    for (int i = 0; i < targetNodeIndices.size(); i++, index++) {
        int blkIndexPos = targetNodeIndices.get(i);
        blkIndices[index] = (byte) blkIndexPos;
        // Location port always equal to logical index of a block,
        // for easier verification
        locs[index] = DFSTestUtil.getLocalDatanodeInfo(basePortValue++);
        locs[index].setLastUpdateMonotonic(Time.monotonicNow());
        storageIDs[index] = locs[index].getDatanodeUuid();
        storageTypes[index] = StorageType.DISK;
        // decommissioned by administrator
        if (decommnNodeIndices.contains(blkIndexPos)) {
            locs[index].setDecommissioned();
            decommNodeInfo.add(locs[index].toString());
            // Removing it from the list to ensure that all the given nodes are
            // successfully marked as decomissioned.
            decommnNodeIndices.remove(new Integer(blkIndexPos));
        }
    }
    return new LocatedStripedBlock(new ExtendedBlock("pool", blockGroupID, cellSize, 1001), locs, storageIDs, storageTypes, blkIndices, 0, false, null);
}

Example 25 with LocatedStripedBlock

use of org.apache.hadoop.hdfs.protocol.LocatedStripedBlock in project hadoop by apache.

the class TestSortLocatedStripedBlock method assertBlockIndexAndTokenPosition.

/**
   * Verify block index and token values. Must update block indices and block
   * tokens after sorting.
   */
private void assertBlockIndexAndTokenPosition(List<LocatedBlock> lbs, List<HashMap<DatanodeInfo, Byte>> locToIndexList, List<HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>> locToTokenList) {
    for (int i = 0; i < lbs.size(); i++) {
        LocatedBlock lb = lbs.get(i);
        LocatedStripedBlock stripedBlk = (LocatedStripedBlock) lb;
        HashMap<DatanodeInfo, Byte> locToIndex = locToIndexList.get(i);
        HashMap<DatanodeInfo, Token<BlockTokenIdentifier>> locToToken = locToTokenList.get(i);
        DatanodeInfo[] di = lb.getLocations();
        for (int j = 0; j < di.length; j++) {
            Assert.assertEquals("Block index value mismatches after sorting", (byte) locToIndex.get(di[j]), stripedBlk.getBlockIndices()[j]);
            Assert.assertEquals("Block token value mismatches after sorting", locToToken.get(di[j]), stripedBlk.getBlockTokens()[j]);
        }
    }
}