Examples with LocatedBlocks org.apache.hadoop.hdfs.protocol.LocatedBlocks used on opensource projects

Example 31 with LocatedBlocks

use of org.apache.hadoop.hdfs.protocol.LocatedBlocks in project hadoop by apache.

the class TestLeaseRecovery2 method hardLeaseRecoveryRestartHelper.

public void hardLeaseRecoveryRestartHelper(boolean doRename, int size) throws Exception {
    if (size < 0) {
        size = AppendTestUtil.nextInt(FILE_SIZE + 1);
    }
    //create a file
    String fileStr = "/hardLeaseRecovery";
    AppendTestUtil.LOG.info("filestr=" + fileStr);
    Path filePath = new Path(fileStr);
    FSDataOutputStream stm = dfs.create(filePath, true, BUF_SIZE, REPLICATION_NUM, BLOCK_SIZE);
    assertTrue(dfs.dfs.exists(fileStr));
    // write bytes into the file.
    AppendTestUtil.LOG.info("size=" + size);
    stm.write(buffer, 0, size);
    String originalLeaseHolder = NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(), fileStr);
    assertFalse("original lease holder should not be the NN", originalLeaseHolder.equals(HdfsServerConstants.NAMENODE_LEASE_HOLDER));
    // hflush file
    AppendTestUtil.LOG.info("hflush");
    stm.hflush();
    // check visible length
    final HdfsDataInputStream in = (HdfsDataInputStream) dfs.open(filePath);
    Assert.assertEquals(size, in.getVisibleLength());
    in.close();
    if (doRename) {
        fileStr += ".renamed";
        Path renamedPath = new Path(fileStr);
        assertTrue(dfs.rename(filePath, renamedPath));
        filePath = renamedPath;
    }
    // kill the lease renewal thread
    AppendTestUtil.LOG.info("leasechecker.interruptAndJoin()");
    dfs.dfs.getLeaseRenewer().interruptAndJoin();
    // won't actually get completed during lease recovery.
    for (DataNode dn : cluster.getDataNodes()) {
        DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
    }
    // set the hard limit to be 1 second 
    cluster.setLeasePeriod(LONG_LEASE_PERIOD, SHORT_LEASE_PERIOD);
    // Make sure lease recovery begins.
    final String path = fileStr;
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            return HdfsServerConstants.NAMENODE_LEASE_HOLDER.equals(NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(), path));
        }
    }, (int) SHORT_LEASE_PERIOD, (int) SHORT_LEASE_PERIOD * 10);
    // Normally, the in-progress edit log would be finalized by
    // FSEditLog#endCurrentLogSegment.  For testing purposes, we
    // disable that here.
    FSEditLog spyLog = spy(cluster.getNameNode().getFSImage().getEditLog());
    doNothing().when(spyLog).endCurrentLogSegment(Mockito.anyBoolean());
    DFSTestUtil.setEditLogForTesting(cluster.getNamesystem(), spyLog);
    cluster.restartNameNode(false);
    checkLease(fileStr, size);
    // Let the DNs send heartbeats again.
    for (DataNode dn : cluster.getDataNodes()) {
        DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, false);
    }
    cluster.waitActive();
    // set the hard limit to be 1 second, to initiate lease recovery. 
    cluster.setLeasePeriod(LONG_LEASE_PERIOD, SHORT_LEASE_PERIOD);
    // wait for lease recovery to complete
    LocatedBlocks locatedBlocks;
    do {
        Thread.sleep(SHORT_LEASE_PERIOD);
        locatedBlocks = dfs.dfs.getLocatedBlocks(fileStr, 0L, size);
    } while (locatedBlocks.isUnderConstruction());
    assertEquals(size, locatedBlocks.getFileLength());
    // make sure that the client can't write data anymore.
    try {
        stm.write('b');
        stm.hflush();
        fail("Should not be able to flush after we've lost the lease");
    } catch (IOException e) {
        LOG.info("Expceted exception on write/hflush", e);
    }
    try {
        stm.close();
        fail("Should not be able to close after we've lost the lease");
    } catch (IOException e) {
        LOG.info("Expected exception on close", e);
    }
    // verify data
    AppendTestUtil.LOG.info("File size is good. Now validating sizes from datanodes...");
    AppendTestUtil.checkFullFile(dfs, filePath, size, buffer, fileStr);
}

Example 32 with LocatedBlocks

use of org.apache.hadoop.hdfs.protocol.LocatedBlocks in project hadoop by apache.

the class TestLeaseRecovery method testBlockRecoveryWithLessMetafile.

/**
   * Block Recovery when the meta file not having crcs for all chunks in block
   * file
   */
@Test
public void testBlockRecoveryWithLessMetafile() throws Exception {
    Configuration conf = new Configuration();
    conf.set(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY, UserGroupInformation.getCurrentUser().getShortUserName());
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    Path file = new Path("/testRecoveryFile");
    DistributedFileSystem dfs = cluster.getFileSystem();
    FSDataOutputStream out = dfs.create(file);
    final int FILE_SIZE = 2 * 1024 * 1024;
    int count = 0;
    while (count < FILE_SIZE) {
        out.writeBytes("Data");
        count += 4;
    }
    out.hsync();
    // abort the original stream
    ((DFSOutputStream) out.getWrappedStream()).abort();
    LocatedBlocks locations = cluster.getNameNodeRpc().getBlockLocations(file.toString(), 0, count);
    ExtendedBlock block = locations.get(0).getBlock();
    // Calculate meta file size
    // From DataNode.java, checksum size is given by:
    // (length of data + BYTE_PER_CHECKSUM - 1)/BYTES_PER_CHECKSUM *
    // CHECKSUM_SIZE
    // CRC32 & CRC32C
    final int CHECKSUM_SIZE = 4;
    final int bytesPerChecksum = conf.getInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT);
    final int metaFileSize = (FILE_SIZE + bytesPerChecksum - 1) / bytesPerChecksum * CHECKSUM_SIZE + // meta file header is 8 bytes
    8;
    final int newMetaFileSize = metaFileSize - CHECKSUM_SIZE;
    // Corrupt the block meta file by dropping checksum for bytesPerChecksum
    // bytes. Lease recovery is expected to recover the uncorrupted file length.
    cluster.truncateMeta(0, block, newMetaFileSize);
    // restart DN to make replica to RWR
    DataNodeProperties dnProp = cluster.stopDataNode(0);
    cluster.restartDataNode(dnProp, true);
    // try to recover the lease
    DistributedFileSystem newdfs = (DistributedFileSystem) FileSystem.newInstance(cluster.getConfiguration(0));
    count = 0;
    while (++count < 10 && !newdfs.recoverLease(file)) {
        Thread.sleep(1000);
    }
    assertTrue("File should be closed", newdfs.recoverLease(file));
    // Verify file length after lease recovery. The new file length should not
    // include the bytes with corrupted checksum.
    final long expectedNewFileLen = FILE_SIZE - bytesPerChecksum;
    final long newFileLen = newdfs.getFileStatus(file).getLen();
    assertEquals(newFileLen, expectedNewFileLen);
}

Example 33 with LocatedBlocks

use of org.apache.hadoop.hdfs.protocol.LocatedBlocks in project hadoop by apache.

the class TestPersistBlocks method testRestartDfsWithAbandonedBlock.

@Test
public void testRestartDfsWithAbandonedBlock() throws Exception {
    final Configuration conf = new HdfsConfiguration();
    // Turn off persistent IPC, so that the DFSClient can survive NN restart
    conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);
    MiniDFSCluster cluster = null;
    long len = 0;
    FSDataOutputStream stream;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
        FileSystem fs = cluster.getFileSystem();
        // Creating a file with 4096 blockSize to write multiple blocks
        stream = fs.create(FILE_PATH, true, BLOCK_SIZE, (short) 1, BLOCK_SIZE);
        stream.write(DATA_BEFORE_RESTART);
        stream.hflush();
        // Wait for all of the blocks to get through
        while (len < BLOCK_SIZE * (NUM_BLOCKS - 1)) {
            FileStatus status = fs.getFileStatus(FILE_PATH);
            len = status.getLen();
            Thread.sleep(100);
        }
        // Abandon the last block
        DFSClient dfsclient = DFSClientAdapter.getDFSClient((DistributedFileSystem) fs);
        HdfsFileStatus fileStatus = dfsclient.getNamenode().getFileInfo(FILE_NAME);
        LocatedBlocks blocks = dfsclient.getNamenode().getBlockLocations(FILE_NAME, 0, BLOCK_SIZE * NUM_BLOCKS);
        assertEquals(NUM_BLOCKS, blocks.getLocatedBlocks().size());
        LocatedBlock b = blocks.getLastLocatedBlock();
        dfsclient.getNamenode().abandonBlock(b.getBlock(), fileStatus.getFileId(), FILE_NAME, dfsclient.clientName);
        // explicitly do NOT close the file.
        cluster.restartNameNode();
        // Check that the file has no less bytes than before the restart
        // This would mean that blocks were successfully persisted to the log
        FileStatus status = fs.getFileStatus(FILE_PATH);
        assertTrue("Length incorrect: " + status.getLen(), status.getLen() == len - BLOCK_SIZE);
        // Verify the data showed up from before restart, sans abandoned block.
        FSDataInputStream readStream = fs.open(FILE_PATH);
        try {
            byte[] verifyBuf = new byte[DATA_BEFORE_RESTART.length - BLOCK_SIZE];
            IOUtils.readFully(readStream, verifyBuf, 0, verifyBuf.length);
            byte[] expectedBuf = new byte[DATA_BEFORE_RESTART.length - BLOCK_SIZE];
            System.arraycopy(DATA_BEFORE_RESTART, 0, expectedBuf, 0, expectedBuf.length);
            assertArrayEquals(expectedBuf, verifyBuf);
        } finally {
            IOUtils.closeStream(readStream);
        }
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 34 with LocatedBlocks

use of org.apache.hadoop.hdfs.protocol.LocatedBlocks in project hadoop by apache.

the class TestMaintenanceState method testFileCloseAfterEnteringMaintenance.

@Test(timeout = 120000)
public void testFileCloseAfterEnteringMaintenance() throws Exception {
    LOG.info("Starting testFileCloseAfterEnteringMaintenance");
    int expirationInMs = 30 * 1000;
    int numDataNodes = 3;
    int numNameNodes = 1;
    getConf().setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, 2);
    startCluster(numNameNodes, numDataNodes);
    getCluster().waitActive();
    FSNamesystem fsn = getCluster().getNameNode().getNamesystem();
    List<String> hosts = new ArrayList<>();
    for (DataNode dn : getCluster().getDataNodes()) {
        hosts.add(dn.getDisplayName());
        putNodeInService(0, dn.getDatanodeUuid());
    }
    assertEquals(numDataNodes, fsn.getNumLiveDataNodes());
    Path openFile = new Path("/testClosingFileInMaintenance.dat");
    // Lets write 2 blocks of data to the openFile
    writeFile(getCluster().getFileSystem(), openFile, (short) 3);
    // Lets write some more data and keep the file open
    FSDataOutputStream fsDataOutputStream = getCluster().getFileSystem().append(openFile);
    byte[] bytes = new byte[1024];
    fsDataOutputStream.write(bytes);
    fsDataOutputStream.hsync();
    LocatedBlocks lbs = NameNodeAdapter.getBlockLocations(getCluster().getNameNode(0), openFile.toString(), 0, 3 * blockSize);
    DatanodeInfo[] dnInfos4LastBlock = lbs.getLastLocatedBlock().getLocations();
    // Request maintenance for DataNodes 1 and 2 which has the last block.
    takeNodeOutofService(0, Lists.newArrayList(dnInfos4LastBlock[0].getDatanodeUuid(), dnInfos4LastBlock[1].getDatanodeUuid()), Time.now() + expirationInMs, null, null, AdminStates.ENTERING_MAINTENANCE);
    // Closing the file should succeed even when the
    // last blocks' nodes are entering maintenance.
    fsDataOutputStream.close();
    cleanupFile(getCluster().getFileSystem(), openFile);
}

Example 35 with LocatedBlocks

use of org.apache.hadoop.hdfs.protocol.LocatedBlocks in project hadoop by apache.

the class TestBalancerWithNodeGroup method testBalancerWithRackLocality.

/**
   * Create a cluster with even distribution, and a new empty node is added to
   * the cluster, then test rack locality for balancer policy. 
   */
@Test(timeout = 60000)
public void testBalancerWithRackLocality() throws Exception {
    Configuration conf = createConf();
    long[] capacities = new long[] { CAPACITY, CAPACITY };
    String[] racks = new String[] { RACK0, RACK1 };
    String[] nodeGroups = new String[] { NODEGROUP0, NODEGROUP1 };
    int numOfDatanodes = capacities.length;
    assertEquals(numOfDatanodes, racks.length);
    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).numDataNodes(capacities.length).racks(racks).simulatedCapacities(capacities);
    MiniDFSClusterWithNodeGroup.setNodeGroups(nodeGroups);
    cluster = new MiniDFSClusterWithNodeGroup(builder);
    try {
        cluster.waitActive();
        client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy();
        long totalCapacity = TestBalancer.sum(capacities);
        // fill up the cluster to be 30% full
        long totalUsedSpace = totalCapacity * 3 / 10;
        long length = totalUsedSpace / numOfDatanodes;
        TestBalancer.createFile(cluster, filePath, length, (short) numOfDatanodes, 0);
        LocatedBlocks lbs = client.getBlockLocations(filePath.toUri().getPath(), 0, length);
        Set<ExtendedBlock> before = getBlocksOnRack(lbs.getLocatedBlocks(), RACK0);
        long newCapacity = CAPACITY;
        String newRack = RACK1;
        String newNodeGroup = NODEGROUP2;
        // start up an empty node with the same capacity and on the same rack
        cluster.startDataNodes(conf, 1, true, null, new String[] { newRack }, new long[] { newCapacity }, new String[] { newNodeGroup });
        totalCapacity += newCapacity;
        // run balancer and validate results
        runBalancerCanFinish(conf, totalUsedSpace, totalCapacity);
        lbs = client.getBlockLocations(filePath.toUri().getPath(), 0, length);
        Set<ExtendedBlock> after = getBlocksOnRack(lbs.getLocatedBlocks(), RACK0);
        assertEquals(before, after);
    } finally {
        cluster.shutdown();
    }
}