Examples with WALActionsListener org.apache.hadoop.hbase.regionserver.wal.WALActionsListener used on opensource projects

Example 6 with WALActionsListener

use of org.apache.hadoop.hbase.regionserver.wal.WALActionsListener in project hbase by apache.

the class TestBlockReorder method testHBaseCluster.

/**
   * Test that the hook works within HBase, including when there are multiple blocks.
   */
@Test()
public void testHBaseCluster() throws Exception {
    byte[] sb = "sb".getBytes();
    htu.startMiniZKCluster();
    MiniHBaseCluster hbm = htu.startMiniHBaseCluster(1, 1);
    hbm.waitForActiveAndReadyMaster();
    HRegionServer targetRs = hbm.getMaster();
    // We want to have a datanode with the same name as the region server, so
    //  we're going to get the regionservername, and start a new datanode with this name.
    String host4 = targetRs.getServerName().getHostname();
    LOG.info("Starting a new datanode with the name=" + host4);
    cluster.startDataNodes(conf, 1, true, null, new String[] { "/r4" }, new String[] { host4 }, null);
    cluster.waitClusterUp();
    final int repCount = 3;
    // We use the regionserver file system & conf as we expect it to have the hook.
    conf = targetRs.getConfiguration();
    HFileSystem rfs = (HFileSystem) targetRs.getFileSystem();
    Table h = htu.createTable(TableName.valueOf(name.getMethodName()), sb);
    // Now, we have 4 datanodes and a replication count of 3. So we don't know if the datanode
    // with the same node will be used. We can't really stop an existing datanode, this would
    // make us fall in nasty hdfs bugs/issues. So we're going to try multiple times.
    // Now we need to find the log file, its locations, and look at it
    String rootDir = new Path(FSUtils.getRootDir(conf) + "/" + HConstants.HREGION_LOGDIR_NAME + "/" + targetRs.getServerName().toString()).toUri().getPath();
    DistributedFileSystem mdfs = (DistributedFileSystem) hbm.getMaster().getMasterFileSystem().getFileSystem();
    int nbTest = 0;
    while (nbTest < 10) {
        final List<Region> regions = targetRs.getOnlineRegions(h.getName());
        final CountDownLatch latch = new CountDownLatch(regions.size());
        // listen for successful log rolls
        final WALActionsListener listener = new WALActionsListener.Base() {

            @Override
            public void postLogRoll(final Path oldPath, final Path newPath) throws IOException {
                latch.countDown();
            }
        };
        for (Region region : regions) {
            ((HRegion) region).getWAL().registerWALActionsListener(listener);
        }
        htu.getAdmin().rollWALWriter(targetRs.getServerName());
        // wait
        try {
            latch.await();
        } catch (InterruptedException exception) {
            LOG.warn("Interrupted while waiting for the wal of '" + targetRs + "' to roll. If later " + "tests fail, it's probably because we should still be waiting.");
            Thread.currentThread().interrupt();
        }
        for (Region region : regions) {
            ((HRegion) region).getWAL().unregisterWALActionsListener(listener);
        }
        // We need a sleep as the namenode is informed asynchronously
        Thread.sleep(100);
        // insert one put to ensure a minimal size
        Put p = new Put(sb);
        p.addColumn(sb, sb, sb);
        h.put(p);
        DirectoryListing dl = dfs.getClient().listPaths(rootDir, HdfsFileStatus.EMPTY_NAME);
        HdfsFileStatus[] hfs = dl.getPartialListing();
        // As we wrote a put, we should have at least one log file.
        Assert.assertTrue(hfs.length >= 1);
        for (HdfsFileStatus hf : hfs) {
            // Because this is a live cluster, log files might get archived while we're processing
            try {
                LOG.info("Log file found: " + hf.getLocalName() + " in " + rootDir);
                String logFile = rootDir + "/" + hf.getLocalName();
                FileStatus fsLog = rfs.getFileStatus(new Path(logFile));
                LOG.info("Checking log file: " + logFile);
                // Now checking that the hook is up and running
                // We can't call directly getBlockLocations, it's not available in HFileSystem
                // We're trying multiple times to be sure, as the order is random
                BlockLocation[] bls = rfs.getFileBlockLocations(fsLog, 0, 1);
                if (bls.length > 0) {
                    BlockLocation bl = bls[0];
                    LOG.info(bl.getHosts().length + " replicas for block 0 in " + logFile + " ");
                    for (int i = 0; i < bl.getHosts().length - 1; i++) {
                        LOG.info(bl.getHosts()[i] + "    " + logFile);
                        Assert.assertNotSame(bl.getHosts()[i], host4);
                    }
                    String last = bl.getHosts()[bl.getHosts().length - 1];
                    LOG.info(last + "    " + logFile);
                    if (host4.equals(last)) {
                        nbTest++;
                        LOG.info(logFile + " is on the new datanode and is ok");
                        if (bl.getHosts().length == 3) {
                            // We can test this case from the file system as well
                            // Checking the underlying file system. Multiple times as the order is random
                            testFromDFS(dfs, logFile, repCount, host4);
                            // now from the master
                            testFromDFS(mdfs, logFile, repCount, host4);
                        }
                    }
                }
            } catch (FileNotFoundException exception) {
                LOG.debug("Failed to find log file '" + hf.getLocalName() + "'; it probably was " + "archived out from under us so we'll ignore and retry. If this test hangs " + "indefinitely you should treat this failure as a symptom.", exception);
            } catch (RemoteException exception) {
                if (exception.unwrapRemoteException() instanceof FileNotFoundException) {
                    LOG.debug("Failed to find log file '" + hf.getLocalName() + "'; it probably was " + "archived out from under us so we'll ignore and retry. If this test hangs " + "indefinitely you should treat this failure as a symptom.", exception);
                } else {
                    throw exception;
                }
            }
        }
    }
}