Examples with MiniHBaseCluster org.apache.hadoop.hbase.MiniHBaseCluster used on opensource projects

Example 46 with MiniHBaseCluster

use of org.apache.hadoop.hbase.MiniHBaseCluster in project hbase by apache.

the class TestSplitOrMergeStatus method waitForMergable.

private void waitForMergable(Admin admin, TableName t) throws InterruptedException, IOException {
    // Wait for the Regions to be mergeable
    MiniHBaseCluster miniCluster = TEST_UTIL.getMiniHBaseCluster();
    int mergeable = 0;
    while (mergeable < 2) {
        Thread.sleep(100);
        admin.majorCompact(t);
        mergeable = 0;
        for (JVMClusterUtil.RegionServerThread regionThread : miniCluster.getRegionServerThreads()) {
            for (Region region : regionThread.getRegionServer().getOnlineRegions(t)) {
                mergeable += ((HRegion) region).isMergeable() ? 1 : 0;
            }
        }
    }
}

Example 47 with MiniHBaseCluster

use of org.apache.hadoop.hbase.MiniHBaseCluster in project hbase by apache.

the class TestFromClientSide method testUnmanagedHConnectionReconnect.

/**
   * test of that unmanaged HConnections are able to reconnect
   * properly (see HBASE-5058)
   *
   * @throws Exception
   */
@Test
public void testUnmanagedHConnectionReconnect() throws Exception {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
    Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
    Table t = conn.getTable(tableName);
    try (Admin admin = conn.getAdmin()) {
        assertTrue(admin.tableExists(tableName));
        assertTrue(t.get(new Get(ROW)).isEmpty());
    }
    // stop the master
    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
    cluster.stopMaster(0, false);
    cluster.waitOnMaster(0);
    // start up a new master
    cluster.startMaster();
    assertTrue(cluster.waitForActiveAndReadyMaster());
    // Admin and can connect to the new master;
    try (Admin admin = conn.getAdmin()) {
        assertTrue(admin.tableExists(tableName));
        assertTrue(admin.getClusterStatus().getServersSize() == SLAVES + 1);
    }
}

Example 48 with MiniHBaseCluster

use of org.apache.hadoop.hbase.MiniHBaseCluster in project hbase by apache.

the class TestBlockReorder method testHBaseCluster.

/**
   * Test that the hook works within HBase, including when there are multiple blocks.
   */
@Test()
public void testHBaseCluster() throws Exception {
    byte[] sb = "sb".getBytes();
    htu.startMiniZKCluster();
    MiniHBaseCluster hbm = htu.startMiniHBaseCluster(1, 1);
    hbm.waitForActiveAndReadyMaster();
    HRegionServer targetRs = hbm.getMaster();
    // We want to have a datanode with the same name as the region server, so
    //  we're going to get the regionservername, and start a new datanode with this name.
    String host4 = targetRs.getServerName().getHostname();
    LOG.info("Starting a new datanode with the name=" + host4);
    cluster.startDataNodes(conf, 1, true, null, new String[] { "/r4" }, new String[] { host4 }, null);
    cluster.waitClusterUp();
    final int repCount = 3;
    // We use the regionserver file system & conf as we expect it to have the hook.
    conf = targetRs.getConfiguration();
    HFileSystem rfs = (HFileSystem) targetRs.getFileSystem();
    Table h = htu.createTable(TableName.valueOf(name.getMethodName()), sb);
    // Now, we have 4 datanodes and a replication count of 3. So we don't know if the datanode
    // with the same node will be used. We can't really stop an existing datanode, this would
    // make us fall in nasty hdfs bugs/issues. So we're going to try multiple times.
    // Now we need to find the log file, its locations, and look at it
    String rootDir = new Path(FSUtils.getRootDir(conf) + "/" + HConstants.HREGION_LOGDIR_NAME + "/" + targetRs.getServerName().toString()).toUri().getPath();
    DistributedFileSystem mdfs = (DistributedFileSystem) hbm.getMaster().getMasterFileSystem().getFileSystem();
    int nbTest = 0;
    while (nbTest < 10) {
        final List<Region> regions = targetRs.getOnlineRegions(h.getName());
        final CountDownLatch latch = new CountDownLatch(regions.size());
        // listen for successful log rolls
        final WALActionsListener listener = new WALActionsListener.Base() {

            @Override
            public void postLogRoll(final Path oldPath, final Path newPath) throws IOException {
                latch.countDown();
            }
        };
        for (Region region : regions) {
            ((HRegion) region).getWAL().registerWALActionsListener(listener);
        }
        htu.getAdmin().rollWALWriter(targetRs.getServerName());
        // wait
        try {
            latch.await();
        } catch (InterruptedException exception) {
            LOG.warn("Interrupted while waiting for the wal of '" + targetRs + "' to roll. If later " + "tests fail, it's probably because we should still be waiting.");
            Thread.currentThread().interrupt();
        }
        for (Region region : regions) {
            ((HRegion) region).getWAL().unregisterWALActionsListener(listener);
        }
        // We need a sleep as the namenode is informed asynchronously
        Thread.sleep(100);
        // insert one put to ensure a minimal size
        Put p = new Put(sb);
        p.addColumn(sb, sb, sb);
        h.put(p);
        DirectoryListing dl = dfs.getClient().listPaths(rootDir, HdfsFileStatus.EMPTY_NAME);
        HdfsFileStatus[] hfs = dl.getPartialListing();
        // As we wrote a put, we should have at least one log file.
        Assert.assertTrue(hfs.length >= 1);
        for (HdfsFileStatus hf : hfs) {
            // Because this is a live cluster, log files might get archived while we're processing
            try {
                LOG.info("Log file found: " + hf.getLocalName() + " in " + rootDir);
                String logFile = rootDir + "/" + hf.getLocalName();
                FileStatus fsLog = rfs.getFileStatus(new Path(logFile));
                LOG.info("Checking log file: " + logFile);
                // Now checking that the hook is up and running
                // We can't call directly getBlockLocations, it's not available in HFileSystem
                // We're trying multiple times to be sure, as the order is random
                BlockLocation[] bls = rfs.getFileBlockLocations(fsLog, 0, 1);
                if (bls.length > 0) {
                    BlockLocation bl = bls[0];
                    LOG.info(bl.getHosts().length + " replicas for block 0 in " + logFile + " ");
                    for (int i = 0; i < bl.getHosts().length - 1; i++) {
                        LOG.info(bl.getHosts()[i] + "    " + logFile);
                        Assert.assertNotSame(bl.getHosts()[i], host4);
                    }
                    String last = bl.getHosts()[bl.getHosts().length - 1];
                    LOG.info(last + "    " + logFile);
                    if (host4.equals(last)) {
                        nbTest++;
                        LOG.info(logFile + " is on the new datanode and is ok");
                        if (bl.getHosts().length == 3) {
                            // We can test this case from the file system as well
                            // Checking the underlying file system. Multiple times as the order is random
                            testFromDFS(dfs, logFile, repCount, host4);
                            // now from the master
                            testFromDFS(mdfs, logFile, repCount, host4);
                        }
                    }
                }
            } catch (FileNotFoundException exception) {
                LOG.debug("Failed to find log file '" + hf.getLocalName() + "'; it probably was " + "archived out from under us so we'll ignore and retry. If this test hangs " + "indefinitely you should treat this failure as a symptom.", exception);
            } catch (RemoteException exception) {
                if (exception.unwrapRemoteException() instanceof FileNotFoundException) {
                    LOG.debug("Failed to find log file '" + hf.getLocalName() + "'; it probably was " + "archived out from under us so we'll ignore and retry. If this test hangs " + "indefinitely you should treat this failure as a symptom.", exception);
                } else {
                    throw exception;
                }
            }
        }
    }
}

Example 49 with MiniHBaseCluster

use of org.apache.hadoop.hbase.MiniHBaseCluster in project hbase by apache.

the class TestHBaseFsckOneRS method testCleanUpDaughtersNotInMetaAfterFailedSplit.

@Test(timeout = 180000)
public void testCleanUpDaughtersNotInMetaAfterFailedSplit() throws Exception {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
    try {
        HTableDescriptor desc = new HTableDescriptor(tableName);
        desc.addFamily(new HColumnDescriptor(Bytes.toBytes("f")));
        createTable(TEST_UTIL, desc, null);
        tbl = connection.getTable(desc.getTableName());
        for (int i = 0; i < 5; i++) {
            Put p1 = new Put(("r" + i).getBytes());
            p1.addColumn(Bytes.toBytes("f"), "q1".getBytes(), "v".getBytes());
            tbl.put(p1);
        }
        admin.flush(desc.getTableName());
        List<HRegion> regions = cluster.getRegions(desc.getTableName());
        int serverWith = cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
        HRegionServer regionServer = cluster.getRegionServer(serverWith);
        byte[] parentRegionName = regions.get(0).getRegionInfo().getRegionName();
        cluster.getServerWith(parentRegionName);
        // Create daughters without adding to META table
        MasterProcedureEnv env = cluster.getMaster().getMasterProcedureExecutor().getEnvironment();
        SplitTableRegionProcedure splitR = new SplitTableRegionProcedure(env, regions.get(0).getRegionInfo(), Bytes.toBytes("r3"));
        splitR.prepareSplitRegion(env);
        splitR.setRegionStateToSplitting(env);
        splitR.closeParentRegionForSplit(env);
        splitR.createDaughterRegions(env);
        AssignmentManager am = cluster.getMaster().getAssignmentManager();
        for (RegionState state : am.getRegionStates().getRegionsInTransition()) {
            am.regionOffline(state.getRegion());
        }
        Map<HRegionInfo, ServerName> regionsMap = new HashMap<>();
        regionsMap.put(regions.get(0).getRegionInfo(), regionServer.getServerName());
        am.assign(regionsMap);
        am.waitForAssignment(regions.get(0).getRegionInfo());
        HBaseFsck hbck = doFsck(conf, false);
        assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED });
        // holes are separate from overlap groups
        assertEquals(0, hbck.getOverlapGroups(tableName).size());
        // fix hole
        assertErrors(doFsck(conf, false, true, false, false, false, false, false, false, false, false, false, false, null), new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED });
        // check that hole fixed
        assertNoErrors(doFsck(conf, false));
        assertEquals(5, countRows());
    } finally {
        if (tbl != null) {
            tbl.close();
            tbl = null;
        }
        cleanupTable(tableName);
    }
}

Example 50 with MiniHBaseCluster

use of org.apache.hadoop.hbase.MiniHBaseCluster in project hbase by apache.

the class TestRegionMover method testExclude.

/**
   * To test that we successfully exclude a server from the unloading process We test for the number
   * of regions on Excluded server and also test that regions are unloaded successfully
   * @throws Exception
   */
@Test
public void testExclude() throws Exception {
    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
    File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
    FileWriter fos = new FileWriter(excludeFile);
    HRegionServer excludeServer = cluster.getRegionServer(1);
    String excludeHostname = excludeServer.getServerName().getHostname();
    int excludeServerPort = excludeServer.getServerName().getPort();
    int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
    String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort);
    fos.write(excludeServerName);
    fos.close();
    HRegionServer regionServer = cluster.getRegionServer(0);
    String rsName = regionServer.getServerName().getHostname();
    int port = regionServer.getServerName().getPort();
    String rs = rsName + ":" + Integer.toString(port);
    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs).ack(true).excludeFile(excludeFile.getCanonicalPath());
    RegionMover rm = rmBuilder.build();
    rm.setConf(TEST_UTIL.getConfiguration());
    rm.unload();
    LOG.info("Unloading " + rs);
    assertEquals(0, regionServer.getNumberOfOnlineRegions());
    assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions());
    LOG.info("Before:" + regionsExcludeServer + " After:" + cluster.getRegionServer(1).getNumberOfOnlineRegions());
}