Examples with HRegionLocation org.apache.hadoop.hbase.HRegionLocation used on opensource projects

Example 81 with HRegionLocation

use of org.apache.hadoop.hbase.HRegionLocation in project hbase by apache.

the class TestServerCustomProtocol method verifyRegionResults.

private void verifyRegionResults(RegionLocator regionLocator, Map<byte[], String> results, String expected, byte[] row) throws Exception {
    for (Map.Entry<byte[], String> e : results.entrySet()) {
        LOG.info("row=" + Bytes.toString(row) + ", expected=" + expected + ", result key=" + Bytes.toString(e.getKey()) + ", value=" + e.getValue());
    }
    HRegionLocation loc = regionLocator.getRegionLocation(row, true);
    byte[] region = loc.getRegionInfo().getRegionName();
    assertTrue("Results should contain region " + Bytes.toStringBinary(region) + " for row '" + Bytes.toStringBinary(row) + "'", results.containsKey(region));
    assertEquals("Invalid result for row '" + Bytes.toStringBinary(row) + "'", expected, results.get(region));
}

Example 82 with HRegionLocation

use of org.apache.hadoop.hbase.HRegionLocation in project hbase by apache.

the class TestServerCustomProtocol method testRowRange.

@Test
public void testRowRange() throws Throwable {
    try (Table table = util.getConnection().getTable(TEST_TABLE);
        RegionLocator locator = util.getConnection().getRegionLocator(TEST_TABLE)) {
        for (HRegionLocation e : locator.getAllRegionLocations()) {
            LOG.info("Region " + e.getRegionInfo().getRegionNameAsString() + ", servername=" + e.getServerName());
        }
        // Here are what regions looked like on a run:
        //
        // test,,1355943549657.c65d4822d8bdecc033a96451f3a0f55d.
        // test,bbb,1355943549661.110393b070dd1ed93441e0bc9b3ffb7e.
        // test,ccc,1355943549665.c3d6d125141359cbbd2a43eaff3cdf74.
        Map<byte[], String> results = ping(table, null, ROW_A);
        // Should contain first region only.
        assertEquals(1, results.size());
        verifyRegionResults(locator, results, ROW_A);
        // Test start row + empty end
        results = ping(table, ROW_BC, null);
        assertEquals(2, results.size());
        // should contain last 2 regions
        HRegionLocation loc = locator.getRegionLocation(ROW_A, true);
        assertNull("Should be missing region for row aaa (prior to start row)", results.get(loc.getRegionInfo().getRegionName()));
        verifyRegionResults(locator, results, ROW_B);
        verifyRegionResults(locator, results, ROW_C);
        // test empty start + end
        results = ping(table, null, ROW_BC);
        // should contain the first 2 regions
        assertEquals(2, results.size());
        verifyRegionResults(locator, results, ROW_A);
        verifyRegionResults(locator, results, ROW_B);
        loc = locator.getRegionLocation(ROW_C, true);
        assertNull("Should be missing region for row ccc (past stop row)", results.get(loc.getRegionInfo().getRegionName()));
        // test explicit start + end
        results = ping(table, ROW_AB, ROW_BC);
        // should contain first 2 regions
        assertEquals(2, results.size());
        verifyRegionResults(locator, results, ROW_A);
        verifyRegionResults(locator, results, ROW_B);
        loc = locator.getRegionLocation(ROW_C, true);
        assertNull("Should be missing region for row ccc (past stop row)", results.get(loc.getRegionInfo().getRegionName()));
        // test single region
        results = ping(table, ROW_B, ROW_BC);
        // should only contain region bbb
        assertEquals(1, results.size());
        verifyRegionResults(locator, results, ROW_B);
        loc = locator.getRegionLocation(ROW_A, true);
        assertNull("Should be missing region for row aaa (prior to start)", results.get(loc.getRegionInfo().getRegionName()));
        loc = locator.getRegionLocation(ROW_C, true);
        assertNull("Should be missing region for row ccc (past stop row)", results.get(loc.getRegionInfo().getRegionName()));
    }
}

Example 83 with HRegionLocation

use of org.apache.hadoop.hbase.HRegionLocation in project hbase by apache.

the class MasterDDLOperationHelper method reOpenAllRegions.

/**
   * Reopen all regions from a table after a schema change operation.
   **/
public static boolean reOpenAllRegions(final MasterProcedureEnv env, final TableName tableName, final List<HRegionInfo> regionInfoList) throws IOException {
    boolean done = false;
    LOG.info("Bucketing regions by region server...");
    List<HRegionLocation> regionLocations = null;
    Connection connection = env.getMasterServices().getConnection();
    try (RegionLocator locator = connection.getRegionLocator(tableName)) {
        regionLocations = locator.getAllRegionLocations();
    }
    // Convert List<HRegionLocation> to Map<HRegionInfo, ServerName>.
    NavigableMap<HRegionInfo, ServerName> hri2Sn = new TreeMap<>();
    for (HRegionLocation location : regionLocations) {
        hri2Sn.put(location.getRegionInfo(), location.getServerName());
    }
    TreeMap<ServerName, List<HRegionInfo>> serverToRegions = Maps.newTreeMap();
    List<HRegionInfo> reRegions = new ArrayList<>();
    for (HRegionInfo hri : regionInfoList) {
        ServerName sn = hri2Sn.get(hri);
        // See HBASE-4578 for more information.
        if (null == sn) {
            LOG.info("Skip " + hri);
            continue;
        }
        if (!serverToRegions.containsKey(sn)) {
            LinkedList<HRegionInfo> hriList = Lists.newLinkedList();
            serverToRegions.put(sn, hriList);
        }
        reRegions.add(hri);
        serverToRegions.get(sn).add(hri);
    }
    LOG.info("Reopening " + reRegions.size() + " regions on " + serverToRegions.size() + " region servers.");
    AssignmentManager am = env.getMasterServices().getAssignmentManager();
    am.setRegionsToReopen(reRegions);
    BulkReOpen bulkReopen = new BulkReOpen(env.getMasterServices(), serverToRegions, am);
    while (true) {
        try {
            if (bulkReopen.bulkReOpen()) {
                done = true;
                break;
            } else {
                LOG.warn("Timeout before reopening all regions");
            }
        } catch (InterruptedException e) {
            LOG.warn("Reopen was interrupted");
            // Preserve the interrupt.
            Thread.currentThread().interrupt();
            break;
        }
    }
    return done;
}

Example 84 with HRegionLocation

use of org.apache.hadoop.hbase.HRegionLocation in project hbase by apache.

the class RegionSplitter method rollingSplit.

static void rollingSplit(TableName tableName, SplitAlgorithm splitAlgo, Configuration conf) throws IOException, InterruptedException {
    final int minOS = conf.getInt("split.outstanding", 2);
    try (Connection connection = ConnectionFactory.createConnection(conf)) {
        // Max outstanding splits. default == 50% of servers
        final int MAX_OUTSTANDING = Math.max(getRegionServerCount(connection) / 2, minOS);
        Path hbDir = FSUtils.getRootDir(conf);
        Path tableDir = FSUtils.getTableDir(hbDir, tableName);
        Path splitFile = new Path(tableDir, "_balancedSplit");
        FileSystem fs = FileSystem.get(conf);
        // Get a list of daughter regions to create
        LinkedList<Pair<byte[], byte[]>> tmpRegionSet = null;
        try (Table table = connection.getTable(tableName)) {
            tmpRegionSet = getSplits(connection, tableName, splitAlgo);
        }
        LinkedList<Pair<byte[], byte[]>> outstanding = Lists.newLinkedList();
        int splitCount = 0;
        final int origCount = tmpRegionSet.size();
        // all splits must compact & we have 1 compact thread, so 2 split
        // requests to the same RS can stall the outstanding split queue.
        // To fix, group the regions into an RS pool and round-robin through it
        LOG.debug("Bucketing regions by regionserver...");
        TreeMap<ServerName, LinkedList<Pair<byte[], byte[]>>> daughterRegions = Maps.newTreeMap();
        // Get a regionLocator.  Need it in below.
        try (RegionLocator regionLocator = connection.getRegionLocator(tableName)) {
            for (Pair<byte[], byte[]> dr : tmpRegionSet) {
                ServerName rsLocation = regionLocator.getRegionLocation(dr.getSecond()).getServerName();
                if (!daughterRegions.containsKey(rsLocation)) {
                    LinkedList<Pair<byte[], byte[]>> entry = Lists.newLinkedList();
                    daughterRegions.put(rsLocation, entry);
                }
                daughterRegions.get(rsLocation).add(dr);
            }
            LOG.debug("Done with bucketing.  Split time!");
            long startTime = System.currentTimeMillis();
            // Open the split file and modify it as splits finish
            byte[] rawData = readFile(fs, splitFile);
            FSDataOutputStream splitOut = fs.create(splitFile);
            try {
                splitOut.write(rawData);
                try {
                    // *** split code ***
                    while (!daughterRegions.isEmpty()) {
                        LOG.debug(daughterRegions.size() + " RS have regions to splt.");
                        // Get ServerName to region count mapping
                        final TreeMap<ServerName, Integer> rsSizes = Maps.newTreeMap();
                        List<HRegionLocation> hrls = regionLocator.getAllRegionLocations();
                        for (HRegionLocation hrl : hrls) {
                            ServerName sn = hrl.getServerName();
                            if (rsSizes.containsKey(sn)) {
                                rsSizes.put(sn, rsSizes.get(sn) + 1);
                            } else {
                                rsSizes.put(sn, 1);
                            }
                        }
                        // first to keep the master from load-balancing regions as we split.
                        for (Map.Entry<ServerName, LinkedList<Pair<byte[], byte[]>>> daughterRegion : daughterRegions.entrySet()) {
                            Pair<byte[], byte[]> dr = null;
                            ServerName rsLoc = daughterRegion.getKey();
                            LinkedList<Pair<byte[], byte[]>> regionList = daughterRegion.getValue();
                            // Find a region in the ServerName list that hasn't been moved
                            LOG.debug("Finding a region on " + rsLoc);
                            while (!regionList.isEmpty()) {
                                dr = regionList.pop();
                                // get current region info
                                byte[] split = dr.getSecond();
                                HRegionLocation regionLoc = regionLocator.getRegionLocation(split);
                                // if this region moved locations
                                ServerName newRs = regionLoc.getServerName();
                                if (newRs.compareTo(rsLoc) != 0) {
                                    LOG.debug("Region with " + splitAlgo.rowToStr(split) + " moved to " + newRs + ". Relocating...");
                                    // relocate it, don't use it right now
                                    if (!daughterRegions.containsKey(newRs)) {
                                        LinkedList<Pair<byte[], byte[]>> entry = Lists.newLinkedList();
                                        daughterRegions.put(newRs, entry);
                                    }
                                    daughterRegions.get(newRs).add(dr);
                                    dr = null;
                                    continue;
                                }
                                // make sure this region wasn't already split
                                byte[] sk = regionLoc.getRegionInfo().getStartKey();
                                if (sk.length != 0) {
                                    if (Bytes.equals(split, sk)) {
                                        LOG.debug("Region already split on " + splitAlgo.rowToStr(split) + ".  Skipping this region...");
                                        ++splitCount;
                                        dr = null;
                                        continue;
                                    }
                                    byte[] start = dr.getFirst();
                                    Preconditions.checkArgument(Bytes.equals(start, sk), splitAlgo.rowToStr(start) + " != " + splitAlgo.rowToStr(sk));
                                }
                                // passed all checks! found a good region
                                break;
                            }
                            if (regionList.isEmpty()) {
                                daughterRegions.remove(rsLoc);
                            }
                            if (dr == null)
                                continue;
                            // we have a good region, time to split!
                            byte[] split = dr.getSecond();
                            LOG.debug("Splitting at " + splitAlgo.rowToStr(split));
                            try (Admin admin = connection.getAdmin()) {
                                admin.split(tableName, split);
                            }
                            LinkedList<Pair<byte[], byte[]>> finished = Lists.newLinkedList();
                            LinkedList<Pair<byte[], byte[]>> local_finished = Lists.newLinkedList();
                            if (conf.getBoolean("split.verify", true)) {
                                // we need to verify and rate-limit our splits
                                outstanding.addLast(dr);
                                // with too many outstanding splits, wait for some to finish
                                while (outstanding.size() >= MAX_OUTSTANDING) {
                                    LOG.debug("Wait for outstanding splits " + outstanding.size());
                                    local_finished = splitScan(outstanding, connection, tableName, splitAlgo);
                                    if (local_finished.isEmpty()) {
                                        Thread.sleep(30 * 1000);
                                    } else {
                                        finished.addAll(local_finished);
                                        outstanding.removeAll(local_finished);
                                        LOG.debug(local_finished.size() + " outstanding splits finished");
                                    }
                                }
                            } else {
                                finished.add(dr);
                            }
                            // mark each finished region as successfully split.
                            for (Pair<byte[], byte[]> region : finished) {
                                splitOut.writeChars("- " + splitAlgo.rowToStr(region.getFirst()) + " " + splitAlgo.rowToStr(region.getSecond()) + "\n");
                                splitCount++;
                                if (splitCount % 10 == 0) {
                                    long tDiff = (System.currentTimeMillis() - startTime) / splitCount;
                                    LOG.debug("STATUS UPDATE: " + splitCount + " / " + origCount + ". Avg Time / Split = " + org.apache.hadoop.util.StringUtils.formatTime(tDiff));
                                }
                            }
                        }
                    }
                    if (conf.getBoolean("split.verify", true)) {
                        while (!outstanding.isEmpty()) {
                            LOG.debug("Finally Wait for outstanding splits " + outstanding.size());
                            LinkedList<Pair<byte[], byte[]>> finished = splitScan(outstanding, connection, tableName, splitAlgo);
                            if (finished.isEmpty()) {
                                Thread.sleep(30 * 1000);
                            } else {
                                outstanding.removeAll(finished);
                                for (Pair<byte[], byte[]> region : finished) {
                                    splitOut.writeChars("- " + splitAlgo.rowToStr(region.getFirst()) + " " + splitAlgo.rowToStr(region.getSecond()) + "\n");
                                    splitCount++;
                                }
                                LOG.debug("Finally " + finished.size() + " outstanding splits finished");
                            }
                        }
                    }
                    LOG.debug("All regions have been successfully split!");
                } finally {
                    long tDiff = System.currentTimeMillis() - startTime;
                    LOG.debug("TOTAL TIME = " + org.apache.hadoop.util.StringUtils.formatTime(tDiff));
                    LOG.debug("Splits = " + splitCount);
                    if (0 < splitCount) {
                        LOG.debug("Avg Time / Split = " + org.apache.hadoop.util.StringUtils.formatTime(tDiff / splitCount));
                    }
                }
            } finally {
                splitOut.close();
                fs.delete(splitFile, false);
            }
        }
    }
}

Example 85 with HRegionLocation

use of org.apache.hadoop.hbase.HRegionLocation in project hbase by apache.

the class TestAccessController method testMove.

@Test(timeout = 180000)
public void testMove() throws Exception {
    List<HRegionLocation> regions;
    try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE)) {
        regions = locator.getAllRegionLocations();
    }
    HRegionLocation location = regions.get(0);
    final HRegionInfo hri = location.getRegionInfo();
    final ServerName server = location.getServerName();
    AccessTestAction action = new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            ACCESS_CONTROLLER.preMove(ObserverContext.createAndPrepare(CP_ENV, null), hri, server, server);
            return null;
        }
    };
    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_GROUP_ADMIN);
    verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE);
}