Examples with HRegionLocation org.apache.hadoop.hbase.HRegionLocation used on opensource projects

Example 86 with HRegionLocation

use of org.apache.hadoop.hbase.HRegionLocation in project hbase by apache.

the class TestAccessController method testAssign.

@Test(timeout = 180000)
public void testAssign() throws Exception {
    List<HRegionLocation> regions;
    try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE)) {
        regions = locator.getAllRegionLocations();
    }
    HRegionLocation location = regions.get(0);
    final HRegionInfo hri = location.getRegionInfo();
    AccessTestAction action = new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            ACCESS_CONTROLLER.preAssign(ObserverContext.createAndPrepare(CP_ENV, null), hri);
            return null;
        }
    };
    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_GROUP_ADMIN);
    verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE);
}

Example 87 with HRegionLocation

use of org.apache.hadoop.hbase.HRegionLocation in project hbase by apache.

the class TestAccessController method testGlobalAuthorizationForNewRegisteredRS.

@Test(timeout = 180000)
public void testGlobalAuthorizationForNewRegisteredRS() throws Exception {
    LOG.debug("Test for global authorization for a new registered RegionServer.");
    MiniHBaseCluster hbaseCluster = TEST_UTIL.getHBaseCluster();
    final Admin admin = TEST_UTIL.getAdmin();
    HTableDescriptor htd = new HTableDescriptor(TEST_TABLE2);
    htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
    createTable(TEST_UTIL, htd);
    // Starting a new RegionServer.
    JVMClusterUtil.RegionServerThread newRsThread = hbaseCluster.startRegionServer();
    final HRegionServer newRs = newRsThread.getRegionServer();
    // Move region to the new RegionServer.
    List<HRegionLocation> regions;
    try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE2)) {
        regions = locator.getAllRegionLocations();
    }
    HRegionLocation location = regions.get(0);
    final HRegionInfo hri = location.getRegionInfo();
    final ServerName server = location.getServerName();
    try (Table table = systemUserConnection.getTable(TEST_TABLE2)) {
        AccessTestAction moveAction = new AccessTestAction() {

            @Override
            public Object run() throws Exception {
                admin.move(hri.getEncodedNameAsBytes(), Bytes.toBytes(newRs.getServerName().getServerName()));
                return null;
            }
        };
        SUPERUSER.runAs(moveAction);
        final int RETRIES_LIMIT = 10;
        int retries = 0;
        while (newRs.getOnlineRegions(TEST_TABLE2).size() < 1 && retries < RETRIES_LIMIT) {
            LOG.debug("Waiting for region to be opened. Already retried " + retries + " times.");
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
            }
            retries++;
            if (retries == RETRIES_LIMIT - 1) {
                fail("Retry exhaust for waiting region to be opened.");
            }
        }
        // Verify write permission for user "admin2" who has the global
        // permissions.
        AccessTestAction putAction = new AccessTestAction() {

            @Override
            public Object run() throws Exception {
                Put put = new Put(Bytes.toBytes("test"));
                put.addColumn(TEST_FAMILY, Bytes.toBytes("qual"), Bytes.toBytes("value"));
                table.put(put);
                return null;
            }
        };
        USER_ADMIN.runAs(putAction);
    }
}

Example 88 with HRegionLocation

use of org.apache.hadoop.hbase.HRegionLocation in project hbase by apache.

the class TestMasterOperationsForRegionReplicas method testCreateTableWithMultipleReplicas.

@Test
public void testCreateTableWithMultipleReplicas() throws Exception {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    final int numRegions = 3;
    final int numReplica = 2;
    try {
        HTableDescriptor desc = new HTableDescriptor(tableName);
        desc.setRegionReplication(numReplica);
        desc.addFamily(new HColumnDescriptor("family"));
        ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
        TEST_UTIL.waitTableEnabled(tableName);
        validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
        List<HRegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
        assert (hris.size() == numRegions * numReplica);
        // check that the master created expected number of RegionState objects
        for (int i = 0; i < numRegions; i++) {
            for (int j = 0; j < numReplica; j++) {
                HRegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j);
                RegionState state = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates().getRegionState(replica);
                assert (state != null);
            }
        }
        List<Result> metaRows = MetaTableAccessor.fullScanRegions(ADMIN.getConnection());
        int numRows = 0;
        for (Result result : metaRows) {
            RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
            HRegionInfo hri = locations.getRegionLocation().getRegionInfo();
            if (!hri.getTable().equals(tableName))
                continue;
            numRows += 1;
            HRegionLocation[] servers = locations.getRegionLocations();
            // have two locations for the replicas of a region, and the locations should be different
            assert (servers.length == 2);
            assert (!servers[0].equals(servers[1]));
        }
        assert (numRows == numRegions);
        // The same verification of the meta as above but with the SnapshotOfRegionAssignmentFromMeta
        // class
        validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica, ADMIN.getConnection());
        // Now kill the master, restart it and see if the assignments are kept
        ServerName master = TEST_UTIL.getHBaseClusterInterface().getClusterStatus().getMaster();
        TEST_UTIL.getHBaseClusterInterface().stopMaster(master);
        TEST_UTIL.getHBaseClusterInterface().waitForMasterToStop(master, 30000);
        TEST_UTIL.getHBaseClusterInterface().startMaster(master.getHostname(), master.getPort());
        TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
        for (int i = 0; i < numRegions; i++) {
            for (int j = 0; j < numReplica; j++) {
                HRegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j);
                RegionState state = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates().getRegionState(replica);
                assert (state != null);
            }
        }
        validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica, ADMIN.getConnection());
        // Now shut the whole cluster down, and verify the assignments are kept so that the
        // availability constraints are met.
        TEST_UTIL.getConfiguration().setBoolean("hbase.master.startup.retainassign", true);
        TEST_UTIL.shutdownMiniHBaseCluster();
        TEST_UTIL.startMiniHBaseCluster(1, numSlaves);
        TEST_UTIL.waitTableEnabled(tableName);
        validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica, ADMIN.getConnection());
        // Now shut the whole cluster down, and verify regions are assigned even if there is only
        // one server running
        TEST_UTIL.shutdownMiniHBaseCluster();
        TEST_UTIL.startMiniHBaseCluster(1, 1);
        TEST_UTIL.waitTableEnabled(tableName);
        validateSingleRegionServerAssignment(ADMIN.getConnection(), numRegions, numReplica);
        for (int i = 1; i < numSlaves; i++) {
            //restore the cluster
            TEST_UTIL.getMiniHBaseCluster().startRegionServer();
        }
        //check on alter table
        ADMIN.disableTable(tableName);
        assert (ADMIN.isTableDisabled(tableName));
        //increase the replica
        desc.setRegionReplication(numReplica + 1);
        ADMIN.modifyTable(tableName, desc);
        ADMIN.enableTable(tableName);
        assert (ADMIN.isTableEnabled(tableName));
        List<HRegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates().getRegionsOfTable(tableName);
        assert (regions.size() == numRegions * (numReplica + 1));
        //decrease the replica(earlier, table was modified to have a replica count of numReplica + 1)
        ADMIN.disableTable(tableName);
        desc.setRegionReplication(numReplica);
        ADMIN.modifyTable(tableName, desc);
        ADMIN.enableTable(tableName);
        assert (ADMIN.isTableEnabled(tableName));
        regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates().getRegionsOfTable(tableName);
        assert (regions.size() == numRegions * numReplica);
        //also make sure the meta table has the replica locations removed
        hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
        assert (hris.size() == numRegions * numReplica);
        //just check that the number of default replica regions in the meta table are the same
        //as the number of regions the table was created with, and the count of the
        //replicas is numReplica for each region
        Map<HRegionInfo, Integer> defaultReplicas = new HashMap<>();
        for (HRegionInfo hri : hris) {
            Integer i;
            HRegionInfo regionReplica0 = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
            defaultReplicas.put(regionReplica0, (i = defaultReplicas.get(regionReplica0)) == null ? 1 : i + 1);
        }
        assert (defaultReplicas.size() == numRegions);
        Collection<Integer> counts = new HashSet<>(defaultReplicas.values());
        assert (counts.size() == 1 && counts.contains(new Integer(numReplica)));
    } finally {
        ADMIN.disableTable(tableName);
        ADMIN.deleteTable(tableName);
    }
}

Example 89 with HRegionLocation

use of org.apache.hadoop.hbase.HRegionLocation in project hbase by apache.

the class TestRegionPlacement method createTable.

/**
   * Create a table with specified table name and region number.
   * @param tablename
   * @param regionNum
   * @return
   * @throws IOException
   */
private static void createTable(TableName tableName, int regionNum) throws IOException {
    int expectedRegions = regionNum;
    byte[][] splitKeys = new byte[expectedRegions - 1][];
    for (int i = 1; i < expectedRegions; i++) {
        byte splitKey = (byte) i;
        splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey };
    }
    HTableDescriptor desc = new HTableDescriptor(tableName);
    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
    admin.createTable(desc, splitKeys);
    try (RegionLocator r = CONNECTION.getRegionLocator(tableName)) {
        List<HRegionLocation> regions = r.getAllRegionLocations();
        assertEquals("Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(), expectedRegions, regions.size());
    }
}

Example 90 with HRegionLocation

use of org.apache.hadoop.hbase.HRegionLocation in project hbase by apache.

the class SnapshotOfRegionAssignmentFromMeta method initialize.

/**
   * Initialize the region assignment snapshot by scanning the hbase:meta table
   * @throws IOException
   */
public void initialize() throws IOException {
    LOG.info("Start to scan the hbase:meta for the current region assignment " + "snappshot");
    // TODO: at some point this code could live in the MetaTableAccessor
    Visitor v = new Visitor() {

        @Override
        public boolean visit(Result result) throws IOException {
            try {
                if (result == null || result.isEmpty())
                    return true;
                RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
                if (rl == null)
                    return true;
                HRegionInfo hri = rl.getRegionLocation(0).getRegionInfo();
                if (hri == null)
                    return true;
                if (hri.getTable() == null)
                    return true;
                if (disabledTables.contains(hri.getTable())) {
                    return true;
                }
                // Are we to include split parents in the list?
                if (excludeOfflinedSplitParents && hri.isSplit())
                    return true;
                HRegionLocation[] hrls = rl.getRegionLocations();
                // Add the current assignment to the snapshot for all replicas
                for (int i = 0; i < hrls.length; i++) {
                    if (hrls[i] == null)
                        continue;
                    hri = hrls[i].getRegionInfo();
                    if (hri == null)
                        continue;
                    addAssignment(hri, hrls[i].getServerName());
                    addRegion(hri);
                }
                hri = rl.getRegionLocation(0).getRegionInfo();
                // the code below is to handle favored nodes
                byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY, FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER);
                if (favoredNodes == null)
                    return true;
                // Add the favored nodes into assignment plan
                ServerName[] favoredServerList = FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes);
                // Add the favored nodes into assignment plan
                existingAssignmentPlan.updateFavoredNodesMap(hri, Arrays.asList(favoredServerList));
                /*
           * Typically there should be FAVORED_NODES_NUM favored nodes for a region in meta. If
           * there is less than FAVORED_NODES_NUM, lets use as much as we can but log a warning.
           */
                if (favoredServerList.length != FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) {
                    LOG.warn("Insufficient favored nodes for region " + hri + " fn: " + Arrays.toString(favoredServerList));
                }
                for (int i = 0; i < favoredServerList.length; i++) {
                    if (i == PRIMARY.ordinal())
                        addPrimaryAssignment(hri, favoredServerList[i]);
                    if (i == SECONDARY.ordinal())
                        addSecondaryAssignment(hri, favoredServerList[i]);
                    if (i == TERTIARY.ordinal())
                        addTeritiaryAssignment(hri, favoredServerList[i]);
                }
                return true;
            } catch (RuntimeException e) {
                LOG.error("Catche remote exception " + e.getMessage() + " when processing" + result);
                throw e;
            }
        }
    };
    // Scan hbase:meta to pick up user regions
    MetaTableAccessor.fullScanRegions(connection, v);
    //regionToRegionServerMap = regions;
    LOG.info("Finished to scan the hbase:meta for the current region assignment" + "snapshot");
}