Examples with RegionLocator org.apache.hadoop.hbase.client.RegionLocator used on opensource projects

Example 21 with RegionLocator

use of org.apache.hadoop.hbase.client.RegionLocator in project hbase by apache.

the class TestLoadIncrementalHFilesSplitRecovery method testSplitTmpFileCleanUp.

/**
   * This test creates a table with many small regions.  The bulk load files
   * would be splitted multiple times before all of them can be loaded successfully.
   */
@Test(timeout = 120000)
public void testSplitTmpFileCleanUp() throws Exception {
    final TableName table = TableName.valueOf(name.getMethodName());
    byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("row_00000010"), Bytes.toBytes("row_00000020"), Bytes.toBytes("row_00000030"), Bytes.toBytes("row_00000040"), Bytes.toBytes("row_00000050") };
    try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) {
        setupTableWithSplitkeys(table, 10, SPLIT_KEYS);
        LoadIncrementalHFiles lih = new LoadIncrementalHFiles(util.getConfiguration());
        // create HFiles
        Path bulk = buildBulkFiles(table, 2);
        try (Table t = connection.getTable(table);
            RegionLocator locator = connection.getRegionLocator(table);
            Admin admin = connection.getAdmin()) {
            lih.doBulkLoad(bulk, admin, t, locator);
        }
        // family path
        Path tmpPath = new Path(bulk, family(0));
        // TMP_DIR under family path
        tmpPath = new Path(tmpPath, LoadIncrementalHFiles.TMP_DIR);
        FileSystem fs = bulk.getFileSystem(util.getConfiguration());
        // HFiles have been splitted, there is TMP_DIR
        assertTrue(fs.exists(tmpPath));
        // TMP_DIR should have been cleaned-up
        assertNull(LoadIncrementalHFiles.TMP_DIR + " should be empty.", FSUtils.listStatus(fs, tmpPath));
        assertExpectedTable(connection, table, ROWCOUNT, 2);
    }
}

Example 22 with RegionLocator

use of org.apache.hadoop.hbase.client.RegionLocator in project hbase by apache.

the class TestLoadIncrementalHFilesSplitRecovery method testGroupOrSplitWhenRegionHoleExistsInMeta.

@Test(timeout = 120000)
public void testGroupOrSplitWhenRegionHoleExistsInMeta() throws Exception {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("row_00000100") };
    // Share connection. We were failing to find the table with our new reverse scan because it
    // looks for first region, not any region -- that is how it works now.  The below removes first
    // region in test.  Was reliant on the Connection caching having first region.
    Connection connection = ConnectionFactory.createConnection(util.getConfiguration());
    Table table = connection.getTable(tableName);
    setupTableWithSplitkeys(tableName, 10, SPLIT_KEYS);
    Path dir = buildBulkFiles(tableName, 2);
    final AtomicInteger countedLqis = new AtomicInteger();
    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration()) {

        @Override
        protected Pair<List<LoadQueueItem>, String> groupOrSplit(Multimap<ByteBuffer, LoadQueueItem> regionGroups, final LoadQueueItem item, final Table htable, final Pair<byte[][], byte[][]> startEndKeys) throws IOException {
            Pair<List<LoadQueueItem>, String> lqis = super.groupOrSplit(regionGroups, item, htable, startEndKeys);
            if (lqis != null && lqis.getFirst() != null) {
                countedLqis.addAndGet(lqis.getFirst().size());
            }
            return lqis;
        }
    };
    // do bulkload when there is no region hole in hbase:meta.
    try (Table t = connection.getTable(tableName);
        RegionLocator locator = connection.getRegionLocator(tableName);
        Admin admin = connection.getAdmin()) {
        loader.doBulkLoad(dir, admin, t, locator);
    } catch (Exception e) {
        LOG.error("exeception=", e);
    }
    // check if all the data are loaded into the table.
    this.assertExpectedTable(tableName, ROWCOUNT, 2);
    dir = buildBulkFiles(tableName, 3);
    // Mess it up by leaving a hole in the hbase:meta
    List<HRegionInfo> regionInfos = MetaTableAccessor.getTableRegions(connection, tableName);
    for (HRegionInfo regionInfo : regionInfos) {
        if (Bytes.equals(regionInfo.getStartKey(), HConstants.EMPTY_BYTE_ARRAY)) {
            MetaTableAccessor.deleteRegion(connection, regionInfo);
            break;
        }
    }
    try (Table t = connection.getTable(tableName);
        RegionLocator locator = connection.getRegionLocator(tableName);
        Admin admin = connection.getAdmin()) {
        loader.doBulkLoad(dir, admin, t, locator);
    } catch (Exception e) {
        LOG.error("exception=", e);
        assertTrue("IOException expected", e instanceof IOException);
    }
    table.close();
    // Make sure at least the one region that still exists can be found.
    regionInfos = MetaTableAccessor.getTableRegions(connection, tableName);
    assertTrue(regionInfos.size() >= 1);
    this.assertExpectedTable(connection, tableName, ROWCOUNT, 2);
    connection.close();
}

Example 23 with RegionLocator

use of org.apache.hadoop.hbase.client.RegionLocator in project hbase by apache.

the class TestAccessController method testUnassign.

@Test(timeout = 180000)
public void testUnassign() throws Exception {
    List<HRegionLocation> regions;
    try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE)) {
        regions = locator.getAllRegionLocations();
    }
    HRegionLocation location = regions.get(0);
    final HRegionInfo hri = location.getRegionInfo();
    AccessTestAction action = new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            ACCESS_CONTROLLER.preUnassign(ObserverContext.createAndPrepare(CP_ENV, null), hri, false);
            return null;
        }
    };
    verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_GROUP_ADMIN);
    verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE);
}

Example 24 with RegionLocator

use of org.apache.hadoop.hbase.client.RegionLocator in project hbase by apache.

the class TestHRegionOnCluster method testDataCorrectnessReplayingRecoveredEdits.

@Test(timeout = 300000)
public void testDataCorrectnessReplayingRecoveredEdits() throws Exception {
    final int NUM_MASTERS = 1;
    final int NUM_RS = 3;
    Admin hbaseAdmin = null;
    TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
    try {
        final TableName tableName = TableName.valueOf(name.getMethodName());
        final byte[] FAMILY = Bytes.toBytes("family");
        MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
        HMaster master = cluster.getMaster();
        // Create table
        HTableDescriptor desc = new HTableDescriptor(tableName);
        desc.addFamily(new HColumnDescriptor(FAMILY));
        hbaseAdmin = master.getConnection().getAdmin();
        hbaseAdmin.createTable(desc);
        assertTrue(hbaseAdmin.isTableAvailable(tableName));
        // Put data: r1->v1
        LOG.info("Loading r1 to v1 into " + tableName);
        Table table = TEST_UTIL.getConnection().getTable(tableName);
        putDataAndVerify(table, "r1", FAMILY, "v1", 1);
        TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
        // Move region to target server
        HRegionInfo regionInfo;
        try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
            regionInfo = locator.getRegionLocation(Bytes.toBytes("r1")).getRegionInfo();
        }
        int originServerNum = cluster.getServerWith(regionInfo.getRegionName());
        HRegionServer originServer = cluster.getRegionServer(originServerNum);
        int targetServerNum = (originServerNum + 1) % NUM_RS;
        HRegionServer targetServer = cluster.getRegionServer(targetServerNum);
        assertFalse(originServer.equals(targetServer));
        TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
        LOG.info("Moving " + regionInfo.getEncodedName() + " to " + targetServer.getServerName());
        hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(), Bytes.toBytes(targetServer.getServerName().getServerName()));
        do {
            Thread.sleep(1);
        } while (cluster.getServerWith(regionInfo.getRegionName()) == originServerNum);
        // Put data: r2->v2
        LOG.info("Loading r2 to v2 into " + tableName);
        putDataAndVerify(table, "r2", FAMILY, "v2", 2);
        TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
        // Move region to origin server
        LOG.info("Moving " + regionInfo.getEncodedName() + " to " + originServer.getServerName());
        hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(), Bytes.toBytes(originServer.getServerName().getServerName()));
        do {
            Thread.sleep(1);
        } while (cluster.getServerWith(regionInfo.getRegionName()) == targetServerNum);
        // Put data: r3->v3
        LOG.info("Loading r3 to v3 into " + tableName);
        putDataAndVerify(table, "r3", FAMILY, "v3", 3);
        // Kill target server
        LOG.info("Killing target server " + targetServer.getServerName());
        targetServer.kill();
        cluster.getRegionServerThreads().get(targetServerNum).join();
        // Wait until finish processing of shutdown
        while (master.getServerManager().areDeadServersInProgress()) {
            Thread.sleep(5);
        }
        // Kill origin server
        LOG.info("Killing origin server " + targetServer.getServerName());
        originServer.kill();
        cluster.getRegionServerThreads().get(originServerNum).join();
        // Put data: r4->v4
        LOG.info("Loading r4 to v4 into " + tableName);
        putDataAndVerify(table, "r4", FAMILY, "v4", 4);
    } finally {
        if (hbaseAdmin != null)
            hbaseAdmin.close();
        TEST_UTIL.shutdownMiniCluster();
    }
}

Example 25 with RegionLocator

use of org.apache.hadoop.hbase.client.RegionLocator in project hbase by apache.

the class TestRemoveRegionMetrics method testMoveRegion.

@Test
public void testMoveRegion() throws IOException, InterruptedException {
    String tableNameString = name.getMethodName();
    TableName tableName = TableName.valueOf(tableNameString);
    Table t = TEST_UTIL.createTable(tableName, Bytes.toBytes("D"));
    TEST_UTIL.waitUntilAllRegionsAssigned(t.getName());
    Admin admin = TEST_UTIL.getAdmin();
    HRegionInfo regionInfo;
    byte[] row = Bytes.toBytes("r1");
    for (int i = 0; i < 30; i++) {
        boolean moved = false;
        try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
            regionInfo = locator.getRegionLocation(row, true).getRegionInfo();
        }
        int currentServerIdx = cluster.getServerWith(regionInfo.getRegionName());
        int destServerIdx = (currentServerIdx + 1) % cluster.getLiveRegionServerThreads().size();
        HRegionServer currentServer = cluster.getRegionServer(currentServerIdx);
        HRegionServer destServer = cluster.getRegionServer(destServerIdx);
        // Do a put. The counters should be non-zero now
        Put p = new Put(row);
        p.addColumn(Bytes.toBytes("D"), Bytes.toBytes("Zero"), Bytes.toBytes("VALUE"));
        t.put(p);
        MetricsRegionAggregateSource currentAgg = currentServer.getRegion(regionInfo.getRegionName()).getMetrics().getSource().getAggregateSource();
        String prefix = "namespace_" + NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR + "_table_" + tableNameString + "_region_" + regionInfo.getEncodedName() + "_metric";
        metricsHelper.assertCounter(prefix + "_mutateCount", 1, currentAgg);
        try {
            TEST_UTIL.moveRegionAndWait(regionInfo, destServer.getServerName());
            moved = true;
        } catch (IOException ioe) {
            moved = false;
        }
        if (moved) {
            MetricsRegionAggregateSource destAgg = destServer.getRegion(regionInfo.getRegionName()).getMetrics().getSource().getAggregateSource();
            metricsHelper.assertCounter(prefix + "_mutateCount", 0, destAgg);
        }
    }
    TEST_UTIL.deleteTable(tableName);
}