use of org.apache.hadoop.hbase.client.RegionLocator in project hbase by apache.
the class TestLoadIncrementalHFilesSplitRecovery method populateTable.
/**
* Populate table with known values.
*/
private void populateTable(final Connection connection, TableName table, int value) throws Exception {
// create HFiles for different column families
LoadIncrementalHFiles lih = new LoadIncrementalHFiles(util.getConfiguration());
Path bulk1 = buildBulkFiles(table, value);
try (Table t = connection.getTable(table);
RegionLocator locator = connection.getRegionLocator(table);
Admin admin = connection.getAdmin()) {
lih.doBulkLoad(bulk1, admin, t, locator);
}
}
use of org.apache.hadoop.hbase.client.RegionLocator in project hbase by apache.
the class TestLoadIncrementalHFilesSplitRecovery method testSplitTmpFileCleanUp.
/**
* This test creates a table with many small regions. The bulk load files
* would be splitted multiple times before all of them can be loaded successfully.
*/
@Test(timeout = 120000)
public void testSplitTmpFileCleanUp() throws Exception {
final TableName table = TableName.valueOf(name.getMethodName());
byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("row_00000010"), Bytes.toBytes("row_00000020"), Bytes.toBytes("row_00000030"), Bytes.toBytes("row_00000040"), Bytes.toBytes("row_00000050") };
try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) {
setupTableWithSplitkeys(table, 10, SPLIT_KEYS);
LoadIncrementalHFiles lih = new LoadIncrementalHFiles(util.getConfiguration());
// create HFiles
Path bulk = buildBulkFiles(table, 2);
try (Table t = connection.getTable(table);
RegionLocator locator = connection.getRegionLocator(table);
Admin admin = connection.getAdmin()) {
lih.doBulkLoad(bulk, admin, t, locator);
}
// family path
Path tmpPath = new Path(bulk, family(0));
// TMP_DIR under family path
tmpPath = new Path(tmpPath, LoadIncrementalHFiles.TMP_DIR);
FileSystem fs = bulk.getFileSystem(util.getConfiguration());
// HFiles have been splitted, there is TMP_DIR
assertTrue(fs.exists(tmpPath));
// TMP_DIR should have been cleaned-up
assertNull(LoadIncrementalHFiles.TMP_DIR + " should be empty.", FSUtils.listStatus(fs, tmpPath));
assertExpectedTable(connection, table, ROWCOUNT, 2);
}
}
use of org.apache.hadoop.hbase.client.RegionLocator in project hbase by apache.
the class TestLoadIncrementalHFilesSplitRecovery method testGroupOrSplitWhenRegionHoleExistsInMeta.
@Test(timeout = 120000)
public void testGroupOrSplitWhenRegionHoleExistsInMeta() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("row_00000100") };
// Share connection. We were failing to find the table with our new reverse scan because it
// looks for first region, not any region -- that is how it works now. The below removes first
// region in test. Was reliant on the Connection caching having first region.
Connection connection = ConnectionFactory.createConnection(util.getConfiguration());
Table table = connection.getTable(tableName);
setupTableWithSplitkeys(tableName, 10, SPLIT_KEYS);
Path dir = buildBulkFiles(tableName, 2);
final AtomicInteger countedLqis = new AtomicInteger();
LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration()) {
@Override
protected Pair<List<LoadQueueItem>, String> groupOrSplit(Multimap<ByteBuffer, LoadQueueItem> regionGroups, final LoadQueueItem item, final Table htable, final Pair<byte[][], byte[][]> startEndKeys) throws IOException {
Pair<List<LoadQueueItem>, String> lqis = super.groupOrSplit(regionGroups, item, htable, startEndKeys);
if (lqis != null && lqis.getFirst() != null) {
countedLqis.addAndGet(lqis.getFirst().size());
}
return lqis;
}
};
// do bulkload when there is no region hole in hbase:meta.
try (Table t = connection.getTable(tableName);
RegionLocator locator = connection.getRegionLocator(tableName);
Admin admin = connection.getAdmin()) {
loader.doBulkLoad(dir, admin, t, locator);
} catch (Exception e) {
LOG.error("exeception=", e);
}
// check if all the data are loaded into the table.
this.assertExpectedTable(tableName, ROWCOUNT, 2);
dir = buildBulkFiles(tableName, 3);
// Mess it up by leaving a hole in the hbase:meta
List<HRegionInfo> regionInfos = MetaTableAccessor.getTableRegions(connection, tableName);
for (HRegionInfo regionInfo : regionInfos) {
if (Bytes.equals(regionInfo.getStartKey(), HConstants.EMPTY_BYTE_ARRAY)) {
MetaTableAccessor.deleteRegion(connection, regionInfo);
break;
}
}
try (Table t = connection.getTable(tableName);
RegionLocator locator = connection.getRegionLocator(tableName);
Admin admin = connection.getAdmin()) {
loader.doBulkLoad(dir, admin, t, locator);
} catch (Exception e) {
LOG.error("exception=", e);
assertTrue("IOException expected", e instanceof IOException);
}
table.close();
// Make sure at least the one region that still exists can be found.
regionInfos = MetaTableAccessor.getTableRegions(connection, tableName);
assertTrue(regionInfos.size() >= 1);
this.assertExpectedTable(connection, tableName, ROWCOUNT, 2);
connection.close();
}
use of org.apache.hadoop.hbase.client.RegionLocator in project hbase by apache.
the class TestAccessController method testUnassign.
@Test(timeout = 180000)
public void testUnassign() throws Exception {
List<HRegionLocation> regions;
try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE)) {
regions = locator.getAllRegionLocations();
}
HRegionLocation location = regions.get(0);
final HRegionInfo hri = location.getRegionInfo();
AccessTestAction action = new AccessTestAction() {
@Override
public Object run() throws Exception {
ACCESS_CONTROLLER.preUnassign(ObserverContext.createAndPrepare(CP_ENV, null), hri, false);
return null;
}
};
verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_GROUP_ADMIN);
verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE);
}
use of org.apache.hadoop.hbase.client.RegionLocator in project hbase by apache.
the class TestHRegionOnCluster method testDataCorrectnessReplayingRecoveredEdits.
@Test(timeout = 300000)
public void testDataCorrectnessReplayingRecoveredEdits() throws Exception {
final int NUM_MASTERS = 1;
final int NUM_RS = 3;
Admin hbaseAdmin = null;
TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
try {
final TableName tableName = TableName.valueOf(name.getMethodName());
final byte[] FAMILY = Bytes.toBytes("family");
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
HMaster master = cluster.getMaster();
// Create table
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(new HColumnDescriptor(FAMILY));
hbaseAdmin = master.getConnection().getAdmin();
hbaseAdmin.createTable(desc);
assertTrue(hbaseAdmin.isTableAvailable(tableName));
// Put data: r1->v1
LOG.info("Loading r1 to v1 into " + tableName);
Table table = TEST_UTIL.getConnection().getTable(tableName);
putDataAndVerify(table, "r1", FAMILY, "v1", 1);
TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
// Move region to target server
HRegionInfo regionInfo;
try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
regionInfo = locator.getRegionLocation(Bytes.toBytes("r1")).getRegionInfo();
}
int originServerNum = cluster.getServerWith(regionInfo.getRegionName());
HRegionServer originServer = cluster.getRegionServer(originServerNum);
int targetServerNum = (originServerNum + 1) % NUM_RS;
HRegionServer targetServer = cluster.getRegionServer(targetServerNum);
assertFalse(originServer.equals(targetServer));
TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
LOG.info("Moving " + regionInfo.getEncodedName() + " to " + targetServer.getServerName());
hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(), Bytes.toBytes(targetServer.getServerName().getServerName()));
do {
Thread.sleep(1);
} while (cluster.getServerWith(regionInfo.getRegionName()) == originServerNum);
// Put data: r2->v2
LOG.info("Loading r2 to v2 into " + tableName);
putDataAndVerify(table, "r2", FAMILY, "v2", 2);
TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
// Move region to origin server
LOG.info("Moving " + regionInfo.getEncodedName() + " to " + originServer.getServerName());
hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(), Bytes.toBytes(originServer.getServerName().getServerName()));
do {
Thread.sleep(1);
} while (cluster.getServerWith(regionInfo.getRegionName()) == targetServerNum);
// Put data: r3->v3
LOG.info("Loading r3 to v3 into " + tableName);
putDataAndVerify(table, "r3", FAMILY, "v3", 3);
// Kill target server
LOG.info("Killing target server " + targetServer.getServerName());
targetServer.kill();
cluster.getRegionServerThreads().get(targetServerNum).join();
// Wait until finish processing of shutdown
while (master.getServerManager().areDeadServersInProgress()) {
Thread.sleep(5);
}
// Kill origin server
LOG.info("Killing origin server " + targetServer.getServerName());
originServer.kill();
cluster.getRegionServerThreads().get(originServerNum).join();
// Put data: r4->v4
LOG.info("Loading r4 to v4 into " + tableName);
putDataAndVerify(table, "r4", FAMILY, "v4", 4);
} finally {
if (hbaseAdmin != null)
hbaseAdmin.close();
TEST_UTIL.shutdownMiniCluster();
}
}
Aggregations