Examples with HRegion org.apache.hadoop.hbase.regionserver.HRegion used on opensource projects

Example 21 with HRegion

use of org.apache.hadoop.hbase.regionserver.HRegion in project hbase by apache.

the class AbstractTestWALReplay method testReplayEditsAfterPartialFlush.

/**
   * Test that we recover correctly when there is a failure in between the
   * flushes. i.e. Some stores got flushed but others did not.
   *
   * Unfortunately, there is no easy hook to flush at a store level. The way
   * we get around this is by flushing at the region level, and then deleting
   * the recently flushed store file for one of the Stores. This would put us
   * back in the situation where all but that store got flushed and the region
   * died.
   *
   * We restart Region again, and verify that the edits were replayed.
   *
   * @throws IOException
   * @throws IllegalAccessException
   * @throws NoSuchFieldException
   * @throws IllegalArgumentException
   * @throws SecurityException
   */
@Test
public void testReplayEditsAfterPartialFlush() throws IOException, SecurityException, IllegalArgumentException, NoSuchFieldException, IllegalAccessException, InterruptedException {
    final TableName tableName = TableName.valueOf("testReplayEditsWrittenViaHRegion");
    final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
    final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
    deleteDir(basedir);
    final byte[] rowName = tableName.getName();
    final int countPerFamily = 10;
    final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
    HRegion region3 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
    HBaseTestingUtility.closeRegionAndWAL(region3);
    // Write countPerFamily edits into the three families.  Do a flush on one
    // of the families during the load of edits so its seqid is not same as
    // others to test we do right thing when different seqids.
    WAL wal = createWAL(this.conf, hbaseRootDir, logName);
    HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
    long seqid = region.getOpenSeqNum();
    for (HColumnDescriptor hcd : htd.getFamilies()) {
        addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
    }
    // Now assert edits made it in.
    final Get g = new Get(rowName);
    Result result = region.get(g);
    assertEquals(countPerFamily * htd.getFamilies().size(), result.size());
    // Let us flush the region
    region.flush(true);
    region.close(true);
    wal.shutdown();
    // delete the store files in the second column family to simulate a failure
    // in between the flushcache();
    // we have 3 families. killing the middle one ensures that taking the maximum
    // will make us fail.
    int cf_count = 0;
    for (HColumnDescriptor hcd : htd.getFamilies()) {
        cf_count++;
        if (cf_count == 2) {
            region.getRegionFileSystem().deleteFamily(hcd.getNameAsString());
        }
    }
    // Let us try to split and recover
    runWALSplit(this.conf);
    WAL wal2 = createWAL(this.conf, hbaseRootDir, logName);
    HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2);
    long seqid2 = region2.getOpenSeqNum();
    assertTrue(seqid + result.size() < seqid2);
    final Result result1b = region2.get(g);
    assertEquals(result.size(), result1b.size());
}

Example 22 with HRegion

use of org.apache.hadoop.hbase.regionserver.HRegion in project hbase by apache.

the class AbstractTestWALReplay method test2727.

/**
   * Tests for hbase-2727.
   * @throws Exception
   * @see <a href="https://issues.apache.org/jira/browse/HBASE-2727">HBASE-2727</a>
   */
@Test
public void test2727() throws Exception {
    // Test being able to have > 1 set of edits in the recovered.edits directory.
    // Ensure edits are replayed properly.
    final TableName tableName = TableName.valueOf("test2727");
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
    Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
    deleteDir(basedir);
    HTableDescriptor htd = createBasic3FamilyHTD(tableName);
    Region region2 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
    HBaseTestingUtility.closeRegionAndWAL(region2);
    final byte[] rowName = tableName.getName();
    WAL wal1 = createWAL(this.conf, hbaseRootDir, logName);
    // Add 1k to each family.
    final int countPerFamily = 1000;
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (byte[] fam : htd.getFamiliesKeys()) {
        scopes.put(fam, 0);
    }
    for (HColumnDescriptor hcd : htd.getFamilies()) {
        addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee, wal1, htd, mvcc, scopes);
    }
    wal1.shutdown();
    runWALSplit(this.conf);
    WAL wal2 = createWAL(this.conf, hbaseRootDir, logName);
    // Add 1k to each family.
    for (HColumnDescriptor hcd : htd.getFamilies()) {
        addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee, wal2, htd, mvcc, scopes);
    }
    wal2.shutdown();
    runWALSplit(this.conf);
    WAL wal3 = createWAL(this.conf, hbaseRootDir, logName);
    try {
        HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3);
        long seqid = region.getOpenSeqNum();
        // The regions opens with sequenceId as 1. With 6k edits, its sequence number reaches 6k + 1.
        // When opened, this region would apply 6k edits, and increment the sequenceId by 1
        assertTrue(seqid > mvcc.getWritePoint());
        assertEquals(seqid - 1, mvcc.getWritePoint());
        LOG.debug("region.getOpenSeqNum(): " + region.getOpenSeqNum() + ", wal3.id: " + mvcc.getReadPoint());
        // TODO: Scan all.
        region.close();
    } finally {
        wal3.close();
    }
}

Example 23 with HRegion

use of org.apache.hadoop.hbase.regionserver.HRegion in project hbase by apache.

the class AbstractTestWALReplay method testReplayEditsAfterAbortingFlush.

/**
   * Test that we could recover the data correctly after aborting flush. In the
   * test, first we abort flush after writing some data, then writing more data
   * and flush again, at last verify the data.
   * @throws IOException
   */
@Test
public void testReplayEditsAfterAbortingFlush() throws IOException {
    final TableName tableName = TableName.valueOf("testReplayEditsAfterAbortingFlush");
    final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
    final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
    deleteDir(basedir);
    final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
    HRegion region3 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
    HBaseTestingUtility.closeRegionAndWAL(region3);
    // Write countPerFamily edits into the three families. Do a flush on one
    // of the families during the load of edits so its seqid is not same as
    // others to test we do right thing when different seqids.
    WAL wal = createWAL(this.conf, hbaseRootDir, logName);
    RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
    Mockito.doReturn(false).when(rsServices).isAborted();
    when(rsServices.getServerName()).thenReturn(ServerName.valueOf("foo", 10, 10));
    Configuration customConf = new Configuration(this.conf);
    customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY, CustomStoreFlusher.class.getName());
    HRegion region = HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal, customConf, rsServices, null);
    int writtenRowCount = 10;
    List<HColumnDescriptor> families = new ArrayList<>(htd.getFamilies());
    for (int i = 0; i < writtenRowCount; i++) {
        Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
        put.addColumn(families.get(i % families.size()).getName(), Bytes.toBytes("q"), Bytes.toBytes("val"));
        region.put(put);
    }
    // Now assert edits made it in.
    RegionScanner scanner = region.getScanner(new Scan());
    assertEquals(writtenRowCount, getScannedCount(scanner));
    // Let us flush the region
    CustomStoreFlusher.throwExceptionWhenFlushing.set(true);
    try {
        region.flush(true);
        fail("Injected exception hasn't been thrown");
    } catch (Throwable t) {
        LOG.info("Expected simulated exception when flushing region," + t.getMessage());
        // simulated to abort server
        Mockito.doReturn(true).when(rsServices).isAborted();
        // region normally does not accept writes after
        region.setClosing(false);
    // DroppedSnapshotException. We mock around it for this test.
    }
    // writing more data
    int moreRow = 10;
    for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
        Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
        put.addColumn(families.get(i % families.size()).getName(), Bytes.toBytes("q"), Bytes.toBytes("val"));
        region.put(put);
    }
    writtenRowCount += moreRow;
    // call flush again
    CustomStoreFlusher.throwExceptionWhenFlushing.set(false);
    try {
        region.flush(true);
    } catch (IOException t) {
        LOG.info("Expected exception when flushing region because server is stopped," + t.getMessage());
    }
    region.close(true);
    wal.shutdown();
    // Let us try to split and recover
    runWALSplit(this.conf);
    WAL wal2 = createWAL(this.conf, hbaseRootDir, logName);
    Mockito.doReturn(false).when(rsServices).isAborted();
    HRegion region2 = HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal2, this.conf, rsServices, null);
    scanner = region2.getScanner(new Scan());
    assertEquals(writtenRowCount, getScannedCount(scanner));
}

Example 24 with HRegion

use of org.apache.hadoop.hbase.regionserver.HRegion in project hbase by apache.

the class TestReplicationEndpoint method testInterClusterReplication.

@Test(timeout = 120000)
public void testInterClusterReplication() throws Exception {
    final String id = "testInterClusterReplication";
    List<HRegion> regions = utility1.getHBaseCluster().getRegions(tableName);
    int totEdits = 0;
    // before shipping edits.
    for (HRegion region : regions) {
        HRegionInfo hri = region.getRegionInfo();
        byte[] row = hri.getStartKey();
        for (int i = 0; i < 100; i++) {
            if (row.length > 0) {
                Put put = new Put(row);
                put.addColumn(famName, row, row);
                region.put(put);
                totEdits++;
            }
        }
    }
    admin.addPeer(id, new ReplicationPeerConfig().setClusterKey(ZKConfig.getZooKeeperClusterKey(conf2)).setReplicationEndpointImpl(InterClusterReplicationEndpointForTest.class.getName()), null);
    final int numEdits = totEdits;
    Waiter.waitFor(conf1, 30000, new Waiter.ExplainingPredicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            return InterClusterReplicationEndpointForTest.replicateCount.get() == numEdits;
        }

        @Override
        public String explainFailure() throws Exception {
            String failure = "Failed to replicate all edits, expected = " + numEdits + " replicated = " + InterClusterReplicationEndpointForTest.replicateCount.get();
            return failure;
        }
    });
    admin.removePeer("testInterClusterReplication");
    utility1.deleteTableData(tableName);
}

Example 25 with HRegion

use of org.apache.hadoop.hbase.regionserver.HRegion in project hbase by apache.

the class TestMasterReplication method rollWALAndWait.

private void rollWALAndWait(final HBaseTestingUtility utility, final TableName table, final byte[] row) throws IOException {
    final Admin admin = utility.getAdmin();
    final MiniHBaseCluster cluster = utility.getMiniHBaseCluster();
    // find the region that corresponds to the given row.
    HRegion region = null;
    for (HRegion candidate : cluster.getRegions(table)) {
        if (HRegion.rowIsInRange(candidate.getRegionInfo(), row)) {
            region = candidate;
            break;
        }
    }
    assertNotNull("Couldn't find the region for row '" + Arrays.toString(row) + "'", region);
    final CountDownLatch latch = new CountDownLatch(1);
    // listen for successful log rolls
    final WALActionsListener listener = new WALActionsListener.Base() {

        @Override
        public void postLogRoll(final Path oldPath, final Path newPath) throws IOException {
            latch.countDown();
        }
    };
    region.getWAL().registerWALActionsListener(listener);
    // request a roll
    admin.rollWALWriter(cluster.getServerHoldingRegion(region.getTableDesc().getTableName(), region.getRegionInfo().getRegionName()));
    // wait
    try {
        latch.await();
    } catch (InterruptedException exception) {
        LOG.warn("Interrupted while waiting for the wal of '" + region + "' to roll. If later " + "replication tests fail, it's probably because we should still be waiting.");
        Thread.currentThread().interrupt();
    }
    region.getWAL().unregisterWALActionsListener(listener);
}