Examples with WALKeyImpl org.apache.hadoop.hbase.wal.WALKeyImpl used on opensource projects

Example 31 with WALKeyImpl

use of org.apache.hadoop.hbase.wal.WALKeyImpl in project hbase by apache.

the class TestWALObserver method testWALCoprocessorReplay.

/**
 * Test WAL replay behavior with WALObserver.
 */
@Test
public void testWALCoprocessorReplay() throws Exception {
    // WAL replay is handled at HRegion::replayRecoveredEdits(), which is
    // ultimately called by HRegion::initialize()
    TableName tableName = TableName.valueOf(currentTest.getMethodName());
    TableDescriptor htd = getBasic3FamilyHTableDescriptor(tableName);
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    // final HRegionInfo hri =
    // createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
    // final HRegionInfo hri1 =
    // createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
    RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build();
    final Path basedir = CommonFSUtils.getTableDir(this.hbaseRootDir, tableName);
    deleteDir(basedir);
    fs.mkdirs(new Path(basedir, hri.getEncodedName()));
    final Configuration newConf = HBaseConfiguration.create(this.conf);
    // WAL wal = new WAL(this.fs, this.dir, this.oldLogDir, this.conf);
    WAL wal = wals.getWAL(null);
    // Put p = creatPutWith2Families(TEST_ROW);
    WALEdit edit = new WALEdit();
    long now = EnvironmentEdgeManager.currentTime();
    final int countPerFamily = 1000;
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (byte[] fam : htd.getColumnFamilyNames()) {
        scopes.put(fam, 0);
    }
    for (byte[] fam : htd.getColumnFamilyNames()) {
        addWALEdits(tableName, hri, TEST_ROW, fam, countPerFamily, EnvironmentEdgeManager.getDelegate(), wal, scopes, mvcc);
    }
    wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes), edit);
    // sync to fs.
    wal.sync();
    User user = HBaseTestingUtil.getDifferentUser(newConf, ".replay.wal.secondtime");
    user.runAs(new PrivilegedExceptionAction<Void>() {

        @Override
        public Void run() throws Exception {
            Path p = runWALSplit(newConf);
            LOG.info("WALSplit path == " + p);
            // Make a new wal for new region open.
            final WALFactory wals2 = new WALFactory(conf, ServerName.valueOf(currentTest.getMethodName() + "2", 16010, EnvironmentEdgeManager.currentTime()).toString());
            WAL wal2 = wals2.getWAL(null);
            HRegion region = HRegion.openHRegion(newConf, FileSystem.get(newConf), hbaseRootDir, hri, htd, wal2, TEST_UTIL.getHBaseCluster().getRegionServer(0), null);
            SampleRegionWALCoprocessor cp2 = region.getCoprocessorHost().findCoprocessor(SampleRegionWALCoprocessor.class);
            // TODO: asserting here is problematic.
            assertNotNull(cp2);
            assertTrue(cp2.isPreWALRestoreCalled());
            assertTrue(cp2.isPostWALRestoreCalled());
            region.close();
            wals2.close();
            return null;
        }
    });
}

Example 32 with WALKeyImpl

use of org.apache.hadoop.hbase.wal.WALKeyImpl in project hbase by apache.

the class TestCompactionAfterBulkLoad method testAvoidRepeatedlyRequestCompactAfterBulkLoad.

@Test
public void testAvoidRepeatedlyRequestCompactAfterBulkLoad() throws IOException {
    final CompactSplit compactSplit = new TestFamily1UnderCompact(HBaseConfiguration.create());
    called.set(0);
    List<Pair<byte[], String>> familyPaths = new ArrayList<>();
    // enough hfile to request compaction
    for (int i = 0; i < 5; i++) {
        familyPaths.addAll(withFamilyPathsFor(family1, family2, family3));
    }
    try {
        conf.setBoolean(COMPACTION_AFTER_BULKLOAD_ENABLE, true);
        when(regionServerServices.getConfiguration()).thenReturn(conf);
        when(regionServerServices.getCompactionRequestor()).thenReturn(compactSplit);
        when(log.appendMarker(any(), any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)))).thenAnswer(new Answer() {

            @Override
            public Object answer(InvocationOnMock invocation) {
                WALKeyImpl walKey = invocation.getArgument(1);
                MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
                if (mvcc != null) {
                    MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
                    walKey.setWriteEntry(we);
                }
                return 01L;
            }
        });
        HRegion region = testRegionWithFamilies(family1, family2, family3);
        region.bulkLoadHFiles(familyPaths, false, null);
        // invoke three times for 2 families
        assertEquals(2, called.get());
    } finally {
        conf.setBoolean(COMPACTION_AFTER_BULKLOAD_ENABLE, false);
    }
}

Example 33 with WALKeyImpl

use of org.apache.hadoop.hbase.wal.WALKeyImpl in project hbase by apache.

the class TestCompactionAfterBulkLoad method shouldRequestCompactAllStoresAfterBulkLoad.

@Test
public void shouldRequestCompactAllStoresAfterBulkLoad() throws IOException {
    final CompactSplit compactSplit = new TestCompactSplit(HBaseConfiguration.create());
    called.set(0);
    List<Pair<byte[], String>> familyPaths = new ArrayList<>();
    // enough hfile to request compaction
    for (int i = 0; i < 5; i++) {
        familyPaths.addAll(withFamilyPathsFor(family1, family2, family3));
    }
    try {
        conf.setBoolean(COMPACTION_AFTER_BULKLOAD_ENABLE, true);
        when(regionServerServices.getConfiguration()).thenReturn(conf);
        when(regionServerServices.getCompactionRequestor()).thenReturn(compactSplit);
        when(log.appendMarker(any(), any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)))).thenAnswer(new Answer() {

            @Override
            public Object answer(InvocationOnMock invocation) {
                WALKeyImpl walKey = invocation.getArgument(1);
                MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
                if (mvcc != null) {
                    MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
                    walKey.setWriteEntry(we);
                }
                return 01L;
            }
        });
        HRegion region = testRegionWithFamilies(family1, family2, family3);
        region.bulkLoadHFiles(familyPaths, false, null);
        assertEquals(3, called.get());
    } finally {
        conf.setBoolean(COMPACTION_AFTER_BULKLOAD_ENABLE, false);
    }
}

Example 34 with WALKeyImpl

use of org.apache.hadoop.hbase.wal.WALKeyImpl in project hbase by apache.

the class TestBulkLoad method shouldBulkLoadSingleFamilyHLog.

@Test
public void shouldBulkLoadSingleFamilyHLog() throws IOException {
    when(log.appendMarker(any(), any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)))).thenAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) {
            WALKeyImpl walKey = invocation.getArgument(1);
            MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
            if (mvcc != null) {
                MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
                walKey.setWriteEntry(we);
            }
            return 01L;
        }
    });
    testRegionWithFamilies(family1).bulkLoadHFiles(withFamilyPathsFor(family1), false, null);
    verify(log).sync(anyLong());
}

Example 35 with WALKeyImpl

use of org.apache.hadoop.hbase.wal.WALKeyImpl in project hbase by apache.

the class TestBulkLoad method shouldBulkLoadManyFamilyHLogEvenWhenTableNameNamespaceSpecified.

@Test
public void shouldBulkLoadManyFamilyHLogEvenWhenTableNameNamespaceSpecified() throws IOException {
    when(log.appendMarker(any(), any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)))).thenAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) {
            WALKeyImpl walKey = invocation.getArgument(1);
            MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
            if (mvcc != null) {
                MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
                walKey.setWriteEntry(we);
            }
            return 01L;
        }
    });
    TableName tableName = TableName.valueOf("test", "test");
    testRegionWithFamiliesAndSpecifiedTableName(tableName, family1, family2).bulkLoadHFiles(withFamilyPathsFor(family1, family2), false, null);
    verify(log).sync(anyLong());
}