Examples with WALKey org.apache.hadoop.hbase.wal.WALKey used on opensource projects

Example 41 with WALKey

use of org.apache.hadoop.hbase.wal.WALKey in project hbase by apache.

the class AbstractTestFSWAL method testFlushSequenceIdIsGreaterThanAllEditsInHFile.

/**
   * Test flush for sure has a sequence id that is beyond the last edit appended. We do this by
   * slowing appends in the background ring buffer thread while in foreground we call flush. The
   * addition of the sync over HRegion in flush should fix an issue where flush was returning before
   * all of its appends had made it out to the WAL (HBASE-11109).
   * @throws IOException
   * @see <a href="https://issues.apache.org/jira/browse/HBASE-11109">HBASE-11109</a>
   */
@Test
public void testFlushSequenceIdIsGreaterThanAllEditsInHFile() throws IOException {
    String testName = currentTest.getMethodName();
    final TableName tableName = TableName.valueOf(testName);
    final HRegionInfo hri = new HRegionInfo(tableName);
    final byte[] rowName = tableName.getName();
    final HTableDescriptor htd = new HTableDescriptor(tableName);
    htd.addFamily(new HColumnDescriptor("f"));
    HRegion r = HBaseTestingUtility.createRegionAndWAL(hri, TEST_UTIL.getDefaultRootDirPath(), TEST_UTIL.getConfiguration(), htd);
    HBaseTestingUtility.closeRegionAndWAL(r);
    final int countPerFamily = 10;
    final AtomicBoolean goslow = new AtomicBoolean(false);
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (byte[] fam : htd.getFamiliesKeys()) {
        scopes.put(fam, 0);
    }
    // subclass and doctor a method.
    AbstractFSWAL<?> wal = newSlowWAL(FS, FSUtils.getWALRootDir(CONF), DIR.toString(), testName, CONF, null, true, null, null, new Runnable() {

        @Override
        public void run() {
            if (goslow.get()) {
                Threads.sleep(100);
                LOG.debug("Sleeping before appending 100ms");
            }
        }
    });
    HRegion region = HRegion.openHRegion(TEST_UTIL.getConfiguration(), TEST_UTIL.getTestFileSystem(), TEST_UTIL.getDefaultRootDirPath(), hri, htd, wal);
    EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
    try {
        List<Put> puts = null;
        for (HColumnDescriptor hcd : htd.getFamilies()) {
            puts = TestWALReplay.addRegionEdits(rowName, hcd.getName(), countPerFamily, ee, region, "x");
        }
        // Now assert edits made it in.
        final Get g = new Get(rowName);
        Result result = region.get(g);
        assertEquals(countPerFamily * htd.getFamilies().size(), result.size());
        // Construct a WALEdit and add it a few times to the WAL.
        WALEdit edits = new WALEdit();
        for (Put p : puts) {
            CellScanner cs = p.cellScanner();
            while (cs.advance()) {
                edits.add(cs.current());
            }
        }
        // Add any old cluster id.
        List<UUID> clusterIds = new ArrayList<>(1);
        clusterIds.add(UUID.randomUUID());
        // Now make appends run slow.
        goslow.set(true);
        for (int i = 0; i < countPerFamily; i++) {
            final HRegionInfo info = region.getRegionInfo();
            final WALKey logkey = new WALKey(info.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), clusterIds, -1, -1, region.getMVCC(), scopes);
            wal.append(info, logkey, edits, true);
            region.getMVCC().completeAndWait(logkey.getWriteEntry());
        }
        region.flush(true);
        // FlushResult.flushSequenceId is not visible here so go get the current sequence id.
        long currentSequenceId = region.getReadPoint(null);
        // Now release the appends
        goslow.set(false);
        assertTrue(currentSequenceId >= region.getReadPoint(null));
    } finally {
        region.close(true);
        wal.close();
    }
}

Example 42 with WALKey

use of org.apache.hadoop.hbase.wal.WALKey in project hbase by apache.

the class TestFSHLog method testUnflushedSeqIdTracking.

/**
   * Test case for https://issues.apache.org/jira/browse/HBASE-16721
   */
@Test(timeout = 30000)
public void testUnflushedSeqIdTracking() throws IOException, InterruptedException {
    final String name = this.name.getMethodName();
    final byte[] b = Bytes.toBytes("b");
    final AtomicBoolean startHoldingForAppend = new AtomicBoolean(false);
    final CountDownLatch holdAppend = new CountDownLatch(1);
    final CountDownLatch flushFinished = new CountDownLatch(1);
    final CountDownLatch putFinished = new CountDownLatch(1);
    try (FSHLog log = new FSHLog(FS, FSUtils.getRootDir(CONF), name, HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null)) {
        log.registerWALActionsListener(new WALActionsListener.Base() {

            @Override
            public void visitLogEntryBeforeWrite(WALKey logKey, WALEdit logEdit) throws IOException {
                if (startHoldingForAppend.get()) {
                    try {
                        holdAppend.await();
                    } catch (InterruptedException e) {
                        LOG.error(e);
                    }
                }
            }
        });
        // open a new region which uses this WAL
        HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(this.name.getMethodName())).addFamily(new HColumnDescriptor(b));
        HRegionInfo hri = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
        final HRegion region = TEST_UTIL.createLocalHRegion(hri, htd, log);
        ExecutorService exec = Executors.newFixedThreadPool(2);
        // do a regular write first because of memstore size calculation.
        region.put(new Put(b).addColumn(b, b, b));
        startHoldingForAppend.set(true);
        exec.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    region.put(new Put(b).addColumn(b, b, b));
                    putFinished.countDown();
                } catch (IOException e) {
                    LOG.error(e);
                }
            }
        });
        // give the put a chance to start
        Threads.sleep(3000);
        exec.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    Region.FlushResult flushResult = region.flush(true);
                    LOG.info("Flush result:" + flushResult.getResult());
                    LOG.info("Flush succeeded:" + flushResult.isFlushSucceeded());
                    flushFinished.countDown();
                } catch (IOException e) {
                    LOG.error(e);
                }
            }
        });
        // give the flush a chance to start. Flush should have got the region lock, and
        // should have been waiting on the mvcc complete after this.
        Threads.sleep(3000);
        // let the append to WAL go through now that the flush already started
        holdAppend.countDown();
        putFinished.await();
        flushFinished.await();
        // check whether flush went through
        assertEquals("Region did not flush?", 1, region.getStoreFileList(new byte[][] { b }).size());
        // now check the region's unflushed seqIds.
        long seqId = log.getEarliestMemstoreSeqNum(hri.getEncodedNameAsBytes());
        assertEquals("Found seqId for the region which is already flushed", HConstants.NO_SEQNUM, seqId);
        region.close();
    }
}

Example 43 with WALKey

use of org.apache.hadoop.hbase.wal.WALKey in project hbase by apache.

the class TestBulkLoad method shouldBulkLoadManyFamilyHLogEvenWhenTableNameNamespaceSpecified.

@Test
public void shouldBulkLoadManyFamilyHLogEvenWhenTableNameNamespaceSpecified() throws IOException {
    when(log.append(any(HRegionInfo.class), any(WALKey.class), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)), any(boolean.class))).thenAnswer(new Answer() {

        public Object answer(InvocationOnMock invocation) {
            WALKey walKey = invocation.getArgumentAt(1, WALKey.class);
            MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
            if (mvcc != null) {
                MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
                walKey.setWriteEntry(we);
            }
            return 01L;
        }

        ;
    });
    TableName tableName = TableName.valueOf("test", "test");
    testRegionWithFamiliesAndSpecifiedTableName(tableName, family1, family2).bulkLoadHFiles(withFamilyPathsFor(family1, family2), false, null);
    verify(log).sync(anyLong());
}

Example 44 with WALKey

use of org.apache.hadoop.hbase.wal.WALKey in project hbase by apache.

the class TestBulkLoad method verifyBulkLoadEvent.

@Test
public void verifyBulkLoadEvent() throws IOException {
    TableName tableName = TableName.valueOf("test", "test");
    List<Pair<byte[], String>> familyPaths = withFamilyPathsFor(family1);
    byte[] familyName = familyPaths.get(0).getFirst();
    String storeFileName = familyPaths.get(0).getSecond();
    storeFileName = (new Path(storeFileName)).getName();
    List<String> storeFileNames = new ArrayList<>();
    storeFileNames.add(storeFileName);
    when(log.append(any(HRegionInfo.class), any(WALKey.class), argThat(bulkLogWalEdit(WALEdit.BULK_LOAD, tableName.toBytes(), familyName, storeFileNames)), any(boolean.class))).thenAnswer(new Answer() {

        public Object answer(InvocationOnMock invocation) {
            WALKey walKey = invocation.getArgumentAt(1, WALKey.class);
            MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
            if (mvcc != null) {
                MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
                walKey.setWriteEntry(we);
            }
            return 01L;
        }

        ;
    });
    testRegionWithFamiliesAndSpecifiedTableName(tableName, family1).bulkLoadHFiles(familyPaths, false, null);
    verify(log).sync(anyLong());
}

Example 45 with WALKey

use of org.apache.hadoop.hbase.wal.WALKey in project hbase by apache.

the class TestBulkLoad method shouldBulkLoadSingleFamilyHLog.

@Test
public void shouldBulkLoadSingleFamilyHLog() throws IOException {
    when(log.append(any(HRegionInfo.class), any(WALKey.class), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)), any(boolean.class))).thenAnswer(new Answer() {

        public Object answer(InvocationOnMock invocation) {
            WALKey walKey = invocation.getArgumentAt(1, WALKey.class);
            MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
            if (mvcc != null) {
                MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
                walKey.setWriteEntry(we);
            }
            return 01L;
        }

        ;
    });
    testRegionWithFamilies(family1).bulkLoadHFiles(withFamilyPathsFor(family1), false, null);
    verify(log).sync(anyLong());
}