Examples with WALEdit org.apache.hadoop.hbase.wal.WALEdit used on opensource projects

Example 56 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class AbstractTestFSWAL method testFlushSequenceIdIsGreaterThanAllEditsInHFile.

/**
 * Test flush for sure has a sequence id that is beyond the last edit appended. We do this by
 * slowing appends in the background ring buffer thread while in foreground we call flush. The
 * addition of the sync over HRegion in flush should fix an issue where flush was returning before
 * all of its appends had made it out to the WAL (HBASE-11109).
 * @throws IOException
 * @see <a href="https://issues.apache.org/jira/browse/HBASE-11109">HBASE-11109</a>
 */
@Test
public void testFlushSequenceIdIsGreaterThanAllEditsInHFile() throws IOException {
    String testName = currentTest.getMethodName();
    final TableName tableName = TableName.valueOf(testName);
    final RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build();
    final byte[] rowName = tableName.getName();
    final TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
    HRegion r = HBaseTestingUtil.createRegionAndWAL(hri, TEST_UTIL.getDefaultRootDirPath(), TEST_UTIL.getConfiguration(), htd);
    HBaseTestingUtil.closeRegionAndWAL(r);
    final int countPerFamily = 10;
    final AtomicBoolean goslow = new AtomicBoolean(false);
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (byte[] fam : htd.getColumnFamilyNames()) {
        scopes.put(fam, 0);
    }
    // subclass and doctor a method.
    AbstractFSWAL<?> wal = newSlowWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(), testName, CONF, null, true, null, null, new Runnable() {

        @Override
        public void run() {
            if (goslow.get()) {
                Threads.sleep(100);
                LOG.debug("Sleeping before appending 100ms");
            }
        }
    });
    HRegion region = HRegion.openHRegion(TEST_UTIL.getConfiguration(), TEST_UTIL.getTestFileSystem(), TEST_UTIL.getDefaultRootDirPath(), hri, htd, wal);
    EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
    try {
        List<Put> puts = null;
        for (byte[] fam : htd.getColumnFamilyNames()) {
            puts = TestWALReplay.addRegionEdits(rowName, fam, countPerFamily, ee, region, "x");
        }
        // Now assert edits made it in.
        final Get g = new Get(rowName);
        Result result = region.get(g);
        assertEquals(countPerFamily * htd.getColumnFamilyNames().size(), result.size());
        // Construct a WALEdit and add it a few times to the WAL.
        WALEdit edits = new WALEdit();
        for (Put p : puts) {
            CellScanner cs = p.cellScanner();
            while (cs.advance()) {
                edits.add(cs.current());
            }
        }
        // Add any old cluster id.
        List<UUID> clusterIds = new ArrayList<>(1);
        clusterIds.add(TEST_UTIL.getRandomUUID());
        // Now make appends run slow.
        goslow.set(true);
        for (int i = 0; i < countPerFamily; i++) {
            final RegionInfo info = region.getRegionInfo();
            final WALKeyImpl logkey = new WALKeyImpl(info.getEncodedNameAsBytes(), tableName, EnvironmentEdgeManager.currentTime(), clusterIds, -1, -1, region.getMVCC(), scopes);
            wal.append(info, logkey, edits, true);
            region.getMVCC().completeAndWait(logkey.getWriteEntry());
        }
        region.flush(true);
        // FlushResult.flushSequenceId is not visible here so go get the current sequence id.
        long currentSequenceId = region.getReadPoint(null);
        // Now release the appends
        goslow.set(false);
        assertTrue(currentSequenceId >= region.getReadPoint(null));
    } finally {
        region.close(true);
        wal.close();
    }
}

Example 57 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class AbstractTestFSWAL method testWriteEntryCanBeNull.

@Test
public void testWriteEntryCanBeNull() throws IOException {
    String testName = currentTest.getMethodName();
    AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(), testName, CONF, null, true, null, null);
    wal.close();
    TableDescriptor td = TableDescriptorBuilder.newBuilder(TableName.valueOf("table")).setColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();
    RegionInfo ri = RegionInfoBuilder.newBuilder(td.getTableName()).build();
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (byte[] fam : td.getColumnFamilyNames()) {
        scopes.put(fam, 0);
    }
    long timestamp = EnvironmentEdgeManager.currentTime();
    byte[] row = Bytes.toBytes("row");
    WALEdit cols = new WALEdit();
    cols.add(new KeyValue(row, row, row, timestamp, row));
    WALKeyImpl key = new WALKeyImpl(ri.getEncodedNameAsBytes(), td.getTableName(), SequenceId.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, scopes);
    try {
        wal.append(ri, key, cols, true);
        fail("Should fail since the wal has already been closed");
    } catch (IOException e) {
        // expected
        assertThat(e.getMessage(), containsString("log is closed"));
        // the WriteEntry should be null since we fail before setting it.
        assertNull(key.getWriteEntry());
    }
}

Example 58 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class AbstractTestFSWAL method addEdits.

protected void addEdits(WAL log, RegionInfo hri, TableDescriptor htd, int times, MultiVersionConcurrencyControl mvcc, NavigableMap<byte[], Integer> scopes, String cf) throws IOException {
    final byte[] row = Bytes.toBytes(cf);
    for (int i = 0; i < times; i++) {
        long timestamp = EnvironmentEdgeManager.currentTime();
        WALEdit cols = new WALEdit();
        cols.add(new KeyValue(row, row, row, timestamp, row));
        WALKeyImpl key = new WALKeyImpl(hri.getEncodedNameAsBytes(), htd.getTableName(), SequenceId.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, scopes);
        log.appendData(hri, key, cols);
    }
    log.sync();
}

Example 59 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class TestAsyncFSWALRollStuck method testRoll.

@Test
public void testRoll() throws Exception {
    byte[] row = Bytes.toBytes("family");
    WALEdit edit = new WALEdit();
    edit.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setFamily(row).setQualifier(row).setRow(row).setValue(row).setTimestamp(EnvironmentEdgeManager.currentTime()).setType(Type.Put).build());
    WALKeyImpl key1 = new WALKeyImpl(RI.getEncodedNameAsBytes(), TN, EnvironmentEdgeManager.currentTime(), MVCC);
    WAL.appendData(RI, key1, edit);
    WALKeyImpl key2 = new WALKeyImpl(RI.getEncodedNameAsBytes(), TN, key1.getWriteTime() + 1, MVCC);
    long txid = WAL.appendData(RI, key2, edit);
    // we need to make sure the two edits have both been added unackedAppends, so we have two syncs
    UTIL.waitFor(10000, () -> FUTURES.size() == 2);
    FUTURES.poll().completeExceptionally(new IOException("inject error"));
    FUTURES.poll().completeExceptionally(new IOException("inject error"));
    ARRIVE.await();
    // resume after 1 seconds, to give us enough time to enter the roll state
    EXECUTOR.schedule(() -> RESUME.countDown(), 1, TimeUnit.SECONDS);
    // let's roll the wal, before the fix in HBASE-25905, it will hang forever inside
    // waitForSafePoint
    WAL.rollWriter();
    // make sure we can finally succeed
    WAL.sync(txid);
}

Example 60 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class TestFSHLog method testUnflushedSeqIdTracking.

/**
 * Test case for https://issues.apache.org/jira/browse/HBASE-16721
 */
@Test
public void testUnflushedSeqIdTracking() throws IOException, InterruptedException {
    final String name = this.name.getMethodName();
    final byte[] b = Bytes.toBytes("b");
    final AtomicBoolean startHoldingForAppend = new AtomicBoolean(false);
    final CountDownLatch holdAppend = new CountDownLatch(1);
    final CountDownLatch flushFinished = new CountDownLatch(1);
    final CountDownLatch putFinished = new CountDownLatch(1);
    try (FSHLog log = new FSHLog(FS, CommonFSUtils.getRootDir(CONF), name, HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null)) {
        log.init();
        log.registerWALActionsListener(new WALActionsListener() {

            @Override
            public void visitLogEntryBeforeWrite(RegionInfo info, WALKey logKey, WALEdit logEdit) {
                if (startHoldingForAppend.get()) {
                    try {
                        holdAppend.await();
                    } catch (InterruptedException e) {
                        LOG.error(e.toString(), e);
                    }
                }
            }
        });
        // open a new region which uses this WAL
        TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name.getMethodName())).setColumnFamily(ColumnFamilyDescriptorBuilder.of(b)).build();
        RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
        ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
        final HRegion region = TEST_UTIL.createLocalHRegion(hri, CONF, htd, log);
        ExecutorService exec = Executors.newFixedThreadPool(2);
        // do a regular write first because of memstore size calculation.
        region.put(new Put(b).addColumn(b, b, b));
        startHoldingForAppend.set(true);
        exec.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    region.put(new Put(b).addColumn(b, b, b));
                    putFinished.countDown();
                } catch (IOException e) {
                    LOG.error(e.toString(), e);
                }
            }
        });
        // give the put a chance to start
        Threads.sleep(3000);
        exec.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    HRegion.FlushResult flushResult = region.flush(true);
                    LOG.info("Flush result:" + flushResult.getResult());
                    LOG.info("Flush succeeded:" + flushResult.isFlushSucceeded());
                    flushFinished.countDown();
                } catch (IOException e) {
                    LOG.error(e.toString(), e);
                }
            }
        });
        // give the flush a chance to start. Flush should have got the region lock, and
        // should have been waiting on the mvcc complete after this.
        Threads.sleep(3000);
        // let the append to WAL go through now that the flush already started
        holdAppend.countDown();
        putFinished.await();
        flushFinished.await();
        // check whether flush went through
        assertEquals("Region did not flush?", 1, region.getStoreFileList(new byte[][] { b }).size());
        // now check the region's unflushed seqIds.
        long seqId = log.getEarliestMemStoreSeqNum(hri.getEncodedNameAsBytes());
        assertEquals("Found seqId for the region which is already flushed", HConstants.NO_SEQNUM, seqId);
        region.close();
    }
}