Examples with WALKeyImpl org.apache.hadoop.hbase.wal.WALKeyImpl used on opensource projects

Example 46 with WALKeyImpl

use of org.apache.hadoop.hbase.wal.WALKeyImpl in project hbase by apache.

the class TestReplicationEndpoint method createEntry.

private Entry createEntry(String tableName, TreeMap<byte[], Integer> scopes, byte[]... kvs) {
    WALKeyImpl key1 = new WALKeyImpl(new byte[0], TableName.valueOf(tableName), EnvironmentEdgeManager.currentTime() - 1L, scopes);
    WALEdit edit1 = new WALEdit();
    for (byte[] kv : kvs) {
        edit1.add(new KeyValue(kv, kv, kv));
    }
    return new Entry(key1, edit1);
}

Example 47 with WALKeyImpl

use of org.apache.hadoop.hbase.wal.WALKeyImpl in project hbase by apache.

the class TestReplicationWALEntryFilters method createEntry.

private Entry createEntry(TreeMap<byte[], Integer> scopes, byte[]... kvs) {
    WALKeyImpl key1 = new WALKeyImpl(new byte[0], TableName.valueOf("foo"), EnvironmentEdgeManager.currentTime(), scopes);
    WALEdit edit1 = new WALEdit();
    for (byte[] kv : kvs) {
        edit1.add(new KeyValue(kv, kv, kv));
    }
    return new Entry(key1, edit1);
}

Example 48 with WALKeyImpl

use of org.apache.hadoop.hbase.wal.WALKeyImpl in project hbase by apache.

the class TestAsyncFSWAL method testBrokenWriter.

@Test
public void testBrokenWriter() throws Exception {
    RegionServerServices services = mock(RegionServerServices.class);
    when(services.getConfiguration()).thenReturn(CONF);
    TableDescriptor td = TableDescriptorBuilder.newBuilder(TableName.valueOf("table")).setColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();
    RegionInfo ri = RegionInfoBuilder.newBuilder(td.getTableName()).build();
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (byte[] fam : td.getColumnFamilyNames()) {
        scopes.put(fam, 0);
    }
    long timestamp = EnvironmentEdgeManager.currentTime();
    String testName = currentTest.getMethodName();
    AtomicInteger failedCount = new AtomicInteger(0);
    try (LogRoller roller = new LogRoller(services);
        AsyncFSWAL wal = new AsyncFSWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(), testName, CONF, null, true, null, null, GROUP, CHANNEL_CLASS) {

            @Override
            protected AsyncWriter createWriterInstance(Path path) throws IOException {
                AsyncWriter writer = super.createWriterInstance(path);
                return new AsyncWriter() {

                    @Override
                    public void close() throws IOException {
                        writer.close();
                    }

                    @Override
                    public long getLength() {
                        return writer.getLength();
                    }

                    @Override
                    public long getSyncedLength() {
                        return writer.getSyncedLength();
                    }

                    @Override
                    public CompletableFuture<Long> sync(boolean forceSync) {
                        CompletableFuture<Long> result = writer.sync(forceSync);
                        if (failedCount.incrementAndGet() < 1000) {
                            CompletableFuture<Long> future = new CompletableFuture<>();
                            FutureUtils.addListener(result, (r, e) -> future.completeExceptionally(new IOException("Inject Error")));
                            return future;
                        } else {
                            return result;
                        }
                    }

                    @Override
                    public void append(Entry entry) {
                        writer.append(entry);
                    }
                };
            }
        }) {
        wal.init();
        roller.addWAL(wal);
        roller.start();
        int numThreads = 10;
        AtomicReference<Exception> error = new AtomicReference<>();
        Thread[] threads = new Thread[numThreads];
        for (int i = 0; i < 10; i++) {
            final int index = i;
            threads[index] = new Thread("Write-Thread-" + index) {

                @Override
                public void run() {
                    byte[] row = Bytes.toBytes("row" + index);
                    WALEdit cols = new WALEdit();
                    cols.add(new KeyValue(row, row, row, timestamp + index, row));
                    WALKeyImpl key = new WALKeyImpl(ri.getEncodedNameAsBytes(), td.getTableName(), SequenceId.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, scopes);
                    try {
                        wal.append(ri, key, cols, true);
                    } catch (IOException e) {
                        // should not happen
                        throw new UncheckedIOException(e);
                    }
                    try {
                        wal.sync();
                    } catch (IOException e) {
                        error.set(e);
                    }
                }
            };
        }
        for (Thread t : threads) {
            t.start();
        }
        for (Thread t : threads) {
            t.join();
        }
        assertNull(error.get());
    }
}

Example 49 with WALKeyImpl

use of org.apache.hadoop.hbase.wal.WALKeyImpl in project hbase by apache.

the class AbstractTestFSWAL method testFlushSequenceIdIsGreaterThanAllEditsInHFile.

/**
 * Test flush for sure has a sequence id that is beyond the last edit appended. We do this by
 * slowing appends in the background ring buffer thread while in foreground we call flush. The
 * addition of the sync over HRegion in flush should fix an issue where flush was returning before
 * all of its appends had made it out to the WAL (HBASE-11109).
 * @throws IOException
 * @see <a href="https://issues.apache.org/jira/browse/HBASE-11109">HBASE-11109</a>
 */
@Test
public void testFlushSequenceIdIsGreaterThanAllEditsInHFile() throws IOException {
    String testName = currentTest.getMethodName();
    final TableName tableName = TableName.valueOf(testName);
    final RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build();
    final byte[] rowName = tableName.getName();
    final TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
    HRegion r = HBaseTestingUtil.createRegionAndWAL(hri, TEST_UTIL.getDefaultRootDirPath(), TEST_UTIL.getConfiguration(), htd);
    HBaseTestingUtil.closeRegionAndWAL(r);
    final int countPerFamily = 10;
    final AtomicBoolean goslow = new AtomicBoolean(false);
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (byte[] fam : htd.getColumnFamilyNames()) {
        scopes.put(fam, 0);
    }
    // subclass and doctor a method.
    AbstractFSWAL<?> wal = newSlowWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(), testName, CONF, null, true, null, null, new Runnable() {

        @Override
        public void run() {
            if (goslow.get()) {
                Threads.sleep(100);
                LOG.debug("Sleeping before appending 100ms");
            }
        }
    });
    HRegion region = HRegion.openHRegion(TEST_UTIL.getConfiguration(), TEST_UTIL.getTestFileSystem(), TEST_UTIL.getDefaultRootDirPath(), hri, htd, wal);
    EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
    try {
        List<Put> puts = null;
        for (byte[] fam : htd.getColumnFamilyNames()) {
            puts = TestWALReplay.addRegionEdits(rowName, fam, countPerFamily, ee, region, "x");
        }
        // Now assert edits made it in.
        final Get g = new Get(rowName);
        Result result = region.get(g);
        assertEquals(countPerFamily * htd.getColumnFamilyNames().size(), result.size());
        // Construct a WALEdit and add it a few times to the WAL.
        WALEdit edits = new WALEdit();
        for (Put p : puts) {
            CellScanner cs = p.cellScanner();
            while (cs.advance()) {
                edits.add(cs.current());
            }
        }
        // Add any old cluster id.
        List<UUID> clusterIds = new ArrayList<>(1);
        clusterIds.add(TEST_UTIL.getRandomUUID());
        // Now make appends run slow.
        goslow.set(true);
        for (int i = 0; i < countPerFamily; i++) {
            final RegionInfo info = region.getRegionInfo();
            final WALKeyImpl logkey = new WALKeyImpl(info.getEncodedNameAsBytes(), tableName, EnvironmentEdgeManager.currentTime(), clusterIds, -1, -1, region.getMVCC(), scopes);
            wal.append(info, logkey, edits, true);
            region.getMVCC().completeAndWait(logkey.getWriteEntry());
        }
        region.flush(true);
        // FlushResult.flushSequenceId is not visible here so go get the current sequence id.
        long currentSequenceId = region.getReadPoint(null);
        // Now release the appends
        goslow.set(false);
        assertTrue(currentSequenceId >= region.getReadPoint(null));
    } finally {
        region.close(true);
        wal.close();
    }
}

Example 50 with WALKeyImpl

use of org.apache.hadoop.hbase.wal.WALKeyImpl in project hbase by apache.

the class AbstractTestFSWAL method testWriteEntryCanBeNull.

@Test
public void testWriteEntryCanBeNull() throws IOException {
    String testName = currentTest.getMethodName();
    AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(), testName, CONF, null, true, null, null);
    wal.close();
    TableDescriptor td = TableDescriptorBuilder.newBuilder(TableName.valueOf("table")).setColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();
    RegionInfo ri = RegionInfoBuilder.newBuilder(td.getTableName()).build();
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (byte[] fam : td.getColumnFamilyNames()) {
        scopes.put(fam, 0);
    }
    long timestamp = EnvironmentEdgeManager.currentTime();
    byte[] row = Bytes.toBytes("row");
    WALEdit cols = new WALEdit();
    cols.add(new KeyValue(row, row, row, timestamp, row));
    WALKeyImpl key = new WALKeyImpl(ri.getEncodedNameAsBytes(), td.getTableName(), SequenceId.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, scopes);
    try {
        wal.append(ri, key, cols, true);
        fail("Should fail since the wal has already been closed");
    } catch (IOException e) {
        // expected
        assertThat(e.getMessage(), containsString("log is closed"));
        // the WriteEntry should be null since we fail before setting it.
        assertNull(key.getWriteEntry());
    }
}