Examples with WALKey org.apache.hadoop.hbase.wal.WALKey used on opensource projects

Example 31 with WALKey

use of org.apache.hadoop.hbase.wal.WALKey in project hbase by apache.

the class TestWALObserver method testWALCoprocessorReplay.

/**
   * Test WAL replay behavior with WALObserver.
   */
@Test
public void testWALCoprocessorReplay() throws Exception {
    // WAL replay is handled at HRegion::replayRecoveredEdits(), which is
    // ultimately called by HRegion::initialize()
    final TableName tableName = TableName.valueOf(currentTest.getMethodName());
    final HTableDescriptor htd = getBasic3FamilyHTableDescriptor(tableName);
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    // final HRegionInfo hri =
    // createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
    // final HRegionInfo hri1 =
    // createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
    final HRegionInfo hri = new HRegionInfo(tableName, null, null);
    final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
    deleteDir(basedir);
    fs.mkdirs(new Path(basedir, hri.getEncodedName()));
    final Configuration newConf = HBaseConfiguration.create(this.conf);
    // WAL wal = new WAL(this.fs, this.dir, this.oldLogDir, this.conf);
    WAL wal = wals.getWAL(UNSPECIFIED_REGION, null);
    // Put p = creatPutWith2Families(TEST_ROW);
    WALEdit edit = new WALEdit();
    long now = EnvironmentEdgeManager.currentTime();
    final int countPerFamily = 1000;
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (HColumnDescriptor hcd : htd.getFamilies()) {
        scopes.put(hcd.getName(), 0);
    }
    for (HColumnDescriptor hcd : htd.getFamilies()) {
        addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily, EnvironmentEdgeManager.getDelegate(), wal, scopes, mvcc);
    }
    wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes), edit, true);
    // sync to fs.
    wal.sync();
    User user = HBaseTestingUtility.getDifferentUser(newConf, ".replay.wal.secondtime");
    user.runAs(new PrivilegedExceptionAction<Void>() {

        public Void run() throws Exception {
            Path p = runWALSplit(newConf);
            LOG.info("WALSplit path == " + p);
            FileSystem newFS = FileSystem.get(newConf);
            // Make a new wal for new region open.
            final WALFactory wals2 = new WALFactory(conf, null, ServerName.valueOf(currentTest.getMethodName() + "2", 16010, System.currentTimeMillis()).toString());
            WAL wal2 = wals2.getWAL(UNSPECIFIED_REGION, null);
            ;
            HRegion region = HRegion.openHRegion(newConf, FileSystem.get(newConf), hbaseRootDir, hri, htd, wal2, TEST_UTIL.getHBaseCluster().getRegionServer(0), null);
            long seqid2 = region.getOpenSeqNum();
            SampleRegionWALObserver cp2 = (SampleRegionWALObserver) region.getCoprocessorHost().findCoprocessor(SampleRegionWALObserver.class.getName());
            // TODO: asserting here is problematic.
            assertNotNull(cp2);
            assertTrue(cp2.isPreWALRestoreCalled());
            assertTrue(cp2.isPostWALRestoreCalled());
            region.close();
            wals2.close();
            return null;
        }
    });
}

Example 32 with WALKey

use of org.apache.hadoop.hbase.wal.WALKey in project hbase by apache.

the class TestWALObserver method verifyWritesSeen.

private void verifyWritesSeen(final WAL log, final SampleRegionWALObserver cp, final boolean seesLegacy) throws Exception {
    HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(TEST_TABLE));
    final HTableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE));
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (byte[] fam : htd.getFamiliesKeys()) {
        scopes.put(fam, 0);
    }
    Path basedir = new Path(this.hbaseRootDir, Bytes.toString(TEST_TABLE));
    deleteDir(basedir);
    fs.mkdirs(new Path(basedir, hri.getEncodedName()));
    // TEST_FAMILY[0] shall be removed from WALEdit.
    // TEST_FAMILY[1] value shall be changed.
    // TEST_FAMILY[2] shall be added to WALEdit, although it's not in the put.
    cp.setTestValues(TEST_TABLE, TEST_ROW, TEST_FAMILY[0], TEST_QUALIFIER[0], TEST_FAMILY[1], TEST_QUALIFIER[1], TEST_FAMILY[2], TEST_QUALIFIER[2]);
    assertFalse(cp.isPreWALWriteCalled());
    assertFalse(cp.isPostWALWriteCalled());
    // TEST_FAMILY[2] is not in the put, however it shall be added by the tested
    // coprocessor.
    // Use a Put to create familyMap.
    Put p = creatPutWith2Families(TEST_ROW);
    Map<byte[], List<Cell>> familyMap = p.getFamilyCellMap();
    WALEdit edit = new WALEdit();
    addFamilyMapToWALEdit(familyMap, edit);
    boolean foundFamily0 = false;
    boolean foundFamily2 = false;
    boolean modifiedFamily1 = false;
    List<Cell> cells = edit.getCells();
    for (Cell cell : cells) {
        if (Arrays.equals(CellUtil.cloneFamily(cell), TEST_FAMILY[0])) {
            foundFamily0 = true;
        }
        if (Arrays.equals(CellUtil.cloneFamily(cell), TEST_FAMILY[2])) {
            foundFamily2 = true;
        }
        if (Arrays.equals(CellUtil.cloneFamily(cell), TEST_FAMILY[1])) {
            if (!Arrays.equals(CellUtil.cloneValue(cell), TEST_VALUE[1])) {
                modifiedFamily1 = true;
            }
        }
    }
    assertTrue(foundFamily0);
    assertFalse(foundFamily2);
    assertFalse(modifiedFamily1);
    // it's where WAL write cp should occur.
    long now = EnvironmentEdgeManager.currentTime();
    // we use HLogKey here instead of WALKey directly to support legacy coprocessors.
    long txid = log.append(hri, new WALKey(hri.getEncodedNameAsBytes(), hri.getTable(), now, new MultiVersionConcurrencyControl(), scopes), edit, true);
    log.sync(txid);
    // the edit shall have been change now by the coprocessor.
    foundFamily0 = false;
    foundFamily2 = false;
    modifiedFamily1 = false;
    for (Cell cell : cells) {
        if (Arrays.equals(CellUtil.cloneFamily(cell), TEST_FAMILY[0])) {
            foundFamily0 = true;
        }
        if (Arrays.equals(CellUtil.cloneFamily(cell), TEST_FAMILY[2])) {
            foundFamily2 = true;
        }
        if (Arrays.equals(CellUtil.cloneFamily(cell), TEST_FAMILY[1])) {
            if (!Arrays.equals(CellUtil.cloneValue(cell), TEST_VALUE[1])) {
                modifiedFamily1 = true;
            }
        }
    }
    assertFalse(foundFamily0);
    assertTrue(foundFamily2);
    assertTrue(modifiedFamily1);
    assertTrue(cp.isPreWALWriteCalled());
    assertTrue(cp.isPostWALWriteCalled());
}

Example 33 with WALKey

use of org.apache.hadoop.hbase.wal.WALKey in project hbase by apache.

the class TestDistributedLogSplitting method makeWAL.

public void makeWAL(HRegionServer hrs, List<HRegionInfo> regions, String tname, String fname, int num_edits, int edit_size, boolean cleanShutdown) throws IOException {
    TableName fullTName = TableName.valueOf(tname);
    // remove root and meta region
    regions.remove(HRegionInfo.FIRST_META_REGIONINFO);
    // using one sequenceId for edits across all regions is ok.
    final AtomicLong sequenceId = new AtomicLong(10);
    for (Iterator<HRegionInfo> iter = regions.iterator(); iter.hasNext(); ) {
        HRegionInfo regionInfo = iter.next();
        if (regionInfo.getTable().isSystemTable()) {
            iter.remove();
        }
    }
    HTableDescriptor htd = new HTableDescriptor(fullTName);
    byte[] family = Bytes.toBytes(fname);
    htd.addFamily(new HColumnDescriptor(family));
    byte[] value = new byte[edit_size];
    List<HRegionInfo> hris = new ArrayList<>();
    for (HRegionInfo region : regions) {
        if (!region.getTable().getNameAsString().equalsIgnoreCase(tname)) {
            continue;
        }
        hris.add(region);
    }
    LOG.info("Creating wal edits across " + hris.size() + " regions.");
    for (int i = 0; i < edit_size; i++) {
        value[i] = (byte) ('a' + (i % 26));
    }
    int n = hris.size();
    int[] counts = new int[n];
    // sync every ~30k to line up with desired wal rolls
    final int syncEvery = 30 * 1024 / edit_size;
    if (n > 0) {
        for (int i = 0; i < num_edits; i += 1) {
            WALEdit e = new WALEdit();
            HRegionInfo curRegionInfo = hris.get(i % n);
            final WAL log = hrs.getWAL(curRegionInfo);
            byte[] startRow = curRegionInfo.getStartKey();
            if (startRow == null || startRow.length == 0) {
                startRow = new byte[] { 0, 0, 0, 0, 1 };
            }
            byte[] row = Bytes.incrementBytes(startRow, counts[i % n]);
            // use last 5 bytes because
            row = Arrays.copyOfRange(row, 3, 8);
            // HBaseTestingUtility.createMultiRegions use 5 bytes
            // key
            byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i));
            e.add(new KeyValue(row, family, qualifier, System.currentTimeMillis(), value));
            log.append(curRegionInfo, new WALKey(curRegionInfo.getEncodedNameAsBytes(), fullTName, System.currentTimeMillis()), e, true);
            if (0 == i % syncEvery) {
                log.sync();
            }
            counts[i % n] += 1;
        }
    }
    // will cause errors if done after.
    for (HRegionInfo info : hris) {
        final WAL log = hrs.getWAL(info);
        log.sync();
    }
    if (cleanShutdown) {
        for (HRegionInfo info : hris) {
            final WAL log = hrs.getWAL(info);
            log.shutdown();
        }
    }
    for (int i = 0; i < n; i++) {
        LOG.info("region " + hris.get(i).getRegionNameAsString() + " has " + counts[i] + " edits");
    }
    return;
}

Example 34 with WALKey

use of org.apache.hadoop.hbase.wal.WALKey in project hbase by apache.

the class TestWALPlayer method testWALKeyValueMapper.

private void testWALKeyValueMapper(final String tableConfigKey) throws Exception {
    Configuration configuration = new Configuration();
    configuration.set(tableConfigKey, "table");
    WALKeyValueMapper mapper = new WALKeyValueMapper();
    WALKey key = mock(WALKey.class);
    when(key.getTablename()).thenReturn(TableName.valueOf("table"));
    @SuppressWarnings("unchecked") Mapper<WALKey, WALEdit, ImmutableBytesWritable, KeyValue>.Context<WALKey, WALEdit, ImmutableBytesWritable, KeyValue> context = mock(Context.class);
    when(context.getConfiguration()).thenReturn(configuration);
    WALEdit value = mock(WALEdit.class);
    ArrayList<Cell> values = new ArrayList<>();
    KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("family"), null);
    values.add(kv1);
    when(value.getCells()).thenReturn(values);
    mapper.setup(context);
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            ImmutableBytesWritable writer = (ImmutableBytesWritable) invocation.getArguments()[0];
            KeyValue key = (KeyValue) invocation.getArguments()[1];
            assertEquals("row", Bytes.toString(writer.get()));
            assertEquals("row", Bytes.toString(CellUtil.cloneRow(key)));
            return null;
        }
    }).when(context).write(any(ImmutableBytesWritable.class), any(KeyValue.class));
    mapper.map(key, value, context);
}

Example 35 with WALKey

use of org.apache.hadoop.hbase.wal.WALKey in project hbase by apache.

the class TestDistributedLogSplitting method testSameVersionUpdatesRecovery.

@Ignore("DLR is broken by HBASE-12751")
@Test(timeout = 300000)
public void testSameVersionUpdatesRecovery() throws Exception {
    LOG.info("testSameVersionUpdatesRecovery");
    conf.setLong("hbase.regionserver.hlog.blocksize", 15 * 1024);
    conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
    startCluster(NUM_RS);
    final AtomicLong sequenceId = new AtomicLong(100);
    final int NUM_REGIONS_TO_CREATE = 40;
    final int NUM_LOG_LINES = 1000;
    // turn off load balancing to prevent regions from moving around otherwise
    // they will consume recovered.edits
    master.balanceSwitch(false);
    List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
    final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
    Table ht = installTable(zkw, name.getMethodName(), "family", NUM_REGIONS_TO_CREATE);
    try {
        List<HRegionInfo> regions = null;
        HRegionServer hrs = null;
        for (int i = 0; i < NUM_RS; i++) {
            boolean isCarryingMeta = false;
            hrs = rsts.get(i).getRegionServer();
            regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
            for (HRegionInfo region : regions) {
                if (region.isMetaRegion()) {
                    isCarryingMeta = true;
                    break;
                }
            }
            if (isCarryingMeta) {
                continue;
            }
            break;
        }
        LOG.info("#regions = " + regions.size());
        Iterator<HRegionInfo> it = regions.iterator();
        while (it.hasNext()) {
            HRegionInfo region = it.next();
            if (region.isMetaTable() || region.getEncodedName().equals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) {
                it.remove();
            }
        }
        if (regions.isEmpty())
            return;
        HRegionInfo curRegionInfo = regions.get(0);
        byte[] startRow = curRegionInfo.getStartKey();
        if (startRow == null || startRow.length == 0) {
            startRow = new byte[] { 0, 0, 0, 0, 1 };
        }
        byte[] row = Bytes.incrementBytes(startRow, 1);
        // use last 5 bytes because HBaseTestingUtility.createMultiRegions use 5 bytes key
        row = Arrays.copyOfRange(row, 3, 8);
        long value = 0;
        TableName tableName = TableName.valueOf(name.getMethodName());
        byte[] family = Bytes.toBytes("family");
        byte[] qualifier = Bytes.toBytes("c1");
        long timeStamp = System.currentTimeMillis();
        HTableDescriptor htd = new HTableDescriptor(tableName);
        htd.addFamily(new HColumnDescriptor(family));
        final WAL wal = hrs.getWAL(curRegionInfo);
        for (int i = 0; i < NUM_LOG_LINES; i += 1) {
            WALEdit e = new WALEdit();
            value++;
            e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value)));
            wal.append(curRegionInfo, new WALKey(curRegionInfo.getEncodedNameAsBytes(), tableName, System.currentTimeMillis()), e, true);
        }
        wal.sync();
        wal.shutdown();
        // wait for abort completes
        this.abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE);
        // verify we got the last value
        LOG.info("Verification Starts...");
        Get g = new Get(row);
        Result r = ht.get(g);
        long theStoredVal = Bytes.toLong(r.getValue(family, qualifier));
        assertEquals(value, theStoredVal);
        // after flush
        LOG.info("Verification after flush...");
        TEST_UTIL.getAdmin().flush(tableName);
        r = ht.get(g);
        theStoredVal = Bytes.toLong(r.getValue(family, qualifier));
        assertEquals(value, theStoredVal);
    } finally {
        if (ht != null)
            ht.close();
        if (zkw != null)
            zkw.close();
    }
}