Examples with WALEdit org.apache.hadoop.hbase.wal.WALEdit used on opensource projects

Example 21 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class TestWALObserver method addWALEdits.

private void addWALEdits(final TableName tableName, final RegionInfo hri, final byte[] rowName, final byte[] family, final int count, EnvironmentEdge ee, final WAL wal, final NavigableMap<byte[], Integer> scopes, final MultiVersionConcurrencyControl mvcc) throws IOException {
    String familyStr = Bytes.toString(family);
    long txid = -1;
    for (int j = 0; j < count; j++) {
        byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
        byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
        WALEdit edit = new WALEdit();
        edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes));
        // uses WALKeyImpl instead of HLogKey on purpose. will only work for tests where we don't care
        // about legacy coprocessors
        txid = wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, ee.currentTime(), mvcc), edit);
    }
    if (-1 != txid) {
        wal.sync(txid);
    }
}

Example 22 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class TestWALObserver method verifyWritesSeen.

private void verifyWritesSeen(final WAL log, final SampleRegionWALCoprocessor cp, final boolean seesLegacy) throws Exception {
    RegionInfo hri = createBasicHRegionInfo(Bytes.toString(TEST_TABLE));
    TableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE));
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (byte[] fam : htd.getColumnFamilyNames()) {
        scopes.put(fam, 0);
    }
    Path basedir = new Path(this.hbaseRootDir, Bytes.toString(TEST_TABLE));
    deleteDir(basedir);
    fs.mkdirs(new Path(basedir, hri.getEncodedName()));
    // TEST_FAMILY[0] shall be removed from WALEdit.
    // TEST_FAMILY[1] value shall be changed.
    // TEST_FAMILY[2] shall be added to WALEdit, although it's not in the put.
    cp.setTestValues(TEST_TABLE, TEST_ROW, TEST_FAMILY[0], TEST_QUALIFIER[0], TEST_FAMILY[1], TEST_QUALIFIER[1], TEST_FAMILY[2], TEST_QUALIFIER[2]);
    assertFalse(cp.isPreWALWriteCalled());
    assertFalse(cp.isPostWALWriteCalled());
    // TEST_FAMILY[2] is not in the put, however it shall be added by the tested
    // coprocessor.
    // Use a Put to create familyMap.
    Put p = creatPutWith2Families(TEST_ROW);
    Map<byte[], List<Cell>> familyMap = p.getFamilyCellMap();
    WALEdit edit = new WALEdit();
    edit.add(familyMap);
    boolean foundFamily0 = false;
    boolean foundFamily2 = false;
    boolean modifiedFamily1 = false;
    List<Cell> cells = edit.getCells();
    for (Cell cell : cells) {
        if (Arrays.equals(CellUtil.cloneFamily(cell), TEST_FAMILY[0])) {
            foundFamily0 = true;
        }
        if (Arrays.equals(CellUtil.cloneFamily(cell), TEST_FAMILY[2])) {
            foundFamily2 = true;
        }
        if (Arrays.equals(CellUtil.cloneFamily(cell), TEST_FAMILY[1])) {
            if (!Arrays.equals(CellUtil.cloneValue(cell), TEST_VALUE[1])) {
                modifiedFamily1 = true;
            }
        }
    }
    assertTrue(foundFamily0);
    assertFalse(foundFamily2);
    assertFalse(modifiedFamily1);
    // it's where WAL write cp should occur.
    long now = EnvironmentEdgeManager.currentTime();
    // we use HLogKey here instead of WALKeyImpl directly to support legacy coprocessors.
    long txid = log.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(), now, new MultiVersionConcurrencyControl(), scopes), edit);
    log.sync(txid);
    // the edit shall have been change now by the coprocessor.
    foundFamily0 = false;
    foundFamily2 = false;
    modifiedFamily1 = false;
    for (Cell cell : cells) {
        if (Arrays.equals(CellUtil.cloneFamily(cell), TEST_FAMILY[0])) {
            foundFamily0 = true;
        }
        if (Arrays.equals(CellUtil.cloneFamily(cell), TEST_FAMILY[2])) {
            foundFamily2 = true;
        }
        if (Arrays.equals(CellUtil.cloneFamily(cell), TEST_FAMILY[1])) {
            if (!Arrays.equals(CellUtil.cloneValue(cell), TEST_VALUE[1])) {
                modifiedFamily1 = true;
            }
        }
    }
    assertFalse(foundFamily0);
    assertTrue(foundFamily2);
    assertTrue(modifiedFamily1);
    assertTrue(cp.isPreWALWriteCalled());
    assertTrue(cp.isPostWALWriteCalled());
}

Example 23 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class TestWALObserver method testEmptyWALEditAreNotSeen.

/**
 * Coprocessors shouldn't get notice of empty waledits.
 */
@Test
public void testEmptyWALEditAreNotSeen() throws Exception {
    RegionInfo hri = createBasicHRegionInfo(Bytes.toString(TEST_TABLE));
    TableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE));
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (byte[] fam : htd.getColumnFamilyNames()) {
        scopes.put(fam, 0);
    }
    WAL log = wals.getWAL(null);
    try {
        SampleRegionWALCoprocessor cp = getCoprocessor(log, SampleRegionWALCoprocessor.class);
        cp.setTestValues(TEST_TABLE, null, null, null, null, null, null, null);
        assertFalse(cp.isPreWALWriteCalled());
        assertFalse(cp.isPostWALWriteCalled());
        final long now = EnvironmentEdgeManager.currentTime();
        long txid = log.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(), now, mvcc, scopes), new WALEdit());
        log.sync(txid);
        assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPreWALWriteCalled());
        assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPostWALWriteCalled());
    } finally {
        log.close();
    }
}

Example 24 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class TestHRegion method testArchiveRecoveredEditsReplay.

@Test
public void testArchiveRecoveredEditsReplay() throws Exception {
    byte[] family = Bytes.toBytes("family");
    this.region = initHRegion(tableName, method, CONF, family);
    final WALFactory wals = new WALFactory(CONF, method);
    try {
        Path regiondir = region.getRegionFileSystem().getRegionDir();
        FileSystem fs = region.getRegionFileSystem().getFileSystem();
        byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
        Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
        long maxSeqId = 1050;
        long minSeqId = 1000;
        for (long i = minSeqId; i <= maxSeqId; i += 10) {
            Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
            fs.create(recoveredEdits);
            WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
            long time = System.nanoTime();
            WALEdit edit = new WALEdit();
            edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i)));
            writer.append(new WAL.Entry(new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
            writer.close();
        }
        MonitoredTask status = TaskMonitor.get().createStatus(method);
        Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
        for (HStore store : region.getStores()) {
            maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId - 1);
        }
        CONF.set("hbase.region.archive.recovered.edits", "true");
        CONF.set(CommonFSUtils.HBASE_WAL_DIR, "/custom_wal_dir");
        long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
        assertEquals(maxSeqId, seqId);
        region.getMVCC().advanceTo(seqId);
        String fakeFamilyName = recoveredEditsDir.getName();
        Path rootDir = new Path(CONF.get(HConstants.HBASE_DIR));
        Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePathForRootDir(rootDir, region.getRegionInfo(), Bytes.toBytes(fakeFamilyName));
        FileStatus[] list = TEST_UTIL.getTestFileSystem().listStatus(storeArchiveDir);
        assertEquals(6, list.length);
    } finally {
        CONF.set("hbase.region.archive.recovered.edits", "false");
        CONF.set(CommonFSUtils.HBASE_WAL_DIR, "");
        HBaseTestingUtil.closeRegionAndWAL(this.region);
        this.region = null;
        wals.close();
    }
}

Example 25 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class TestRecoveredEdits method verifyAllEditsMadeItIn.

/**
 * @return Return how many edits seen.
 */
// Used by TestWALPlayer over in hbase-mapreduce too.
public static int verifyAllEditsMadeItIn(final FileSystem fs, final Configuration conf, final Path edits, final HRegion region) throws IOException {
    int count = 0;
    // Read all cells from recover edits
    List<Cell> walCells = new ArrayList<>();
    try (WAL.Reader reader = WALFactory.createReader(fs, edits, conf)) {
        WAL.Entry entry;
        while ((entry = reader.next()) != null) {
            WALKey key = entry.getKey();
            WALEdit val = entry.getEdit();
            count++;
            // Check this edit is for this region.
            if (!Bytes.equals(key.getEncodedRegionName(), region.getRegionInfo().getEncodedNameAsBytes())) {
                continue;
            }
            Cell previous = null;
            for (Cell cell : val.getCells()) {
                if (WALEdit.isMetaEditFamily(cell)) {
                    continue;
                }
                if (previous != null && CellComparatorImpl.COMPARATOR.compareRows(previous, cell) == 0) {
                    continue;
                }
                previous = cell;
                walCells.add(cell);
            }
        }
    }
    // Read all cells from region
    List<Cell> regionCells = new ArrayList<>();
    try (RegionScanner scanner = region.getScanner(new Scan())) {
        List<Cell> tmpCells;
        do {
            tmpCells = new ArrayList<>();
            scanner.nextRaw(tmpCells);
            regionCells.addAll(tmpCells);
        } while (!tmpCells.isEmpty());
    }
    Collections.sort(walCells, CellComparatorImpl.COMPARATOR);
    int found = 0;
    for (int i = 0, j = 0; i < walCells.size() && j < regionCells.size(); ) {
        int compareResult = PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, walCells.get(i), regionCells.get(j));
        if (compareResult == 0) {
            i++;
            j++;
            found++;
        } else if (compareResult > 0) {
            j++;
        } else {
            i++;
        }
    }
    assertEquals("Only found " + found + " cells in region, but there are " + walCells.size() + " cells in recover edits", found, walCells.size());
    return count;
}