Examples with WALFactory org.apache.hadoop.hbase.wal.WALFactory used on opensource projects

Example 61 with WALFactory

use of org.apache.hadoop.hbase.wal.WALFactory in project hbase by apache.

the class TestHRegion method testSkipRecoveredEditsReplayTheLastFileIgnored.

@Test
public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
    byte[] family = Bytes.toBytes("family");
    this.region = initHRegion(tableName, method, CONF, family);
    final WALFactory wals = new WALFactory(CONF, method);
    try {
        Path regiondir = region.getRegionFileSystem().getRegionDir();
        FileSystem fs = region.getRegionFileSystem().getFileSystem();
        byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
        byte[][] columns = region.getTableDescriptor().getColumnFamilyNames().toArray(new byte[0][]);
        assertEquals(0, region.getStoreFileList(columns).size());
        Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
        long maxSeqId = 1050;
        long minSeqId = 1000;
        for (long i = minSeqId; i <= maxSeqId; i += 10) {
            Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
            fs.create(recoveredEdits);
            WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
            long time = System.nanoTime();
            WALEdit edit = null;
            if (i == maxSeqId) {
                edit = WALEdit.createCompaction(region.getRegionInfo(), CompactionDescriptor.newBuilder().setTableName(ByteString.copyFrom(tableName.getName())).setFamilyName(ByteString.copyFrom(regionName)).setEncodedRegionName(ByteString.copyFrom(regionName)).setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString()))).setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName())).build());
            } else {
                edit = new WALEdit();
                edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i)));
            }
            writer.append(new WAL.Entry(new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
            writer.close();
        }
        long recoverSeqId = 1030;
        Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
        MonitoredTask status = TaskMonitor.get().createStatus(method);
        for (HStore store : region.getStores()) {
            maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1);
        }
        long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
        assertEquals(maxSeqId, seqId);
        // assert that the files are flushed
        assertEquals(1, region.getStoreFileList(columns).size());
    } finally {
        HBaseTestingUtil.closeRegionAndWAL(this.region);
        this.region = null;
        wals.close();
    }
}

Example 62 with WALFactory

use of org.apache.hadoop.hbase.wal.WALFactory in project hbase by apache.

the class TestHMobStore method init.

private void init(String methodName, Configuration conf, ColumnFamilyDescriptor cfd, boolean testStore) throws IOException {
    TableDescriptor td = TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setColumnFamily(cfd).build();
    // Setting up tje Region and Store
    Path basedir = new Path(DIR + methodName);
    Path tableDir = CommonFSUtils.getTableDir(basedir, td.getTableName());
    String logName = "logs";
    Path logdir = new Path(basedir, logName);
    FileSystem fs = FileSystem.get(conf);
    fs.delete(logdir, true);
    RegionInfo info = RegionInfoBuilder.newBuilder(td.getTableName()).build();
    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
    final Configuration walConf = new Configuration(conf);
    CommonFSUtils.setRootDir(walConf, basedir);
    final WALFactory wals = new WALFactory(walConf, methodName);
    region = new HRegion(tableDir, wals.getWAL(info), fs, conf, info, td, null);
    region.setMobFileCache(new MobFileCache(conf));
    store = new HMobStore(region, cfd, conf, false);
    if (testStore) {
        init(conf, cfd);
    }
}

Example 63 with WALFactory

use of org.apache.hadoop.hbase.wal.WALFactory in project hbase by apache.

the class TestHRegion method testSkipRecoveredEditsReplaySomeIgnored.

@Test
public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
    byte[] family = Bytes.toBytes("family");
    this.region = initHRegion(tableName, method, CONF, family);
    final WALFactory wals = new WALFactory(CONF, method);
    try {
        Path regiondir = region.getRegionFileSystem().getRegionDir();
        FileSystem fs = region.getRegionFileSystem().getFileSystem();
        byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
        Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
        long maxSeqId = 1050;
        long minSeqId = 1000;
        for (long i = minSeqId; i <= maxSeqId; i += 10) {
            Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
            fs.create(recoveredEdits);
            WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
            long time = System.nanoTime();
            WALEdit edit = new WALEdit();
            edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i)));
            writer.append(new WAL.Entry(new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
            writer.close();
        }
        long recoverSeqId = 1030;
        MonitoredTask status = TaskMonitor.get().createStatus(method);
        Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
        for (HStore store : region.getStores()) {
            maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1);
        }
        long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
        assertEquals(maxSeqId, seqId);
        region.getMVCC().advanceTo(seqId);
        Get get = new Get(row);
        Result result = region.get(get);
        for (long i = minSeqId; i <= maxSeqId; i += 10) {
            List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
            if (i < recoverSeqId) {
                assertEquals(0, kvs.size());
            } else {
                assertEquals(1, kvs.size());
                assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
            }
        }
    } finally {
        HBaseTestingUtil.closeRegionAndWAL(this.region);
        this.region = null;
        wals.close();
    }
}

Example 64 with WALFactory

use of org.apache.hadoop.hbase.wal.WALFactory in project hbase by apache.

the class TestWALMonotonicallyIncreasingSeqId method initHRegion.

private HRegion initHRegion(TableDescriptor htd, byte[] startKey, byte[] stopKey, int replicaId) throws IOException {
    Configuration conf = TEST_UTIL.getConfiguration();
    conf.set("hbase.wal.provider", walProvider);
    conf.setBoolean("hbase.hregion.mvcc.preassign", false);
    Path tableDir = CommonFSUtils.getTableDir(testDir, htd.getTableName());
    RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(startKey).setEndKey(stopKey).setReplicaId(replicaId).setRegionId(0).build();
    fileSystem = tableDir.getFileSystem(conf);
    final Configuration walConf = new Configuration(conf);
    CommonFSUtils.setRootDir(walConf, tableDir);
    this.walConf = walConf;
    wals = new WALFactory(walConf, "log_" + replicaId);
    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
    HRegion region = HRegion.createHRegion(info, TEST_UTIL.getDefaultRootDirPath(), conf, htd, wals.getWAL(info));
    return region;
}