Examples with WALFactory org.apache.hadoop.hbase.wal.WALFactory used on opensource projects

Example 31 with WALFactory

use of org.apache.hadoop.hbase.wal.WALFactory in project hbase by apache.

the class TestReplicationSourceManager method testLogRoll.

@Test
public void testLogRoll() throws Exception {
    long baseline = 1000;
    long time = baseline;
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    KeyValue kv = new KeyValue(r1, f1, r1);
    WALEdit edit = new WALEdit();
    edit.add(kv);
    List<WALActionsListener> listeners = new ArrayList<>(1);
    listeners.add(replication);
    final WALFactory wals = new WALFactory(utility.getConfiguration(), listeners, URLEncoder.encode("regionserver:60020", "UTF8"));
    final WAL wal = wals.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace());
    manager.init();
    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("tableame"));
    htd.addFamily(new HColumnDescriptor(f1));
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (byte[] fam : htd.getFamiliesKeys()) {
        scopes.put(fam, 0);
    }
    // Testing normal log rolling every 20
    for (long i = 1; i < 101; i++) {
        if (i > 1 && i % 20 == 0) {
            wal.rollWriter();
        }
        LOG.info(i);
        final long txid = wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit, true);
        wal.sync(txid);
    }
    // Simulate a rapid insert that's followed
    // by a report that's still not totally complete (missing last one)
    LOG.info(baseline + " and " + time);
    baseline += 101;
    time = baseline;
    LOG.info(baseline + " and " + time);
    for (int i = 0; i < 3; i++) {
        wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit, true);
    }
    wal.sync();
    int logNumber = 0;
    for (Map.Entry<String, SortedSet<String>> entry : manager.getWALs().get(slaveId).entrySet()) {
        logNumber += entry.getValue().size();
    }
    assertEquals(6, logNumber);
    wal.rollWriter();
    manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(), "1", 0, false, false);
    wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit, true);
    wal.sync();
    assertEquals(1, manager.getWALs().size());
// TODO Need a case with only 2 WALs and we only want to delete the first one
}

Example 32 with WALFactory

use of org.apache.hadoop.hbase.wal.WALFactory in project hbase by apache.

the class TestWALEntryStream method setUp.

@Before
public void setUp() throws Exception {
    walQueue = new PriorityBlockingQueue<>();
    List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
    pathWatcher = new PathWatcher();
    listeners.add(pathWatcher);
    final WALFactory wals = new WALFactory(conf, listeners, tn.getMethodName());
    log = wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace());
}

Example 33 with WALFactory

use of org.apache.hadoop.hbase.wal.WALFactory in project hbase by apache.

the class MetaUtils method getLog.

/**
   * @return the WAL associated with the given region
   * @throws IOException e
   */
public synchronized WAL getLog(HRegionInfo info) throws IOException {
    if (this.walFactory == null) {
        String logName = HConstants.HREGION_LOGDIR_NAME + "_" + System.currentTimeMillis();
        final Configuration walConf = new Configuration(this.conf);
        FSUtils.setRootDir(walConf, fs.getHomeDirectory());
        this.walFactory = new WALFactory(walConf, null, logName);
    }
    final byte[] region = info.getEncodedNameAsBytes();
    final byte[] namespace = info.getTable().getNamespace();
    return info.isMetaRegion() ? walFactory.getMetaWAL(region) : walFactory.getWAL(region, namespace);
}

Example 34 with WALFactory

use of org.apache.hadoop.hbase.wal.WALFactory in project hbase by apache.

the class TestWALObserver method testWALCoprocessorReplay.

/**
   * Test WAL replay behavior with WALObserver.
   */
@Test
public void testWALCoprocessorReplay() throws Exception {
    // WAL replay is handled at HRegion::replayRecoveredEdits(), which is
    // ultimately called by HRegion::initialize()
    final TableName tableName = TableName.valueOf(currentTest.getMethodName());
    final HTableDescriptor htd = getBasic3FamilyHTableDescriptor(tableName);
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    // final HRegionInfo hri =
    // createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
    // final HRegionInfo hri1 =
    // createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
    final HRegionInfo hri = new HRegionInfo(tableName, null, null);
    final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
    deleteDir(basedir);
    fs.mkdirs(new Path(basedir, hri.getEncodedName()));
    final Configuration newConf = HBaseConfiguration.create(this.conf);
    // WAL wal = new WAL(this.fs, this.dir, this.oldLogDir, this.conf);
    WAL wal = wals.getWAL(UNSPECIFIED_REGION, null);
    // Put p = creatPutWith2Families(TEST_ROW);
    WALEdit edit = new WALEdit();
    long now = EnvironmentEdgeManager.currentTime();
    final int countPerFamily = 1000;
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (HColumnDescriptor hcd : htd.getFamilies()) {
        scopes.put(hcd.getName(), 0);
    }
    for (HColumnDescriptor hcd : htd.getFamilies()) {
        addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily, EnvironmentEdgeManager.getDelegate(), wal, scopes, mvcc);
    }
    wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes), edit, true);
    // sync to fs.
    wal.sync();
    User user = HBaseTestingUtility.getDifferentUser(newConf, ".replay.wal.secondtime");
    user.runAs(new PrivilegedExceptionAction<Void>() {

        public Void run() throws Exception {
            Path p = runWALSplit(newConf);
            LOG.info("WALSplit path == " + p);
            FileSystem newFS = FileSystem.get(newConf);
            // Make a new wal for new region open.
            final WALFactory wals2 = new WALFactory(conf, null, ServerName.valueOf(currentTest.getMethodName() + "2", 16010, System.currentTimeMillis()).toString());
            WAL wal2 = wals2.getWAL(UNSPECIFIED_REGION, null);
            ;
            HRegion region = HRegion.openHRegion(newConf, FileSystem.get(newConf), hbaseRootDir, hri, htd, wal2, TEST_UTIL.getHBaseCluster().getRegionServer(0), null);
            long seqid2 = region.getOpenSeqNum();
            SampleRegionWALObserver cp2 = (SampleRegionWALObserver) region.getCoprocessorHost().findCoprocessor(SampleRegionWALObserver.class.getName());
            // TODO: asserting here is problematic.
            assertNotNull(cp2);
            assertTrue(cp2.isPreWALRestoreCalled());
            assertTrue(cp2.isPostWALRestoreCalled());
            region.close();
            wals2.close();
            return null;
        }
    });
}

Example 35 with WALFactory

use of org.apache.hadoop.hbase.wal.WALFactory in project hbase by apache.

the class AbstractTestWALReplay method setUp.

@Before
public void setUp() throws Exception {
    this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
    this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
    this.hbaseRootDir = FSUtils.getRootDir(this.conf);
    this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
    String serverName = ServerName.valueOf(currentTest.getMethodName() + "-manual", 16010, System.currentTimeMillis()).toString();
    this.logName = AbstractFSWALProvider.getWALDirectoryName(serverName);
    this.logDir = new Path(this.hbaseRootDir, logName);
    if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
        TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
    }
    this.mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ? RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING);
    this.wals = new WALFactory(conf, null, currentTest.getMethodName());
}