Examples with AbstractFSWAL org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL used on opensource projects

Example 6 with AbstractFSWAL

use of org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL in project hbase by apache.

the class TestSequenceIdMonotonicallyIncreasing method getMaxSeqId.

private long getMaxSeqId(HRegionServer rs, RegionInfo region) throws IOException {
    Path walFile = ((AbstractFSWAL<?>) rs.getWAL(null)).getCurrentFileName();
    long maxSeqId = -1L;
    try (WAL.Reader reader = WALFactory.createReader(UTIL.getTestFileSystem(), walFile, UTIL.getConfiguration())) {
        for (; ; ) {
            WAL.Entry entry = reader.next();
            if (entry == null) {
                break;
            }
            if (Bytes.equals(region.getEncodedNameAsBytes(), entry.getKey().getEncodedRegionName())) {
                maxSeqId = Math.max(maxSeqId, entry.getKey().getSequenceId());
            }
        }
    }
    return maxSeqId;
}

Example 7 with AbstractFSWAL

use of org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL in project hbase by apache.

the class TestMasterRegionOnTwoFileSystems method testFlushAndCompact.

@Test
public void testFlushAndCompact() throws Exception {
    int compactMinMinusOne = COMPACT_MIN - 1;
    for (int i = 0; i < compactMinMinusOne; i++) {
        final int index = i;
        region.update(r -> r.put(new Put(Bytes.toBytes(index)).addColumn(CF, CQ, Bytes.toBytes(index))));
        region.flush(true);
    }
    byte[] bytes = Bytes.toBytes(compactMinMinusOne);
    region.update(r -> r.put(new Put(bytes).addColumn(CF, CQ, bytes)));
    region.flusherAndCompactor.requestFlush();
    HFILE_UTIL.waitFor(15000, () -> getStorefilesCount() == 1);
    // make sure the archived hfiles are on the root fs
    Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePathForRootDir(HFILE_UTIL.getDataTestDir(), region.region.getRegionInfo(), CF);
    FileSystem rootFs = storeArchiveDir.getFileSystem(HFILE_UTIL.getConfiguration());
    HFILE_UTIL.waitFor(15000, () -> {
        try {
            FileStatus[] fses = rootFs.listStatus(storeArchiveDir);
            return fses != null && fses.length == COMPACT_MIN;
        } catch (FileNotFoundException e) {
            return false;
        }
    });
    LOG.info("hfile archive content {}", Arrays.stream(rootFs.listStatus(storeArchiveDir)).map(f -> f.getPath().toString()).collect(Collectors.joining(",")));
    // make sure the archived wal files are on the wal fs
    Path walArchiveDir = new Path(CommonFSUtils.getWALRootDir(HFILE_UTIL.getConfiguration()), HConstants.HREGION_OLDLOGDIR_NAME);
    LOG.info("wal archive dir {}", walArchiveDir);
    AbstractFSWAL<?> wal = (AbstractFSWAL<?>) region.region.getWAL();
    Path currentWALFile = wal.getCurrentFileName();
    for (; ; ) {
        region.requestRollAll();
        region.waitUntilWalRollFinished();
        Path newWALFile = wal.getCurrentFileName();
        // make sure we actually rolled the wal
        if (!newWALFile.equals(currentWALFile)) {
            break;
        }
    }
    HFILE_UTIL.waitFor(15000, () -> {
        try {
            FileStatus[] fses = WAL_UTIL.getTestFileSystem().listStatus(walArchiveDir);
            if (fses != null && fses.length > 0) {
                LOG.info("wal archive dir content {}", Arrays.stream(fses).map(f -> f.getPath().toString()).collect(Collectors.joining(",")));
            } else {
                LOG.info("none found");
            }
            return fses != null && fses.length >= 1;
        } catch (FileNotFoundException e) {
            return false;
        }
    });
}

Example 8 with AbstractFSWAL

use of org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL in project hbase by apache.

the class TestAddToSerialReplicationPeer method waitUntilReplicatedToTheCurrentWALFile.

private void waitUntilReplicatedToTheCurrentWALFile(HRegionServer rs, final String oldWalName) throws Exception {
    Path path = ((AbstractFSWAL<?>) rs.getWAL(null)).getCurrentFileName();
    String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(path.getName());
    UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            ReplicationSourceManager manager = ((Replication) rs.getReplicationSourceService()).getReplicationManager();
            // Make sure replication moves to the new file.
            return (manager.getWALs().get(PEER_ID).get(logPrefix).size() == 1) && !oldWalName.equals(manager.getWALs().get(PEER_ID).get(logPrefix).first());
        }

        @Override
        public String explainFailure() throws Exception {
            return "Still not replicated to the current WAL file yet";
        }
    });
}

Example 9 with AbstractFSWAL

use of org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL in project hbase by apache.

the class TestWALMonotonicallyIncreasingSeqId method testWALMonotonicallyIncreasingSeqId.

@Test
public void testWALMonotonicallyIncreasingSeqId() throws Exception {
    List<Thread> putThreads = new ArrayList<>();
    for (int i = 0; i < 1; i++) {
        putThreads.add(new PutThread(region));
    }
    IncThread incThread = new IncThread(region);
    for (int i = 0; i < 1; i++) {
        putThreads.get(i).start();
    }
    incThread.start();
    incThread.join();
    Path logPath = ((AbstractFSWAL<?>) region.getWAL()).getCurrentFileName();
    region.getWAL().rollWriter();
    Thread.sleep(10);
    Path hbaseDir = new Path(walConf.get(HConstants.HBASE_DIR));
    Path oldWalsDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME);
    try (WAL.Reader reader = createReader(logPath, oldWalsDir)) {
        long currentMaxSeqid = 0;
        for (WAL.Entry e; (e = reader.next()) != null; ) {
            if (!WALEdit.isMetaEditFamily(e.getEdit().getCells().get(0))) {
                long currentSeqid = e.getKey().getSequenceId();
                if (currentSeqid > currentMaxSeqid) {
                    currentMaxSeqid = currentSeqid;
                } else {
                    fail("Current max Seqid is " + currentMaxSeqid + ", but the next seqid in wal is smaller:" + currentSeqid);
                }
            }
        }
    }
}