Examples with WALProcedureStore org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore used on opensource projects

Example 1 with WALProcedureStore

use of org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore in project hbase by apache.

the class TestOpenRegionProcedureHang method test.

@Test
public void test() throws InterruptedException, KeeperException, IOException {
    RegionInfo region = UTIL.getMiniHBaseCluster().getRegions(NAME).get(0).getRegionInfo();
    AssignmentManager am = UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager();
    HRegionServer rs1 = UTIL.getRSForFirstRegionInTable(NAME);
    HRegionServer rs2 = UTIL.getOtherRegionServer(rs1);
    ARRIVE = new CountDownLatch(1);
    RESUME = new CountDownLatch(1);
    FINISH = new CountDownLatch(1);
    ABORT = new CountDownLatch(1);
    am.moveAsync(new RegionPlan(region, rs1.getServerName(), rs2.getServerName()));
    ARRIVE.await();
    ARRIVE = null;
    HMaster master = UTIL.getMiniHBaseCluster().getMaster();
    master.getZooKeeper().close();
    UTIL.waitFor(30000, () -> {
        for (MasterThread mt : UTIL.getMiniHBaseCluster().getMasterThreads()) {
            if (mt.getMaster() != master && mt.getMaster().isActiveMaster()) {
                return mt.getMaster().isInitialized();
            }
        }
        return false;
    });
    ProcedureExecutor<MasterProcedureEnv> procExec = UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
    UTIL.waitFor(30000, () -> procExec.getProcedures().stream().filter(p -> p instanceof OpenRegionProcedure).map(p -> (OpenRegionProcedure) p).anyMatch(p -> p.region.getTable().equals(NAME)));
    OpenRegionProcedure proc = procExec.getProcedures().stream().filter(p -> p instanceof OpenRegionProcedure).map(p -> (OpenRegionProcedure) p).filter(p -> p.region.getTable().equals(NAME)).findFirst().get();
    // wait a bit to let the OpenRegionProcedure send out the request
    Thread.sleep(2000);
    RESUME.countDown();
    if (!FINISH.await(15, TimeUnit.SECONDS)) {
        LOG.info("Wait reportRegionStateTransition to finish timed out, this is possible if" + " we update the procedure store, as the WALProcedureStore" + " will retry forever to roll the writer if it is not closed");
    }
    FINISH = null;
    // if the reportRegionTransition is finished, wait a bit to let it return the data to RS
    Thread.sleep(2000);
    ABORT.countDown();
    UTIL.waitFor(30000, () -> procExec.isFinished(proc.getProcId()));
    UTIL.waitFor(30000, () -> procExec.isFinished(proc.getParentProcId()));
}

Example 2 with WALProcedureStore

use of org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore in project hbase by apache.

the class TestRegionProcedureStoreMigration method setUp.

@Before
public void setUp() throws IOException {
    htu = new HBaseCommonTestingUtil();
    Configuration conf = htu.getConfiguration();
    conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, false);
    // Runs on local filesystem. Test does not need sync. Turn off checks.
    conf.setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, false);
    Path testDir = htu.getDataTestDir();
    CommonFSUtils.setRootDir(conf, testDir);
    walStore = new WALProcedureStore(conf, new LeaseRecovery() {

        @Override
        public void recoverFileLease(FileSystem fs, Path path) throws IOException {
        }
    });
    walStore.start(1);
    walStore.recoverLease();
    walStore.load(new LoadCounter());
    server = RegionProcedureStoreTestHelper.mockServer(conf);
    region = MasterRegionFactory.create(server);
}

Example 3 with WALProcedureStore

use of org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore in project hbase by apache.

the class RegionProcedureStore method tryMigrate.

@SuppressWarnings("deprecation")
private void tryMigrate(FileSystem fs) throws IOException {
    Configuration conf = server.getConfiguration();
    Path procWALDir = new Path(CommonFSUtils.getWALRootDir(conf), WALProcedureStore.MASTER_PROCEDURE_LOGDIR);
    if (!fs.exists(procWALDir)) {
        return;
    }
    LOG.info("The old WALProcedureStore wal directory {} exists, migrating...", procWALDir);
    WALProcedureStore store = new WALProcedureStore(conf, leaseRecovery);
    store.start(numThreads);
    store.recoverLease();
    MutableLong maxProcIdSet = new MutableLong(-1);
    List<Procedure<?>> procs = new ArrayList<>();
    Map<Class<?>, List<Procedure<?>>> activeProcsByType = new HashMap<>();
    store.load(new ProcedureLoader() {

        @Override
        public void setMaxProcId(long maxProcId) {
            maxProcIdSet.setValue(maxProcId);
        }

        @Override
        public void load(ProcedureIterator procIter) throws IOException {
            while (procIter.hasNext()) {
                Procedure<?> proc = procIter.next();
                procs.add(proc);
                if (!proc.isFinished()) {
                    activeProcsByType.computeIfAbsent(proc.getClass(), k -> new ArrayList<>()).add(proc);
                }
            }
        }

        @Override
        public void handleCorrupted(ProcedureIterator procIter) throws IOException {
            long corruptedCount = 0;
            while (procIter.hasNext()) {
                LOG.error("Corrupted procedure {}", procIter.next());
                corruptedCount++;
            }
            if (corruptedCount > 0) {
                throw new IOException("There are " + corruptedCount + " corrupted procedures when" + " migrating from the old WAL based store to the new region based store, please" + " fix them before upgrading again.");
            }
        }
    });
    // check whether there are unsupported procedures, this could happen when we are migrating from
    // 2.1-. We used to do this in HMaster, after loading all the procedures from procedure store,
    // but here we have to do it before migrating, otherwise, if we find some unsupported
    // procedures, the users can not go back to 2.1 to finish them any more, as all the data are now
    // in the new region based procedure store, which is not supported in 2.1-.
    checkUnsupportedProcedure(activeProcsByType);
    MutableLong maxProcIdFromProcs = new MutableLong(-1);
    for (Procedure<?> proc : procs) {
        update(proc);
        if (proc.getProcId() > maxProcIdFromProcs.longValue()) {
            maxProcIdFromProcs.setValue(proc.getProcId());
        }
    }
    LOG.info("Migrated {} existing procedures from the old storage format.", procs.size());
    LOG.info("The WALProcedureStore max pid is {}, and the max pid of all loaded procedures is {}", maxProcIdSet.longValue(), maxProcIdFromProcs.longValue());
    // anyway, let's do a check here.
    if (maxProcIdSet.longValue() > maxProcIdFromProcs.longValue()) {
        if (maxProcIdSet.longValue() > 0) {
            // let's add a fake row to retain the max proc id
            region.update(r -> r.put(new Put(Bytes.toBytes(maxProcIdSet.longValue())).addColumn(PROC_FAMILY, PROC_QUALIFIER, EMPTY_BYTE_ARRAY)));
        }
    } else if (maxProcIdSet.longValue() < maxProcIdFromProcs.longValue()) {
        LOG.warn("The WALProcedureStore max pid is less than the max pid of all loaded procedures");
    }
    store.stop(false);
    if (!fs.delete(procWALDir, true)) {
        throw new IOException("Failed to delete the WALProcedureStore migrated proc wal directory " + procWALDir);
    }
    LOG.info("Migration of WALProcedureStore finished");
}

Example 4 with WALProcedureStore

use of org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore in project hbase by apache.

the class TestWALProcedureStore method testFileNotFoundDuringLeaseRecovery.

@Test
public void testFileNotFoundDuringLeaseRecovery() throws IOException {
    final TestProcedure[] procs = new TestProcedure[3];
    for (int i = 0; i < procs.length; ++i) {
        procs[i] = new TestProcedure(i + 1, 0);
        procStore.insert(procs[i], null);
    }
    procStore.rollWriterForTesting();
    for (int i = 0; i < procs.length; ++i) {
        procStore.update(procs[i]);
        procStore.rollWriterForTesting();
    }
    procStore.stop(false);
    FileStatus[] status = fs.listStatus(logDir);
    assertEquals(procs.length + 1, status.length);
    // simulate another active master removing the wals
    procStore = new WALProcedureStore(htu.getConfiguration(), logDir, null, new LeaseRecovery() {

        private int count = 0;

        @Override
        public void recoverFileLease(FileSystem fs, Path path) throws IOException {
            if (++count <= 2) {
                fs.delete(path, false);
                LOG.debug("Simulate FileNotFound at count=" + count + " for " + path);
                throw new FileNotFoundException("test file not found " + path);
            }
            LOG.debug("Simulate recoverFileLease() at count=" + count + " for " + path);
        }
    });
    final LoadCounter loader = new LoadCounter();
    procStore.start(PROCEDURE_STORE_SLOTS);
    procStore.recoverLease();
    procStore.load(loader);
    assertEquals(procs.length, loader.getMaxProcId());
    assertEquals(1, loader.getRunnableCount());
    assertEquals(0, loader.getCompletedCount());
    assertEquals(0, loader.getCorruptedCount());
}

Example 5 with WALProcedureStore

use of org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore in project hbase by apache.

the class WALProcedureStore method main.

/**
 * Parses a directory of WALs building up ProcedureState.
 * For testing parse and profiling.
 * @param args Include pointer to directory of WAL files for a store instance to parse & load.
 */
public static void main(String[] args) throws IOException {
    Configuration conf = HBaseConfiguration.create();
    if (args == null || args.length != 1) {
        System.out.println("ERROR: Empty arguments list; pass path to MASTERPROCWALS_DIR.");
        System.out.println("Usage: WALProcedureStore MASTERPROCWALS_DIR");
        System.exit(-1);
    }
    WALProcedureStore store = new WALProcedureStore(conf, new Path(args[0]), null, new LeaseRecovery() {

        @Override
        public void recoverFileLease(FileSystem fs, Path path) throws IOException {
        // no-op
        }
    });
    try {
        store.start(16);
        ProcedureExecutor<?> pe = new ProcedureExecutor<>(conf, new Object(), /*Pass anything*/
        store);
        pe.init(1, true);
    } finally {
        store.stop(true);
    }
}