Examples with ProcedureStore org.apache.hadoop.hbase.procedure2.store.ProcedureStore used on opensource projects

Example 1 with ProcedureStore

use of org.apache.hadoop.hbase.procedure2.store.ProcedureStore in project hbase by apache.

the class HMaster method startProcedureExecutor.

private void startProcedureExecutor() throws IOException {
    final MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
    final Path walDir = new Path(FSUtils.getWALRootDir(this.conf), MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR);
    procedureStore = new WALProcedureStore(conf, walDir.getFileSystem(conf), walDir, new MasterProcedureEnv.WALStoreLeaseRecovery(this));
    procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this));
    procedureExecutor = new ProcedureExecutor(conf, procEnv, procedureStore, procEnv.getProcedureScheduler());
    configurationManager.registerObserver(procEnv);
    final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, Math.max(Runtime.getRuntime().availableProcessors(), MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS));
    final boolean abortOnCorruption = conf.getBoolean(MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION, MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION);
    procedureStore.start(numThreads);
    procedureExecutor.start(numThreads, abortOnCorruption);
}

Example 2 with ProcedureStore

use of org.apache.hadoop.hbase.procedure2.store.ProcedureStore in project hbase by apache.

the class TestMasterProcedureWalLease method testWALfencing.

public void testWALfencing(boolean walRolls) throws IOException {
    final ProcedureStore procStore = getMasterProcedureExecutor().getStore();
    assertTrue("expected WALStore for this test", procStore instanceof WALProcedureStore);
    HMaster firstMaster = UTIL.getHBaseCluster().getMaster();
    // cause WAL rolling after a delete in WAL:
    firstMaster.getConfiguration().setLong(WALProcedureStore.ROLL_THRESHOLD_CONF_KEY, 1);
    HMaster backupMaster3 = Mockito.mock(HMaster.class);
    Mockito.doReturn(firstMaster.getConfiguration()).when(backupMaster3).getConfiguration();
    Mockito.doReturn(true).when(backupMaster3).isActiveMaster();
    final WALProcedureStore procStore2 = new WALProcedureStore(firstMaster.getConfiguration(), firstMaster.getMasterFileSystem().getFileSystem(), ((WALProcedureStore) procStore).getWALDir(), new MasterProcedureEnv.WALStoreLeaseRecovery(backupMaster3));
    // start a second store which should fence the first one out
    LOG.info("Starting new WALProcedureStore");
    procStore2.start(1);
    procStore2.recoverLease();
    // to delete the old WAL files).
    if (walRolls) {
        LOG.info("Inserting into second WALProcedureStore, causing WAL rolls");
        for (int i = 0; i < 512; i++) {
            // insert something to the second store then delete it, causing a WAL roll(s)
            Procedure proc2 = new TestProcedure(i);
            procStore2.insert(proc2, null);
            // delete the procedure so that the WAL is removed later
            procStore2.delete(proc2.getProcId());
        }
    }
    // Now, insert something to the first store, should fail.
    // If the store does a WAL roll and continue with another logId without checking higher logIds
    // it will incorrectly succeed.
    LOG.info("Inserting into first WALProcedureStore");
    try {
        procStore.insert(new TestProcedure(11), null);
        fail("Inserting into Procedure Store should have failed");
    } catch (Exception ex) {
        LOG.info("Received expected exception", ex);
    }
}

Example 3 with ProcedureStore

use of org.apache.hadoop.hbase.procedure2.store.ProcedureStore in project hbase by apache.

the class TestMasterProcedureWalLease method testWalRecoverLease.

@Test
public void testWalRecoverLease() throws Exception {
    final ProcedureStore masterStore = getMasterProcedureExecutor().getStore();
    assertTrue("expected WALStore for this test", masterStore instanceof WALProcedureStore);
    HMaster firstMaster = UTIL.getHBaseCluster().getMaster();
    // Abort Latch for the master store
    final CountDownLatch masterStoreAbort = new CountDownLatch(1);
    masterStore.registerListener(new ProcedureStore.ProcedureStoreListener() {

        @Override
        public void postSync() {
        }

        @Override
        public void abortProcess() {
            LOG.debug("Abort store of Master");
            masterStoreAbort.countDown();
        }
    });
    // startup a fake master the new WAL store will take the lease
    // and the active master should abort.
    HMaster backupMaster3 = Mockito.mock(HMaster.class);
    Mockito.doReturn(firstMaster.getConfiguration()).when(backupMaster3).getConfiguration();
    Mockito.doReturn(true).when(backupMaster3).isActiveMaster();
    final WALProcedureStore backupStore3 = new WALProcedureStore(firstMaster.getConfiguration(), firstMaster.getMasterFileSystem().getFileSystem(), ((WALProcedureStore) masterStore).getWALDir(), new MasterProcedureEnv.WALStoreLeaseRecovery(backupMaster3));
    // Abort Latch for the test store
    final CountDownLatch backupStore3Abort = new CountDownLatch(1);
    backupStore3.registerListener(new ProcedureStore.ProcedureStoreListener() {

        @Override
        public void postSync() {
        }

        @Override
        public void abortProcess() {
            LOG.debug("Abort store of backupMaster3");
            backupStore3Abort.countDown();
            backupStore3.stop(true);
        }
    });
    backupStore3.start(1);
    backupStore3.recoverLease();
    // Try to trigger a command on the master (WAL lease expired on the active one)
    HTableDescriptor htd = MasterProcedureTestingUtility.createHTD(TableName.valueOf(name.getMethodName()), "f");
    HRegionInfo[] regions = ModifyRegionUtils.createHRegionInfos(htd, null);
    LOG.debug("submit proc");
    try {
        getMasterProcedureExecutor().submitProcedure(new CreateTableProcedure(getMasterProcedureExecutor().getEnvironment(), htd, regions));
        fail("expected RuntimeException 'sync aborted'");
    } catch (RuntimeException e) {
        LOG.info("got " + e.getMessage());
    }
    LOG.debug("wait master store abort");
    masterStoreAbort.await();
    // Now the real backup master should start up
    LOG.debug("wait backup master to startup");
    MasterProcedureTestingUtility.waitBackupMaster(UTIL, firstMaster);
    assertEquals(true, firstMaster.isStopped());
    // wait the store in here to abort (the test will fail due to timeout if it doesn't)
    LOG.debug("wait the store to abort");
    backupStore3.getStoreTracker().setDeleted(1, false);
    try {
        backupStore3.delete(1);
        fail("expected RuntimeException 'sync aborted'");
    } catch (RuntimeException e) {
        LOG.info("got " + e.getMessage());
    }
    backupStore3Abort.await();
}

Example 4 with ProcedureStore

use of org.apache.hadoop.hbase.procedure2.store.ProcedureStore in project hbase by apache.

the class ProcedureTestingUtility method restart.

public static <TEnv> void restart(final ProcedureExecutor<TEnv> procExecutor, final boolean avoidTestKillDuringRestart, final boolean failOnCorrupted, final Callable<Void> stopAction, final Callable<Void> startAction) throws Exception {
    final ProcedureStore procStore = procExecutor.getStore();
    final int storeThreads = procExecutor.getCorePoolSize();
    final int execThreads = procExecutor.getCorePoolSize();
    final ProcedureExecutor.Testing testing = procExecutor.testing;
    if (avoidTestKillDuringRestart) {
        procExecutor.testing = null;
    }
    // stop
    LOG.info("RESTART - Stop");
    procExecutor.stop();
    procStore.stop(false);
    if (stopAction != null) {
        stopAction.call();
    }
    procExecutor.join();
    procExecutor.getScheduler().clear();
    // nothing running...
    // re-start
    LOG.info("RESTART - Start");
    procStore.start(storeThreads);
    procExecutor.start(execThreads, failOnCorrupted);
    if (startAction != null) {
        startAction.call();
    }
    if (avoidTestKillDuringRestart) {
        procExecutor.testing = testing;
    }
}