Examples with Procedure org.apache.hadoop.hbase.procedure2.Procedure used on opensource projects

Example 46 with Procedure

use of org.apache.hadoop.hbase.procedure2.Procedure in project hbase by apache.

the class ProcedureTestingUtility method submitAndWait.

public static <TEnv> long submitAndWait(Configuration conf, TEnv env, Procedure<TEnv> proc) throws IOException {
    NoopProcedureStore procStore = new NoopProcedureStore();
    ProcedureExecutor<TEnv> procExecutor = new ProcedureExecutor<>(conf, env, procStore);
    procStore.start(1);
    procExecutor.start(1, false);
    try {
        return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);
    } finally {
        procStore.stop(false);
        procExecutor.stop();
    }
}

Example 47 with Procedure

use of org.apache.hadoop.hbase.procedure2.Procedure in project hbase by apache.

the class TestWALProcedureStore method testCorruptedTrailersRebuild.

@Test
public void testCorruptedTrailersRebuild() throws Exception {
    final Procedure[] procs = new Procedure[6];
    for (int i = 0; i < procs.length; ++i) {
        procs[i] = new TestSequentialProcedure();
    }
    // Log State (I=insert, U=updated, D=delete)
    //   | log 1 | log 2 | log 3 |
    // 0 | I, D  |       |       |
    // 1 | I     |       |       |
    // 2 | I     | D     |       |
    // 3 | I     | U     |       |
    // 4 |       | I     | D     |
    // 5 |       |       | I     |
    procStore.insert(procs[0], null);
    procStore.insert(procs[1], null);
    procStore.insert(procs[2], null);
    procStore.insert(procs[3], null);
    procStore.delete(procs[0].getProcId());
    procStore.rollWriterForTesting();
    procStore.delete(procs[2].getProcId());
    procStore.update(procs[3]);
    procStore.insert(procs[4], null);
    procStore.rollWriterForTesting();
    procStore.delete(procs[4].getProcId());
    procStore.insert(procs[5], null);
    // Stop the store
    procStore.stop(false);
    // Remove 4 byte from the trailers
    final FileStatus[] logs = fs.listStatus(logDir);
    assertEquals(3, logs.length);
    for (int i = 0; i < logs.length; ++i) {
        corruptLog(logs[i], 4);
    }
    // Restart the store (avoid cleaning up the files, to check the rebuilded trackers)
    htu.getConfiguration().setBoolean(WALProcedureStore.EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY, false);
    final LoadCounter loader = new LoadCounter();
    storeRestart(loader);
    // procs 1, 3 and 5
    assertEquals(3, loader.getLoadedCount());
    assertEquals(0, loader.getCorruptedCount());
    // Check the Trackers
    final ArrayList<ProcedureWALFile> walFiles = procStore.getActiveLogs();
    LOG.info("WALs " + walFiles);
    assertEquals(4, walFiles.size());
    LOG.info("Checking wal " + walFiles.get(0));
    assertUpdated(walFiles.get(0).getTracker(), procs, new int[] { 0, 1, 2, 3 }, new int[] { 4, 5 });
    LOG.info("Checking wal " + walFiles.get(1));
    assertUpdated(walFiles.get(1).getTracker(), procs, new int[] { 2, 3, 4 }, new int[] { 0, 1, 5 });
    LOG.info("Checking wal " + walFiles.get(2));
    assertUpdated(walFiles.get(2).getTracker(), procs, new int[] { 4, 5 }, new int[] { 0, 1, 2, 3 });
    LOG.info("Checking global tracker ");
    assertDeleted(procStore.getStoreTracker(), procs, new int[] { 0, 2, 4 }, new int[] { 1, 3, 5 });
}

Example 48 with Procedure

use of org.apache.hadoop.hbase.procedure2.Procedure in project hbase by apache.

the class TestWALProcedureStore method testCorruptedProcedures.

@Test
public void testCorruptedProcedures() throws Exception {
    // Insert root-procedures
    TestProcedure[] rootProcs = new TestProcedure[10];
    for (int i = 1; i <= rootProcs.length; i++) {
        rootProcs[i - 1] = new TestProcedure(i, 0);
        procStore.insert(rootProcs[i - 1], null);
        rootProcs[i - 1].addStackId(0);
        procStore.update(rootProcs[i - 1]);
    }
    // insert root-child txn
    procStore.rollWriterForTesting();
    for (int i = 1; i <= rootProcs.length; i++) {
        TestProcedure b = new TestProcedure(rootProcs.length + i, i);
        rootProcs[i - 1].addStackId(1);
        procStore.insert(rootProcs[i - 1], new Procedure[] { b });
    }
    // insert child updates
    procStore.rollWriterForTesting();
    for (int i = 1; i <= rootProcs.length; i++) {
        procStore.update(new TestProcedure(rootProcs.length + i, i));
    }
    // Stop the store
    procStore.stop(false);
    // the first log was removed,
    // we have insert-txn and updates in the others so everything is fine
    FileStatus[] logs = fs.listStatus(logDir);
    assertEquals(Arrays.toString(logs), 2, logs.length);
    Arrays.sort(logs, new Comparator<FileStatus>() {

        @Override
        public int compare(FileStatus o1, FileStatus o2) {
            return o1.getPath().getName().compareTo(o2.getPath().getName());
        }
    });
    LoadCounter loader = new LoadCounter();
    storeRestart(loader);
    assertEquals(rootProcs.length * 2, loader.getLoadedCount());
    assertEquals(0, loader.getCorruptedCount());
    // Remove the second log, we have lost all the root/parent references
    fs.delete(logs[0].getPath(), false);
    loader.reset();
    storeRestart(loader);
    assertEquals(0, loader.getLoadedCount());
    assertEquals(rootProcs.length, loader.getCorruptedCount());
    for (Procedure proc : loader.getCorrupted()) {
        assertTrue(proc.toString(), proc.getParentProcId() <= rootProcs.length);
        assertTrue(proc.toString(), proc.getProcId() > rootProcs.length && proc.getProcId() <= (rootProcs.length * 2));
    }
}

Example 49 with Procedure

use of org.apache.hadoop.hbase.procedure2.Procedure in project hbase by apache.

the class TestWALProcedureStore method testLoad.

@Test
public void testLoad() throws Exception {
    Set<Long> procIds = new HashSet<>();
    // Insert something in the log
    Procedure proc1 = new TestSequentialProcedure();
    procIds.add(proc1.getProcId());
    procStore.insert(proc1, null);
    Procedure proc2 = new TestSequentialProcedure();
    Procedure[] child2 = new Procedure[2];
    child2[0] = new TestSequentialProcedure();
    child2[1] = new TestSequentialProcedure();
    procIds.add(proc2.getProcId());
    procIds.add(child2[0].getProcId());
    procIds.add(child2[1].getProcId());
    procStore.insert(proc2, child2);
    // Verify that everything is there
    verifyProcIdsOnRestart(procIds);
    // Update and delete something
    procStore.update(proc1);
    procStore.update(child2[1]);
    procStore.delete(child2[1].getProcId());
    procIds.remove(child2[1].getProcId());
    // Verify that everything is there
    verifyProcIdsOnRestart(procIds);
    // Remove 4 byte from the trailers
    procStore.stop(false);
    FileStatus[] logs = fs.listStatus(logDir);
    assertEquals(3, logs.length);
    for (int i = 0; i < logs.length; ++i) {
        corruptLog(logs[i], 4);
    }
    verifyProcIdsOnRestart(procIds);
}

Example 50 with Procedure

use of org.apache.hadoop.hbase.procedure2.Procedure in project hbase by apache.

the class TestWALProcedureStore method testNoTrailerDoubleRestart.

@Test
public void testNoTrailerDoubleRestart() throws Exception {
    // log-0001: proc 0, 1 and 2 are inserted
    Procedure proc0 = new TestSequentialProcedure();
    procStore.insert(proc0, null);
    Procedure proc1 = new TestSequentialProcedure();
    procStore.insert(proc1, null);
    Procedure proc2 = new TestSequentialProcedure();
    procStore.insert(proc2, null);
    procStore.rollWriterForTesting();
    // log-0002: proc 1 deleted
    procStore.delete(proc1.getProcId());
    procStore.rollWriterForTesting();
    // log-0003: proc 2 is update
    procStore.update(proc2);
    procStore.rollWriterForTesting();
    // log-0004: proc 2 deleted
    procStore.delete(proc2.getProcId());
    // stop the store and remove the trailer
    procStore.stop(false);
    FileStatus[] logs = fs.listStatus(logDir);
    assertEquals(4, logs.length);
    for (int i = 0; i < logs.length; ++i) {
        corruptLog(logs[i], 4);
    }
    // Test Load 1
    // Restart the store (avoid cleaning up the files, to check the rebuilded trackers)
    htu.getConfiguration().setBoolean(WALProcedureStore.EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY, false);
    LoadCounter loader = new LoadCounter();
    storeRestart(loader);
    assertEquals(1, loader.getLoadedCount());
    assertEquals(0, loader.getCorruptedCount());
    // Test Load 2
    assertEquals(5, fs.listStatus(logDir).length);
    loader = new LoadCounter();
    storeRestart(loader);
    assertEquals(1, loader.getLoadedCount());
    assertEquals(0, loader.getCorruptedCount());
    // remove proc-0
    procStore.delete(proc0.getProcId());
    procStore.periodicRollForTesting();
    assertEquals(1, fs.listStatus(logDir).length);
    storeRestart(loader);
}