Examples with Procedure org.apache.hadoop.hbase.procedure2.Procedure used on opensource projects

Example 36 with Procedure

use of org.apache.hadoop.hbase.procedure2.Procedure in project hbase by apache.

the class TestMasterProcedureScheduler method testVerifyRwLocks.

/**
   * Verify the correct logic of RWLocks on the queue
   */
@Test
public void testVerifyRwLocks() throws Exception {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    queue.addBack(new TestTableProcedure(1, tableName, TableProcedureInterface.TableOperationType.EDIT));
    queue.addBack(new TestTableProcedure(2, tableName, TableProcedureInterface.TableOperationType.READ));
    queue.addBack(new TestTableProcedure(3, tableName, TableProcedureInterface.TableOperationType.EDIT));
    queue.addBack(new TestTableProcedure(4, tableName, TableProcedureInterface.TableOperationType.READ));
    queue.addBack(new TestTableProcedure(5, tableName, TableProcedureInterface.TableOperationType.READ));
    // Fetch the 1st item and take the write lock
    Procedure proc = queue.poll();
    assertEquals(1, proc.getProcId());
    assertEquals(false, queue.waitTableExclusiveLock(proc, tableName));
    // Fetch the 2nd item and verify that the lock can't be acquired
    assertEquals(null, queue.poll(0));
    // Release the write lock and acquire the read lock
    queue.wakeTableExclusiveLock(proc, tableName);
    // Fetch the 2nd item and take the read lock
    Procedure rdProc = queue.poll();
    assertEquals(2, rdProc.getProcId());
    assertEquals(false, queue.waitTableSharedLock(rdProc, tableName));
    // Fetch the 3rd item and verify that the lock can't be acquired
    assertEquals(null, queue.poll(0));
    // release the rdlock of item 2 and take the wrlock for the 3d item
    queue.wakeTableSharedLock(rdProc, tableName);
    // Fetch the 3rd item and take the write lock
    Procedure wrProc = queue.poll();
    assertEquals(false, queue.waitTableExclusiveLock(wrProc, tableName));
    // Fetch 4th item and verify that the lock can't be acquired
    assertEquals(null, queue.poll(0));
    // Release the write lock and acquire the read lock
    queue.wakeTableExclusiveLock(wrProc, tableName);
    // Fetch the 4th item and take the read lock
    rdProc = queue.poll();
    assertEquals(4, rdProc.getProcId());
    assertEquals(false, queue.waitTableSharedLock(rdProc, tableName));
    // Fetch the 4th item and take the read lock
    Procedure rdProc2 = queue.poll();
    assertEquals(5, rdProc2.getProcId());
    assertEquals(false, queue.waitTableSharedLock(rdProc2, tableName));
    // Release 4th and 5th read-lock
    queue.wakeTableSharedLock(rdProc, tableName);
    queue.wakeTableSharedLock(rdProc2, tableName);
    // remove table queue
    assertEquals(0, queue.size());
    assertTrue("queue should be deleted", queue.markTableAsDeleted(tableName, wrProc));
}

Example 37 with Procedure

use of org.apache.hadoop.hbase.procedure2.Procedure in project hbase by apache.

the class TestMasterProcedureSchedulerConcurrency method testConcurrentWriteOps.

/**
   * Verify that "write" operations for a single table are serialized,
   * but different tables can be executed in parallel.
   */
@Test(timeout = 60000)
public void testConcurrentWriteOps() throws Exception {
    final TestTableProcSet procSet = new TestTableProcSet(queue);
    final int NUM_ITEMS = 10;
    final int NUM_TABLES = 4;
    final AtomicInteger opsCount = new AtomicInteger(0);
    for (int i = 0; i < NUM_TABLES; ++i) {
        TableName tableName = TableName.valueOf(String.format("testtb-%04d", i));
        for (int j = 1; j < NUM_ITEMS; ++j) {
            procSet.addBack(new TestTableProcedure(i * 100 + j, tableName, TableProcedureInterface.TableOperationType.EDIT));
            opsCount.incrementAndGet();
        }
    }
    assertEquals(opsCount.get(), queue.size());
    final Thread[] threads = new Thread[NUM_TABLES * 2];
    final HashSet<TableName> concurrentTables = new HashSet<>();
    final ArrayList<String> failures = new ArrayList<>();
    final AtomicInteger concurrentCount = new AtomicInteger(0);
    for (int i = 0; i < threads.length; ++i) {
        threads[i] = new Thread() {

            @Override
            public void run() {
                while (opsCount.get() > 0) {
                    try {
                        Procedure proc = procSet.acquire();
                        if (proc == null) {
                            queue.signalAll();
                            if (opsCount.get() > 0) {
                                continue;
                            }
                            break;
                        }
                        TableName tableId = procSet.getTableName(proc);
                        synchronized (concurrentTables) {
                            assertTrue("unexpected concurrency on " + tableId, concurrentTables.add(tableId));
                        }
                        assertTrue(opsCount.decrementAndGet() >= 0);
                        try {
                            long procId = proc.getProcId();
                            int concurrent = concurrentCount.incrementAndGet();
                            assertTrue("inc-concurrent=" + concurrent + " 1 <= concurrent <= " + NUM_TABLES, concurrent >= 1 && concurrent <= NUM_TABLES);
                            LOG.debug("[S] tableId=" + tableId + " procId=" + procId + " concurrent=" + concurrent);
                            Thread.sleep(2000);
                            concurrent = concurrentCount.decrementAndGet();
                            LOG.debug("[E] tableId=" + tableId + " procId=" + procId + " concurrent=" + concurrent);
                            assertTrue("dec-concurrent=" + concurrent, concurrent < NUM_TABLES);
                        } finally {
                            synchronized (concurrentTables) {
                                assertTrue(concurrentTables.remove(tableId));
                            }
                            procSet.release(proc);
                        }
                    } catch (Throwable e) {
                        LOG.error("Failed " + e.getMessage(), e);
                        synchronized (failures) {
                            failures.add(e.getMessage());
                        }
                    } finally {
                        queue.signalAll();
                    }
                }
            }
        };
        threads[i].start();
    }
    for (int i = 0; i < threads.length; ++i) {
        threads[i].join();
    }
    assertTrue(failures.toString(), failures.isEmpty());
    assertEquals(0, opsCount.get());
    assertEquals(0, queue.size());
    for (int i = 1; i <= NUM_TABLES; ++i) {
        final TableName table = TableName.valueOf(String.format("testtb-%04d", i));
        final TestTableProcedure dummyProc = new TestTableProcedure(100, table, TableProcedureInterface.TableOperationType.DELETE);
        assertTrue("queue should be deleted, table=" + table, queue.markTableAsDeleted(table, dummyProc));
    }
}

Example 38 with Procedure

use of org.apache.hadoop.hbase.procedure2.Procedure in project hbase by apache.

the class TestMasterProcedureWalLease method testWALfencing.

public void testWALfencing(boolean walRolls) throws IOException {
    final ProcedureStore procStore = getMasterProcedureExecutor().getStore();
    assertTrue("expected WALStore for this test", procStore instanceof WALProcedureStore);
    HMaster firstMaster = UTIL.getHBaseCluster().getMaster();
    // cause WAL rolling after a delete in WAL:
    firstMaster.getConfiguration().setLong(WALProcedureStore.ROLL_THRESHOLD_CONF_KEY, 1);
    HMaster backupMaster3 = Mockito.mock(HMaster.class);
    Mockito.doReturn(firstMaster.getConfiguration()).when(backupMaster3).getConfiguration();
    Mockito.doReturn(true).when(backupMaster3).isActiveMaster();
    final WALProcedureStore procStore2 = new WALProcedureStore(firstMaster.getConfiguration(), firstMaster.getMasterFileSystem().getFileSystem(), ((WALProcedureStore) procStore).getWALDir(), new MasterProcedureEnv.WALStoreLeaseRecovery(backupMaster3));
    // start a second store which should fence the first one out
    LOG.info("Starting new WALProcedureStore");
    procStore2.start(1);
    procStore2.recoverLease();
    // to delete the old WAL files).
    if (walRolls) {
        LOG.info("Inserting into second WALProcedureStore, causing WAL rolls");
        for (int i = 0; i < 512; i++) {
            // insert something to the second store then delete it, causing a WAL roll(s)
            Procedure proc2 = new TestProcedure(i);
            procStore2.insert(proc2, null);
            // delete the procedure so that the WAL is removed later
            procStore2.delete(proc2.getProcId());
        }
    }
    // Now, insert something to the first store, should fail.
    // If the store does a WAL roll and continue with another logId without checking higher logIds
    // it will incorrectly succeed.
    LOG.info("Inserting into first WALProcedureStore");
    try {
        procStore.insert(new TestProcedure(11), null);
        fail("Inserting into Procedure Store should have failed");
    } catch (Exception ex) {
        LOG.info("Received expected exception", ex);
    }
}

Example 39 with Procedure

use of org.apache.hadoop.hbase.procedure2.Procedure in project hbase by apache.

the class TestMasterProcedureScheduler method testVerifyNamespaceRwLocks.

@Test
public void testVerifyNamespaceRwLocks() throws Exception {
    String nsName1 = "ns1";
    String nsName2 = "ns2";
    TableName tableName1 = TableName.valueOf(nsName1, name.getMethodName());
    TableName tableName2 = TableName.valueOf(nsName2, name.getMethodName());
    queue.addBack(new TestNamespaceProcedure(1, nsName1, TableProcedureInterface.TableOperationType.EDIT));
    queue.addBack(new TestTableProcedure(2, tableName1, TableProcedureInterface.TableOperationType.EDIT));
    queue.addBack(new TestTableProcedure(3, tableName2, TableProcedureInterface.TableOperationType.EDIT));
    queue.addBack(new TestNamespaceProcedure(4, nsName2, TableProcedureInterface.TableOperationType.EDIT));
    // Fetch the 1st item and take the write lock
    Procedure procNs1 = queue.poll();
    assertEquals(1, procNs1.getProcId());
    assertEquals(false, queue.waitNamespaceExclusiveLock(procNs1, nsName1));
    // System tables have 2 as default priority
    Procedure procNs2 = queue.poll();
    assertEquals(4, procNs2.getProcId());
    assertEquals(false, queue.waitNamespaceExclusiveLock(procNs2, nsName2));
    queue.wakeNamespaceExclusiveLock(procNs2, nsName2);
    // add procNs2 back in the queue
    queue.yield(procNs2);
    // table on ns1 is locked, so we get table on ns2
    procNs2 = queue.poll();
    assertEquals(3, procNs2.getProcId());
    assertEquals(false, queue.waitTableExclusiveLock(procNs2, tableName2));
    // ns2 is not available (TODO we may avoid this one)
    Procedure procNs2b = queue.poll();
    assertEquals(4, procNs2b.getProcId());
    assertEquals(true, queue.waitNamespaceExclusiveLock(procNs2b, nsName2));
    // release the ns1 lock
    queue.wakeNamespaceExclusiveLock(procNs1, nsName1);
    // we are now able to execute table of ns1
    long procId = queue.poll().getProcId();
    assertEquals(2, procId);
    // release ns2
    queue.wakeTableExclusiveLock(procNs2, tableName2);
    // we are now able to execute ns2
    procId = queue.poll().getProcId();
    assertEquals(4, procId);
}

Example 40 with Procedure

use of org.apache.hadoop.hbase.procedure2.Procedure in project hbase by apache.

the class TestMasterProcedureScheduler method testVerifyNamespaceXLock.

@Test
public void testVerifyNamespaceXLock() throws Exception {
    String nsName = "ns1";
    TableName tableName = TableName.valueOf(nsName, name.getMethodName());
    queue.addBack(new TestNamespaceProcedure(1, nsName, TableProcedureInterface.TableOperationType.CREATE));
    queue.addBack(new TestTableProcedure(2, tableName, TableProcedureInterface.TableOperationType.READ));
    // Fetch the ns item and take the xlock
    Procedure proc = queue.poll();
    assertEquals(1, proc.getProcId());
    assertEquals(false, queue.waitNamespaceExclusiveLock(proc, nsName));
    // the table operation can't be executed because the ns is locked
    assertEquals(null, queue.poll(0));
    // release the ns lock
    queue.wakeNamespaceExclusiveLock(proc, nsName);
    proc = queue.poll();
    assertEquals(2, proc.getProcId());
    assertEquals(false, queue.waitTableExclusiveLock(proc, tableName));
    queue.wakeTableExclusiveLock(proc, tableName);
}