Examples with RpcControllerFactory org.apache.hadoop.hbase.ipc.RpcControllerFactory used on opensource projects

Example 1 with RpcControllerFactory

use of org.apache.hadoop.hbase.ipc.RpcControllerFactory in project hbase by apache.

the class TestSnapshotFromAdmin method testBackoffLogic.

/**
   * Test that the logic for doing 'correct' back-off based on exponential increase and the max-time
   * passed from the server ensures the correct overall waiting for the snapshot to finish.
   * @throws Exception
   */
@Test(timeout = 60000)
public void testBackoffLogic() throws Exception {
    final int pauseTime = 100;
    final int maxWaitTime = HConstants.RETRY_BACKOFF[HConstants.RETRY_BACKOFF.length - 1] * pauseTime;
    final int numRetries = HConstants.RETRY_BACKOFF.length;
    // calculate the wait time, if we just do straight backoff (ignoring the expected time from
    // master)
    long ignoreExpectedTime = 0;
    for (int i = 0; i < HConstants.RETRY_BACKOFF.length; i++) {
        ignoreExpectedTime += HConstants.RETRY_BACKOFF[i] * pauseTime;
    }
    // the correct wait time, capping at the maxTime/tries + fudge room
    final long time = pauseTime * 3 + ((maxWaitTime / numRetries) * 3) + 300;
    assertTrue("Capped snapshot wait time isn't less that the uncapped backoff time " + "- further testing won't prove anything.", time < ignoreExpectedTime);
    // setup the mocks
    ConnectionImplementation mockConnection = Mockito.mock(ConnectionImplementation.class);
    Configuration conf = HBaseConfiguration.create();
    // setup the conf to match the expected properties
    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, numRetries);
    conf.setLong("hbase.client.pause", pauseTime);
    // mock the master admin to our mock
    MasterKeepAliveConnection mockMaster = Mockito.mock(MasterKeepAliveConnection.class);
    Mockito.when(mockConnection.getConfiguration()).thenReturn(conf);
    Mockito.when(mockConnection.getKeepAliveMasterService()).thenReturn(mockMaster);
    // we need a real retrying caller
    RpcRetryingCallerFactory callerFactory = new RpcRetryingCallerFactory(conf);
    RpcControllerFactory controllerFactory = Mockito.mock(RpcControllerFactory.class);
    Mockito.when(controllerFactory.newController()).thenReturn(Mockito.mock(HBaseRpcController.class));
    Mockito.when(mockConnection.getRpcRetryingCallerFactory()).thenReturn(callerFactory);
    Mockito.when(mockConnection.getRpcControllerFactory()).thenReturn(controllerFactory);
    // set the max wait time for the snapshot to complete
    SnapshotResponse response = SnapshotResponse.newBuilder().setExpectedTimeout(maxWaitTime).build();
    Mockito.when(mockMaster.snapshot((RpcController) Mockito.any(), Mockito.any(SnapshotRequest.class))).thenReturn(response);
    // setup the response
    IsSnapshotDoneResponse.Builder builder = IsSnapshotDoneResponse.newBuilder();
    builder.setDone(false);
    // first five times, we return false, last we get success
    Mockito.when(mockMaster.isSnapshotDone((RpcController) Mockito.any(), Mockito.any(IsSnapshotDoneRequest.class))).thenReturn(builder.build(), builder.build(), builder.build(), builder.build(), builder.build(), builder.setDone(true).build());
    // setup the admin and run the test
    Admin admin = new HBaseAdmin(mockConnection);
    String snapshot = "snapshot";
    final TableName table = TableName.valueOf(name.getMethodName());
    // get start time
    long start = System.currentTimeMillis();
    admin.snapshot(snapshot, table);
    long finish = System.currentTimeMillis();
    long elapsed = (finish - start);
    assertTrue("Elapsed time:" + elapsed + " is more than expected max:" + time, elapsed <= time);
    admin.close();
}

Example 2 with RpcControllerFactory

use of org.apache.hadoop.hbase.ipc.RpcControllerFactory in project hbase by apache.

the class TestClientScanner method testExceptionsFromReplicasArePropagated.

/**
   * Tests the case where all replicas of a region throw an exception. It should not cause a hang
   * but the exception should propagate to the client
   */
@Test(timeout = 30000)
public void testExceptionsFromReplicasArePropagated() throws IOException {
    scan.setConsistency(Consistency.TIMELINE);
    // Mock a caller which calls the callable for ScannerCallableWithReplicas,
    // but throws an exception for the actual scanner calls via callWithRetries.
    rpcFactory = new MockRpcRetryingCallerFactory(conf);
    conf.set(RpcRetryingCallerFactory.CUSTOM_CALLER_CONF_KEY, MockRpcRetryingCallerFactory.class.getName());
    // mock 3 replica locations
    when(clusterConn.locateRegion((TableName) any(), (byte[]) any(), anyBoolean(), anyBoolean(), anyInt())).thenReturn(new RegionLocations(null, null, null));
    try (MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf(name.getMethodName()), clusterConn, rpcFactory, new RpcControllerFactory(conf), pool, Integer.MAX_VALUE)) {
        Iterator<Result> iter = scanner.iterator();
        while (iter.hasNext()) {
            iter.next();
        }
        fail("Should have failed with RetriesExhaustedException");
    } catch (RuntimeException expected) {
        assertThat(expected.getCause(), instanceOf(RetriesExhaustedException.class));
    }
}

Example 3 with RpcControllerFactory

use of org.apache.hadoop.hbase.ipc.RpcControllerFactory in project hbase by apache.

the class TestMetaTableLocator method testVerifyMetaRegionLocationFails.

/**
   * Test get of meta region fails properly if nothing to connect to.
   * @throws IOException
   * @throws InterruptedException
   * @throws KeeperException
   * @throws ServiceException
   */
@Test
public void testVerifyMetaRegionLocationFails() throws IOException, InterruptedException, KeeperException, ServiceException {
    ClusterConnection connection = Mockito.mock(ClusterConnection.class);
    ServiceException connectException = new ServiceException(new ConnectException("Connection refused"));
    final AdminProtos.AdminService.BlockingInterface implementation = Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
    Mockito.when(implementation.getRegionInfo((RpcController) Mockito.any(), (GetRegionInfoRequest) Mockito.any())).thenThrow(connectException);
    Mockito.when(connection.getAdmin(Mockito.any(ServerName.class))).thenReturn(implementation);
    RpcControllerFactory controllerFactory = Mockito.mock(RpcControllerFactory.class);
    Mockito.when(controllerFactory.newController()).thenReturn(Mockito.mock(HBaseRpcController.class));
    Mockito.when(connection.getRpcControllerFactory()).thenReturn(controllerFactory);
    ServerName sn = ServerName.valueOf("example.com", 1234, System.currentTimeMillis());
    MetaTableLocator.setMetaLocation(this.watcher, sn, RegionState.State.OPENING);
    assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100));
    MetaTableLocator.setMetaLocation(this.watcher, sn, RegionState.State.OPEN);
    assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100));
}

Example 4 with RpcControllerFactory

use of org.apache.hadoop.hbase.ipc.RpcControllerFactory in project hbase by apache.

the class TestHCM method testCallableSleep.

@Test
public void testCallableSleep() throws Exception {
    long pauseTime;
    long baseTime = 100;
    final TableName tableName = TableName.valueOf(name.getMethodName());
    TEST_UTIL.createTable(tableName, FAM_NAM);
    ClientServiceCallable<Object> regionServerCallable = new ClientServiceCallable<Object>(TEST_UTIL.getConnection(), tableName, ROW, new RpcControllerFactory(TEST_UTIL.getConfiguration()).newController()) {

        @Override
        protected Object rpcCall() throws Exception {
            return null;
        }
    };
    regionServerCallable.prepare(false);
    for (int i = 0; i < HConstants.RETRY_BACKOFF.length; i++) {
        pauseTime = regionServerCallable.sleep(baseTime, i);
        assertTrue(pauseTime >= (baseTime * HConstants.RETRY_BACKOFF[i]));
        assertTrue(pauseTime <= (baseTime * HConstants.RETRY_BACKOFF[i] * 1.01f));
    }
    RegionAdminServiceCallable<Object> regionAdminServiceCallable = new RegionAdminServiceCallable<Object>((ClusterConnection) TEST_UTIL.getConnection(), new RpcControllerFactory(TEST_UTIL.getConfiguration()), tableName, ROW) {

        @Override
        public Object call(HBaseRpcController controller) throws Exception {
            return null;
        }
    };
    regionAdminServiceCallable.prepare(false);
    for (int i = 0; i < HConstants.RETRY_BACKOFF.length; i++) {
        pauseTime = regionAdminServiceCallable.sleep(baseTime, i);
        assertTrue(pauseTime >= (baseTime * HConstants.RETRY_BACKOFF[i]));
        assertTrue(pauseTime <= (baseTime * HConstants.RETRY_BACKOFF[i] * 1.01f));
    }
    MasterCallable<Object> masterCallable = new MasterCallable<Object>(TEST_UTIL.getConnection(), new RpcControllerFactory(TEST_UTIL.getConfiguration())) {

        @Override
        protected Object rpcCall() throws Exception {
            return null;
        }
    };
    try {
        for (int i = 0; i < HConstants.RETRY_BACKOFF.length; i++) {
            pauseTime = masterCallable.sleep(baseTime, i);
            assertTrue(pauseTime >= (baseTime * HConstants.RETRY_BACKOFF[i]));
            assertTrue(pauseTime <= (baseTime * HConstants.RETRY_BACKOFF[i] * 1.01f));
        }
    } finally {
        masterCallable.close();
    }
}

Example 5 with RpcControllerFactory

use of org.apache.hadoop.hbase.ipc.RpcControllerFactory in project hbase by apache.

the class TestReplicaWithCluster method testBulkLoad.

@Test(timeout = 30000)
public void testBulkLoad() throws IOException {
    // Create table then get the single region for our new table.
    LOG.debug("Creating test table");
    HTableDescriptor hdt = HTU.createTableDescriptor("testBulkLoad");
    hdt.setRegionReplication(NB_SERVERS);
    hdt.addCoprocessor(SlowMeCopro.class.getName());
    Table table = HTU.createTable(hdt, new byte[][] { f }, null);
    // create hfiles to load.
    LOG.debug("Creating test data");
    Path dir = HTU.getDataTestDirOnTestFS("testBulkLoad");
    final int numRows = 10;
    final byte[] qual = Bytes.toBytes("qual");
    final byte[] val = Bytes.toBytes("val");
    final List<Pair<byte[], String>> famPaths = new ArrayList<>();
    for (HColumnDescriptor col : hdt.getColumnFamilies()) {
        Path hfile = new Path(dir, col.getNameAsString());
        TestHRegionServerBulkLoad.createHFile(HTU.getTestFileSystem(), hfile, col.getName(), qual, val, numRows);
        famPaths.add(new Pair<>(col.getName(), hfile.toString()));
    }
    // bulk load HFiles
    LOG.debug("Loading test data");
    final ClusterConnection conn = (ClusterConnection) HTU.getAdmin().getConnection();
    table = conn.getTable(hdt.getTableName());
    final String bulkToken = new SecureBulkLoadClient(HTU.getConfiguration(), table).prepareBulkLoad(conn);
    ClientServiceCallable<Void> callable = new ClientServiceCallable<Void>(conn, hdt.getTableName(), TestHRegionServerBulkLoad.rowkey(0), new RpcControllerFactory(HTU.getConfiguration()).newController()) {

        @Override
        protected Void rpcCall() throws Exception {
            LOG.debug("Going to connect to server " + getLocation() + " for row " + Bytes.toStringBinary(getRow()));
            SecureBulkLoadClient secureClient = null;
            byte[] regionName = getLocation().getRegionInfo().getRegionName();
            try (Table table = conn.getTable(getTableName())) {
                secureClient = new SecureBulkLoadClient(HTU.getConfiguration(), table);
                secureClient.secureBulkLoadHFiles(getStub(), famPaths, regionName, true, null, bulkToken);
            }
            return null;
        }
    };
    RpcRetryingCallerFactory factory = new RpcRetryingCallerFactory(HTU.getConfiguration());
    RpcRetryingCaller<Void> caller = factory.newCaller();
    caller.callWithRetries(callable, 10000);
    // verify we can read them from the primary
    LOG.debug("Verifying data load");
    for (int i = 0; i < numRows; i++) {
        byte[] row = TestHRegionServerBulkLoad.rowkey(i);
        Get g = new Get(row);
        Result r = table.get(g);
        Assert.assertFalse(r.isStale());
    }
    // verify we can read them from the replica
    LOG.debug("Verifying replica queries");
    try {
        SlowMeCopro.cdl.set(new CountDownLatch(1));
        for (int i = 0; i < numRows; i++) {
            byte[] row = TestHRegionServerBulkLoad.rowkey(i);
            Get g = new Get(row);
            g.setConsistency(Consistency.TIMELINE);
            Result r = table.get(g);
            Assert.assertTrue(r.isStale());
        }
        SlowMeCopro.cdl.get().countDown();
    } finally {
        SlowMeCopro.cdl.get().countDown();
        SlowMeCopro.sleepTime.set(0);
    }
    HTU.getAdmin().disableTable(hdt.getTableName());
    HTU.deleteTable(hdt.getTableName());
}