Examples with SnapshotResponse org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SnapshotResponse used on opensource projects

Example 1 with SnapshotResponse

use of org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SnapshotResponse in project hbase by apache.

the class TestSnapshotFromAdmin method testBackoffLogic.

/**
   * Test that the logic for doing 'correct' back-off based on exponential increase and the max-time
   * passed from the server ensures the correct overall waiting for the snapshot to finish.
   * @throws Exception
   */
@Test(timeout = 60000)
public void testBackoffLogic() throws Exception {
    final int pauseTime = 100;
    final int maxWaitTime = HConstants.RETRY_BACKOFF[HConstants.RETRY_BACKOFF.length - 1] * pauseTime;
    final int numRetries = HConstants.RETRY_BACKOFF.length;
    // calculate the wait time, if we just do straight backoff (ignoring the expected time from
    // master)
    long ignoreExpectedTime = 0;
    for (int i = 0; i < HConstants.RETRY_BACKOFF.length; i++) {
        ignoreExpectedTime += HConstants.RETRY_BACKOFF[i] * pauseTime;
    }
    // the correct wait time, capping at the maxTime/tries + fudge room
    final long time = pauseTime * 3 + ((maxWaitTime / numRetries) * 3) + 300;
    assertTrue("Capped snapshot wait time isn't less that the uncapped backoff time " + "- further testing won't prove anything.", time < ignoreExpectedTime);
    // setup the mocks
    ConnectionImplementation mockConnection = Mockito.mock(ConnectionImplementation.class);
    Configuration conf = HBaseConfiguration.create();
    // setup the conf to match the expected properties
    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, numRetries);
    conf.setLong("hbase.client.pause", pauseTime);
    // mock the master admin to our mock
    MasterKeepAliveConnection mockMaster = Mockito.mock(MasterKeepAliveConnection.class);
    Mockito.when(mockConnection.getConfiguration()).thenReturn(conf);
    Mockito.when(mockConnection.getKeepAliveMasterService()).thenReturn(mockMaster);
    // we need a real retrying caller
    RpcRetryingCallerFactory callerFactory = new RpcRetryingCallerFactory(conf);
    RpcControllerFactory controllerFactory = Mockito.mock(RpcControllerFactory.class);
    Mockito.when(controllerFactory.newController()).thenReturn(Mockito.mock(HBaseRpcController.class));
    Mockito.when(mockConnection.getRpcRetryingCallerFactory()).thenReturn(callerFactory);
    Mockito.when(mockConnection.getRpcControllerFactory()).thenReturn(controllerFactory);
    // set the max wait time for the snapshot to complete
    SnapshotResponse response = SnapshotResponse.newBuilder().setExpectedTimeout(maxWaitTime).build();
    Mockito.when(mockMaster.snapshot((RpcController) Mockito.any(), Mockito.any(SnapshotRequest.class))).thenReturn(response);
    // setup the response
    IsSnapshotDoneResponse.Builder builder = IsSnapshotDoneResponse.newBuilder();
    builder.setDone(false);
    // first five times, we return false, last we get success
    Mockito.when(mockMaster.isSnapshotDone((RpcController) Mockito.any(), Mockito.any(IsSnapshotDoneRequest.class))).thenReturn(builder.build(), builder.build(), builder.build(), builder.build(), builder.build(), builder.setDone(true).build());
    // setup the admin and run the test
    Admin admin = new HBaseAdmin(mockConnection);
    String snapshot = "snapshot";
    final TableName table = TableName.valueOf(name.getMethodName());
    // get start time
    long start = System.currentTimeMillis();
    admin.snapshot(snapshot, table);
    long finish = System.currentTimeMillis();
    long elapsed = (finish - start);
    assertTrue("Elapsed time:" + elapsed + " is more than expected max:" + time, elapsed <= time);
    admin.close();
}

Example 2 with SnapshotResponse

use of org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SnapshotResponse in project hbase by apache.

the class HBaseAdmin method asyncSnapshot.

private SnapshotResponse asyncSnapshot(HBaseProtos.SnapshotDescription snapshot) throws IOException {
    ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
    final SnapshotRequest request = SnapshotRequest.newBuilder().setSnapshot(snapshot).build();
    // run the snapshot on the master
    return executeCallable(new MasterCallable<SnapshotResponse>(getConnection(), getRpcControllerFactory()) {

        @Override
        protected SnapshotResponse rpcCall() throws Exception {
            return master.snapshot(getRpcController(), request);
        }
    });
}

Example 3 with SnapshotResponse

use of org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SnapshotResponse in project hbase by apache.

the class HBaseAdmin method snapshot.

@Override
public void snapshot(SnapshotDescription snapshotDesc) throws IOException, SnapshotCreationException, IllegalArgumentException {
    // actually take the snapshot
    HBaseProtos.SnapshotDescription snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshotDesc);
    SnapshotResponse response = asyncSnapshot(snapshot);
    final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot).build();
    IsSnapshotDoneResponse done = null;
    long start = EnvironmentEdgeManager.currentTime();
    long max = response.getExpectedTimeout();
    long maxPauseTime = max / this.numRetries;
    int tries = 0;
    LOG.debug("Waiting a max of " + max + " ms for snapshot '" + ClientSnapshotDescriptionUtils.toString(snapshot) + "'' to complete. (max " + maxPauseTime + " ms per retry)");
    while (tries == 0 || ((EnvironmentEdgeManager.currentTime() - start) < max && !done.getDone())) {
        try {
            // sleep a backoff <= pauseTime amount
            long sleep = getPauseTime(tries++);
            sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
            LOG.debug("(#" + tries + ") Sleeping: " + sleep + "ms while waiting for snapshot completion.");
            Thread.sleep(sleep);
        } catch (InterruptedException e) {
            throw (InterruptedIOException) new InterruptedIOException("Interrupted").initCause(e);
        }
        LOG.debug("Getting current status of snapshot from master...");
        done = executeCallable(new MasterCallable<IsSnapshotDoneResponse>(getConnection(), getRpcControllerFactory()) {

            @Override
            protected IsSnapshotDoneResponse rpcCall() throws Exception {
                return master.isSnapshotDone(getRpcController(), request);
            }
        });
    }
    if (!done.getDone()) {
        throw new SnapshotCreationException("Snapshot '" + snapshot.getName() + "' wasn't completed in expectedTime:" + max + " ms", snapshotDesc);
    }
}

Example 4 with SnapshotResponse

use of org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SnapshotResponse in project hbase by apache.

the class RawAsyncHBaseAdmin method snapshot.

@Override
public CompletableFuture<Void> snapshot(SnapshotDescription snapshotDesc) {
    SnapshotProtos.SnapshotDescription snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshotDesc);
    try {
        ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
    } catch (IllegalArgumentException e) {
        return failedFuture(e);
    }
    CompletableFuture<Void> future = new CompletableFuture<>();
    final SnapshotRequest request = SnapshotRequest.newBuilder().setSnapshot(snapshot).build();
    addListener(this.<Long>newMasterCaller().action((controller, stub) -> this.<SnapshotRequest, SnapshotResponse, Long>call(controller, stub, request, (s, c, req, done) -> s.snapshot(c, req, done), resp -> resp.getExpectedTimeout())).call(), (expectedTimeout, err) -> {
        if (err != null) {
            future.completeExceptionally(err);
            return;
        }
        TimerTask pollingTask = new TimerTask() {

            int tries = 0;

            long startTime = EnvironmentEdgeManager.currentTime();

            long endTime = startTime + expectedTimeout;

            long maxPauseTime = expectedTimeout / maxAttempts;

            @Override
            public void run(Timeout timeout) throws Exception {
                if (EnvironmentEdgeManager.currentTime() < endTime) {
                    addListener(isSnapshotFinished(snapshotDesc), (done, err2) -> {
                        if (err2 != null) {
                            future.completeExceptionally(err2);
                        } else if (done) {
                            future.complete(null);
                        } else {
                            // retry again after pauseTime.
                            long pauseTime = ConnectionUtils.getPauseTime(TimeUnit.NANOSECONDS.toMillis(pauseNs), ++tries);
                            pauseTime = Math.min(pauseTime, maxPauseTime);
                            AsyncConnectionImpl.RETRY_TIMER.newTimeout(this, pauseTime, TimeUnit.MILLISECONDS);
                        }
                    });
                } else {
                    future.completeExceptionally(new SnapshotCreationException("Snapshot '" + snapshot.getName() + "' wasn't completed in expectedTime:" + expectedTimeout + " ms", snapshotDesc));
                }
            }
        };
        AsyncConnectionImpl.RETRY_TIMER.newTimeout(pollingTask, 1, TimeUnit.MILLISECONDS);
    });
    return future;
}

Example 5 with SnapshotResponse

use of org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SnapshotResponse in project hbase by apache.

the class TestSnapshotFromAdmin method testValidateSnapshotName.

/**
   * Make sure that we validate the snapshot name and the table name before we pass anything across
   * the wire
   * @throws Exception on failure
   */
@Test
public void testValidateSnapshotName() throws Exception {
    ConnectionImplementation mockConnection = Mockito.mock(ConnectionImplementation.class);
    Configuration conf = HBaseConfiguration.create();
    Mockito.when(mockConnection.getConfiguration()).thenReturn(conf);
    // we need a real retrying caller
    RpcRetryingCallerFactory callerFactory = new RpcRetryingCallerFactory(conf);
    RpcControllerFactory controllerFactory = Mockito.mock(RpcControllerFactory.class);
    Mockito.when(controllerFactory.newController()).thenReturn(Mockito.mock(HBaseRpcController.class));
    Mockito.when(mockConnection.getRpcRetryingCallerFactory()).thenReturn(callerFactory);
    Mockito.when(mockConnection.getRpcControllerFactory()).thenReturn(controllerFactory);
    Admin admin = new HBaseAdmin(mockConnection);
    // check that invalid snapshot names fail
    failSnapshotStart(admin, new SnapshotDescription(HConstants.SNAPSHOT_DIR_NAME));
    failSnapshotStart(admin, new SnapshotDescription("-snapshot"));
    failSnapshotStart(admin, new SnapshotDescription("snapshot fails"));
    failSnapshotStart(admin, new SnapshotDescription("snap$hot"));
    failSnapshotStart(admin, new SnapshotDescription("snap:hot"));
    // check the table name also get verified
    failSnapshotDescriptorCreation("snapshot", ".table");
    failSnapshotDescriptorCreation("snapshot", "-table");
    failSnapshotDescriptorCreation("snapshot", "table fails");
    failSnapshotDescriptorCreation("snapshot", "tab%le");
    // mock the master connection
    MasterKeepAliveConnection master = Mockito.mock(MasterKeepAliveConnection.class);
    Mockito.when(mockConnection.getKeepAliveMasterService()).thenReturn(master);
    SnapshotResponse response = SnapshotResponse.newBuilder().setExpectedTimeout(0).build();
    Mockito.when(master.snapshot((RpcController) Mockito.any(), Mockito.any(SnapshotRequest.class))).thenReturn(response);
    IsSnapshotDoneResponse doneResponse = IsSnapshotDoneResponse.newBuilder().setDone(true).build();
    Mockito.when(master.isSnapshotDone((RpcController) Mockito.any(), Mockito.any(IsSnapshotDoneRequest.class))).thenReturn(doneResponse);
    // make sure that we can use valid names
    admin.snapshot(new SnapshotDescription("snapshot", TableName.valueOf(name.getMethodName())));
}