Examples with PleaseHoldException org.apache.hadoop.hbase.PleaseHoldException used on opensource projects

Example 1 with PleaseHoldException

use of org.apache.hadoop.hbase.PleaseHoldException in project hbase by apache.

the class RemoteProcedureResultReporter method run.

@Override
public void run() {
    ReportProcedureDoneRequest.Builder builder = ReportProcedureDoneRequest.newBuilder();
    int tries = 0;
    while (!server.isStopped()) {
        if (builder.getResultCount() == 0) {
            try {
                builder.addResult(results.take());
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                continue;
            }
        }
        while (builder.getResultCount() < MAX_BATCH) {
            RemoteProcedureResult result = results.poll();
            if (result == null) {
                break;
            }
            builder.addResult(result);
        }
        ReportProcedureDoneRequest request = builder.build();
        try {
            server.reportProcedureDone(builder.build());
            builder.clear();
            tries = 0;
        } catch (IOException e) {
            boolean pause = e instanceof ServerNotRunningYetException || e instanceof PleaseHoldException;
            long pauseTime;
            if (pause) {
                // Do backoff else we flood the Master with requests.
                pauseTime = ConnectionUtils.getPauseTime(server.getRetryPauseTime(), tries);
            } else {
                // Reset.
                pauseTime = server.getRetryPauseTime();
            }
            LOG.info("Failed procedure report " + TextFormat.shortDebugString(request) + "; retry (#" + tries + ")" + (pause ? " after " + pauseTime + "ms delay (Master is coming online...)." : " immediately."), e);
            Threads.sleep(pauseTime);
            tries++;
        }
    }
}

Example 2 with PleaseHoldException

use of org.apache.hadoop.hbase.PleaseHoldException in project hbase by apache.

the class AssignmentManager method reportRegionStateTransition.

public ReportRegionStateTransitionResponse reportRegionStateTransition(final ReportRegionStateTransitionRequest req) throws PleaseHoldException {
    ReportRegionStateTransitionResponse.Builder builder = ReportRegionStateTransitionResponse.newBuilder();
    ServerName serverName = ProtobufUtil.toServerName(req.getServer());
    ServerStateNode serverNode = regionStates.getOrCreateServer(serverName);
    // here we have to acquire a read lock instead of a simple exclusive lock. This is because that
    // we should not block other reportRegionStateTransition call from the same region server. This
    // is not only about performance, but also to prevent dead lock. Think of the meta region is
    // also on the same region server and you hold the lock which blocks the
    // reportRegionStateTransition for meta, and since meta is not online, you will block inside the
    // lock protection to wait for meta online...
    serverNode.readLock().lock();
    try {
        // above in submitServerCrash method and HBASE-21508 for more details.
        if (serverNode.isInState(ServerState.ONLINE)) {
            try {
                reportRegionStateTransition(builder, serverName, req.getTransitionList());
            } catch (PleaseHoldException e) {
                LOG.trace("Failed transition ", e);
                throw e;
            } catch (UnsupportedOperationException | IOException e) {
                // TODO: at the moment we have a single error message and the RS will abort
                // if the master says that one of the region transitions failed.
                LOG.warn("Failed transition", e);
                builder.setErrorMessage("Failed transition " + e.getMessage());
            }
        } else {
            LOG.warn("The region server {} is already dead, skip reportRegionStateTransition call", serverName);
            builder.setErrorMessage("You are dead");
        }
    } finally {
        serverNode.readLock().unlock();
    }
    return builder.build();
}

Example 3 with PleaseHoldException

use of org.apache.hadoop.hbase.PleaseHoldException in project hbase by apache.

the class MasterRpcServices method reportRegionStateTransition.

@Override
public ReportRegionStateTransitionResponse reportRegionStateTransition(RpcController c, ReportRegionStateTransitionRequest req) throws ServiceException {
    try {
        master.checkServiceStarted();
        RegionStateTransition rt = req.getTransition(0);
        RegionStates regionStates = master.getAssignmentManager().getRegionStates();
        for (RegionInfo ri : rt.getRegionInfoList()) {
            TableName tableName = ProtobufUtil.toTableName(ri.getTableName());
            if (!(TableName.META_TABLE_NAME.equals(tableName) && regionStates.getRegionState(HRegionInfo.FIRST_META_REGIONINFO) != null) && !master.getAssignmentManager().isFailoverCleanupDone()) {
                // failover cleanup. So no need this check for it
                throw new PleaseHoldException("Master is rebuilding user regions");
            }
        }
        ServerName sn = ProtobufUtil.toServerName(req.getServer());
        String error = master.getAssignmentManager().onRegionTransition(sn, rt);
        ReportRegionStateTransitionResponse.Builder rrtr = ReportRegionStateTransitionResponse.newBuilder();
        if (error != null) {
            rrtr.setErrorMessage(error);
        }
        return rrtr.build();
    } catch (IOException ioe) {
        throw new ServiceException(ioe);
    }
}

Example 4 with PleaseHoldException

use of org.apache.hadoop.hbase.PleaseHoldException in project hbase by apache.

the class TestHBaseAdminNoCluster method testMasterMonitorCallableRetries.

/**
   * Verify that PleaseHoldException gets retried.
   * HBASE-8764
   * @throws IOException
   * @throws ZooKeeperConnectionException
   * @throws MasterNotRunningException
   * @throws ServiceException
   * @throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException 
   */
//TODO: Clean up, with Procedure V2 and nonce to prevent the same procedure to call mulitple
// time, this test is invalid anymore. Just keep the test around for some time before
// fully removing it.
@Ignore
@Test
public void testMasterMonitorCallableRetries() throws MasterNotRunningException, ZooKeeperConnectionException, IOException, org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException {
    Configuration configuration = HBaseConfiguration.create();
    // Set the pause and retry count way down.
    configuration.setLong(HConstants.HBASE_CLIENT_PAUSE, 1);
    final int count = 10;
    configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, count);
    // Get mocked connection.   Getting the connection will register it so when HBaseAdmin is
    // constructed with same configuration, it will find this mocked connection.
    ClusterConnection connection = HConnectionTestingUtility.getMockedConnection(configuration);
    // Mock so we get back the master interface.  Make it so when createTable is called, we throw
    // the PleaseHoldException.
    MasterKeepAliveConnection masterAdmin = Mockito.mock(MasterKeepAliveConnection.class);
    Mockito.when(masterAdmin.createTable((RpcController) Mockito.any(), (CreateTableRequest) Mockito.any())).thenThrow(new ServiceException("Test fail").initCause(new PleaseHoldException("test")));
    Mockito.when(connection.getKeepAliveMasterService()).thenReturn(masterAdmin);
    Admin admin = new HBaseAdmin(connection);
    try {
        HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
        // Pass any old htable descriptor; not important
        try {
            admin.createTable(htd, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
            fail();
        } catch (RetriesExhaustedException e) {
            LOG.info("Expected fail", e);
        }
        // Assert we were called 'count' times.
        Mockito.verify(masterAdmin, Mockito.atLeast(count)).createTable((RpcController) Mockito.any(), (CreateTableRequest) Mockito.any());
    } finally {
        admin.close();
        if (connection != null)
            connection.close();
    }
}

Example 5 with PleaseHoldException

use of org.apache.hadoop.hbase.PleaseHoldException in project hbase by apache.

the class HRegionServer method reportRegionStateTransition.

@Override
public boolean reportRegionStateTransition(final RegionStateTransitionContext context) {
    if (TEST_SKIP_REPORTING_TRANSITION) {
        return skipReportingTransition(context);
    }
    final ReportRegionStateTransitionRequest request = createReportRegionStateTransitionRequest(context);
    int tries = 0;
    long pauseTime = this.retryPauseTime;
    // HRegionServer does down.
    while (this.asyncClusterConnection != null && !this.asyncClusterConnection.isClosed()) {
        RegionServerStatusService.BlockingInterface rss = rssStub;
        try {
            if (rss == null) {
                createRegionServerStatusStub();
                continue;
            }
            ReportRegionStateTransitionResponse response = rss.reportRegionStateTransition(null, request);
            if (response.hasErrorMessage()) {
                LOG.info("TRANSITION FAILED " + request + ": " + response.getErrorMessage());
                break;
            }
            // know if were successful after an attempt showed in logs as failed.
            if (tries > 0 || LOG.isTraceEnabled()) {
                LOG.info("TRANSITION REPORTED " + request);
            }
            // NOTE: Return mid-method!!!
            return true;
        } catch (ServiceException se) {
            IOException ioe = ProtobufUtil.getRemoteException(se);
            boolean pause = ioe instanceof ServerNotRunningYetException || ioe instanceof PleaseHoldException || ioe instanceof CallQueueTooBigException;
            if (pause) {
                // Do backoff else we flood the Master with requests.
                pauseTime = ConnectionUtils.getPauseTime(this.retryPauseTime, tries);
            } else {
                // Reset.
                pauseTime = this.retryPauseTime;
            }
            LOG.info("Failed report transition " + TextFormat.shortDebugString(request) + "; retry (#" + tries + ")" + (pause ? " after " + pauseTime + "ms delay (Master is coming online...)." : " immediately."), ioe);
            if (pause) {
                Threads.sleep(pauseTime);
            }
            tries++;
            if (rssStub == rss) {
                rssStub = null;
            }
        }
    }
    return false;
}