Examples with ScanResponse org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanResponse used on opensource projects

Example 1 with ScanResponse

use of org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanResponse in project hbase by apache.

the class RSRpcServices method scan.

/**
   * Scan data in a table.
   *
   * @param controller the RPC controller
   * @param request the scan request
   * @throws ServiceException
   */
@Override
public ScanResponse scan(final RpcController controller, final ScanRequest request) throws ServiceException {
    if (controller != null && !(controller instanceof HBaseRpcController)) {
        throw new UnsupportedOperationException("We only do " + "HBaseRpcControllers! FIX IF A PROBLEM: " + controller);
    }
    if (!request.hasScannerId() && !request.hasScan()) {
        throw new ServiceException(new DoNotRetryIOException("Missing required input: scannerId or scan"));
    }
    try {
        checkOpen();
    } catch (IOException e) {
        if (request.hasScannerId()) {
            String scannerName = Long.toString(request.getScannerId());
            if (LOG.isDebugEnabled()) {
                LOG.debug("Server shutting down and client tried to access missing scanner " + scannerName);
            }
            if (regionServer.leases != null) {
                try {
                    regionServer.leases.cancelLease(scannerName);
                } catch (LeaseException le) {
                    // No problem, ignore
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("Un-able to cancel lease of scanner. It could already be closed.");
                    }
                }
            }
        }
        throw new ServiceException(e);
    }
    requestCount.increment();
    rpcScanRequestCount.increment();
    RegionScannerHolder rsh;
    ScanResponse.Builder builder = ScanResponse.newBuilder();
    try {
        if (request.hasScannerId()) {
            rsh = getRegionScanner(request);
        } else {
            rsh = newRegionScanner(request, builder);
        }
    } catch (IOException e) {
        if (e == SCANNER_ALREADY_CLOSED) {
            // the old client will still send a close request to us. Just ignore it and return.
            return builder.build();
        }
        throw new ServiceException(e);
    }
    Region region = rsh.r;
    String scannerName = rsh.scannerName;
    Leases.Lease lease;
    try {
        // Remove lease while its being processed in server; protects against case
        // where processing of request takes > lease expiration time.
        lease = regionServer.leases.removeLease(scannerName);
    } catch (LeaseException e) {
        throw new ServiceException(e);
    }
    if (request.hasRenew() && request.getRenew()) {
        // add back and return
        addScannerLeaseBack(lease);
        try {
            checkScanNextCallSeq(request, rsh);
        } catch (OutOfOrderScannerNextException e) {
            throw new ServiceException(e);
        }
        return builder.build();
    }
    OperationQuota quota;
    try {
        quota = getQuotaManager().checkQuota(region, OperationQuota.OperationType.SCAN);
    } catch (IOException e) {
        addScannerLeaseBack(lease);
        throw new ServiceException(e);
    }
    ;
    try {
        checkScanNextCallSeq(request, rsh);
    } catch (OutOfOrderScannerNextException e) {
        addScannerLeaseBack(lease);
        throw new ServiceException(e);
    }
    // Now we have increased the next call sequence. If we give client an error, the retry will
    // never success. So we'd better close the scanner and return a DoNotRetryIOException to client
    // and then client will try to open a new scanner.
    boolean closeScanner = request.hasCloseScanner() ? request.getCloseScanner() : false;
    // this is scan.getCaching
    int rows;
    if (request.hasNumberOfRows()) {
        rows = request.getNumberOfRows();
    } else {
        rows = closeScanner ? 0 : 1;
    }
    RpcCallContext context = RpcServer.getCurrentCall();
    // now let's do the real scan.
    long maxQuotaResultSize = Math.min(maxScannerResultSize, quota.getReadAvailable());
    RegionScanner scanner = rsh.s;
    boolean moreResults = true;
    boolean moreResultsInRegion = true;
    // this is the limit of rows for this scan, if we the number of rows reach this value, we will
    // close the scanner.
    int limitOfRows;
    if (request.hasLimitOfRows()) {
        limitOfRows = request.getLimitOfRows();
        rows = Math.min(rows, limitOfRows);
    } else {
        limitOfRows = -1;
    }
    MutableObject lastBlock = new MutableObject();
    boolean scannerClosed = false;
    try {
        List<Result> results = new ArrayList<>();
        if (rows > 0) {
            boolean done = false;
            // Call coprocessor. Get region info from scanner.
            if (region.getCoprocessorHost() != null) {
                Boolean bypass = region.getCoprocessorHost().preScannerNext(scanner, results, rows);
                if (!results.isEmpty()) {
                    for (Result r : results) {
                        lastBlock.setValue(addSize(context, r, lastBlock.getValue()));
                    }
                }
                if (bypass != null && bypass.booleanValue()) {
                    done = true;
                }
            }
            if (!done) {
                moreResultsInRegion = scan((HBaseRpcController) controller, request, rsh, maxQuotaResultSize, rows, results, builder, lastBlock, context);
            }
        }
        quota.addScanResult(results);
        if (scanner.isFilterDone() && results.isEmpty()) {
            // If the scanner's filter - if any - is done with the scan
            // only set moreResults to false if the results is empty. This is used to keep compatible
            // with the old scan implementation where we just ignore the returned results if moreResults
            // is false. Can remove the isEmpty check after we get rid of the old implementation.
            moreResults = false;
        } else if (limitOfRows > 0 && !results.isEmpty() && !results.get(results.size() - 1).mayHaveMoreCellsInRow() && ConnectionUtils.numberOfIndividualRows(results) >= limitOfRows) {
            // if we have reached the limit of rows
            moreResults = false;
        }
        addResults(builder, results, (HBaseRpcController) controller, RegionReplicaUtil.isDefaultReplica(region.getRegionInfo()), isClientCellBlockSupport(context));
        if (!moreResults || !moreResultsInRegion || closeScanner) {
            scannerClosed = true;
            closeScanner(region, scanner, scannerName, context);
        }
        builder.setMoreResults(moreResults);
        return builder.build();
    } catch (Exception e) {
        try {
            // scanner is closed here
            scannerClosed = true;
            // The scanner state might be left in a dirty state, so we will tell the Client to
            // fail this RPC and close the scanner while opening up another one from the start of
            // row that the client has last seen.
            closeScanner(region, scanner, scannerName, context);
            // the client. See ClientScanner code to see how it deals with these special exceptions.
            if (e instanceof DoNotRetryIOException) {
                throw e;
            }
            // DoNotRetryIOException. This can avoid the retry in ClientScanner.
            if (e instanceof FileNotFoundException) {
                throw new DoNotRetryIOException(e);
            }
            // a special exception to save an RPC.
            if (VersionInfoUtil.hasMinimumVersion(context.getClientVersionInfo(), 1, 4)) {
                // 1.4.0+ clients know how to handle
                throw new ScannerResetException("Scanner is closed on the server-side", e);
            } else {
                // older clients do not know about SRE. Just throw USE, which they will handle
                throw new UnknownScannerException("Throwing UnknownScannerException to reset the client" + " scanner state for clients older than 1.3.", e);
            }
        } catch (IOException ioe) {
            throw new ServiceException(ioe);
        }
    } finally {
        if (!scannerClosed) {
            // the closeCallBack will be set in closeScanner so here we only care about shippedCallback
            if (context != null) {
                context.setCallBack(rsh.shippedCallback);
            } else {
                // When context != null, adding back the lease will be done in callback set above.
                addScannerLeaseBack(lease);
            }
        }
        quota.close();
    }
}

Example 2 with ScanResponse

use of org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanResponse in project hbase by apache.

the class ScannerCallable method openScanner.

private ScanResponse openScanner() throws IOException {
    incRPCcallsMetrics();
    ScanRequest request = RequestConverter.buildScanRequest(getLocation().getRegionInfo().getRegionName(), this.scan, this.caching, false);
    try {
        ScanResponse response = getStub().scan(getRpcController(), request);
        long id = response.getScannerId();
        if (logScannerActivity) {
            LOG.info("Open scanner=" + id + " for scan=" + scan.toString() + " on region " + getLocation().toString());
        }
        if (response.hasMvccReadPoint()) {
            this.scan.setMvccReadPoint(response.getMvccReadPoint());
        }
        this.scannerId = id;
        return response;
    } catch (Exception e) {
        throw ProtobufUtil.handleRemoteException(e);
    }
}

Example 3 with ScanResponse

use of org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanResponse in project hbase by apache.

the class ScannerCallable method rpcCall.

@Override
protected Result[] rpcCall() throws Exception {
    if (Thread.interrupted()) {
        throw new InterruptedIOException();
    }
    if (closed) {
        close();
        return null;
    }
    ScanResponse response;
    if (this.scannerId == -1L) {
        response = openScanner();
    } else {
        response = next();
    }
    long timestamp = System.currentTimeMillis();
    setHeartbeatMessage(response.hasHeartbeatMessage() && response.getHeartbeatMessage());
    Result[] rrs = ResponseConverter.getResults(getRpcControllerCellScanner(), response);
    if (logScannerActivity) {
        long now = System.currentTimeMillis();
        if (now - timestamp > logCutOffLatency) {
            int rows = rrs == null ? 0 : rrs.length;
            LOG.info("Took " + (now - timestamp) + "ms to fetch " + rows + " rows from scanner=" + scannerId);
        }
    }
    updateServerSideMetrics(response);
    // moreResults is only used for the case where a filter exhausts all elements
    if (response.hasMoreResults()) {
        if (response.getMoreResults()) {
            setMoreResultsForScan(MoreResults.YES);
        } else {
            setMoreResultsForScan(MoreResults.NO);
            setAlreadyClosed();
        }
    } else {
        setMoreResultsForScan(MoreResults.UNKNOWN);
    }
    if (response.hasMoreResultsInRegion()) {
        if (response.getMoreResultsInRegion()) {
            setMoreResultsInRegion(MoreResults.YES);
        } else {
            setMoreResultsInRegion(MoreResults.NO);
            setAlreadyClosed();
        }
    } else {
        setMoreResultsInRegion(MoreResults.UNKNOWN);
    }
    updateResultsMetrics(rrs);
    return rrs;
}

Example 4 with ScanResponse

use of org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanResponse in project hbase by apache.

the class TestClientNoCluster method doMetaScanResponse.

static ScanResponse doMetaScanResponse(final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta, final AtomicLong sequenceids, final ScanRequest request) {
    ScanResponse.Builder builder = ScanResponse.newBuilder();
    int max = request.getNumberOfRows();
    int count = 0;
    Map<byte[], Pair<HRegionInfo, ServerName>> tail = request.hasScan() ? meta.tailMap(request.getScan().getStartRow().toByteArray()) : meta;
    ClientProtos.Result.Builder resultBuilder = ClientProtos.Result.newBuilder();
    for (Map.Entry<byte[], Pair<HRegionInfo, ServerName>> e : tail.entrySet()) {
        // Can be 0 on open of a scanner -- i.e. rpc to setup scannerid only.
        if (max <= 0)
            break;
        if (++count > max)
            break;
        HRegionInfo hri = e.getValue().getFirst();
        ByteString row = UnsafeByteOperations.unsafeWrap(hri.getRegionName());
        resultBuilder.clear();
        resultBuilder.addCell(getRegionInfo(row, hri));
        resultBuilder.addCell(getServer(row, e.getValue().getSecond()));
        resultBuilder.addCell(getStartCode(row));
        builder.addResults(resultBuilder.build());
        // Set more to false if we are on the last region in table.
        if (hri.getEndKey().length <= 0)
            builder.setMoreResults(false);
        else
            builder.setMoreResults(true);
    }
    // If no scannerid, set one.
    builder.setScannerId(request.hasScannerId() ? request.getScannerId() : sequenceids.incrementAndGet());
    return builder.build();
}

Example 5 with ScanResponse

use of org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanResponse in project hbase by apache.

the class TestMetaTableAccessorNoCluster method testRideOverServerNotRunning.

/**
   * Test that MetaTableAccessor will ride over server throwing
   * "Server not running" IOEs.
   * @see @link {https://issues.apache.org/jira/browse/HBASE-3446}
   * @throws IOException
   * @throws InterruptedException
   */
@Test
public void testRideOverServerNotRunning() throws IOException, InterruptedException, ServiceException {
    // Need a zk watcher.
    ZooKeeperWatcher zkw = new ZooKeeperWatcher(UTIL.getConfiguration(), this.getClass().getSimpleName(), ABORTABLE, true);
    // This is a servername we use in a few places below.
    ServerName sn = ServerName.valueOf("example.com", 1234, System.currentTimeMillis());
    ClusterConnection connection = null;
    try {
        // Mock an ClientProtocol. Our mock implementation will fail a few
        // times when we go to open a scanner.
        final ClientProtos.ClientService.BlockingInterface implementation = Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
        // When scan called throw IOE 'Server not running' a few times
        // before we return a scanner id.  Whats WEIRD is that these
        // exceptions do not show in the log because they are caught and only
        // printed if we FAIL.  We eventually succeed after retry so these don't
        // show.  We will know if they happened or not because we will ask
        // mockito at the end of this test to verify that scan was indeed
        // called the wanted number of times.
        List<Cell> kvs = new ArrayList<>();
        final byte[] rowToVerify = Bytes.toBytes("rowToVerify");
        kvs.add(new KeyValue(rowToVerify, HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, HRegionInfo.FIRST_META_REGIONINFO.toByteArray()));
        kvs.add(new KeyValue(rowToVerify, HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes(sn.getHostAndPort())));
        kvs.add(new KeyValue(rowToVerify, HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(sn.getStartcode())));
        final List<CellScannable> cellScannables = new ArrayList<>(1);
        cellScannables.add(Result.create(kvs));
        final ScanResponse.Builder builder = ScanResponse.newBuilder();
        for (CellScannable result : cellScannables) {
            builder.addCellsPerResult(((Result) result).size());
        }
        Mockito.when(implementation.scan((RpcController) Mockito.any(), (ScanRequest) Mockito.any())).thenThrow(new ServiceException("Server not running (1 of 3)")).thenThrow(new ServiceException("Server not running (2 of 3)")).thenThrow(new ServiceException("Server not running (3 of 3)")).thenAnswer(new Answer<ScanResponse>() {

            public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
                ((HBaseRpcController) invocation.getArguments()[0]).setCellScanner(CellUtil.createCellScanner(cellScannables));
                return builder.setScannerId(1234567890L).setMoreResults(false).build();
            }
        });
        // Associate a spied-upon Connection with UTIL.getConfiguration.  Need
        // to shove this in here first so it gets picked up all over; e.g. by
        // HTable.
        connection = HConnectionTestingUtility.getSpiedConnection(UTIL.getConfiguration());
        // Fix the location lookup so it 'works' though no network.  First
        // make an 'any location' object.
        final HRegionLocation anyLocation = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, sn);
        final RegionLocations rl = new RegionLocations(anyLocation);
        // Return the RegionLocations object when locateRegion
        // The ugly format below comes of 'Important gotcha on spying real objects!' from
        // http://mockito.googlecode.com/svn/branches/1.6/javadoc/org/mockito/Mockito.html
        Mockito.doReturn(rl).when(connection).locateRegion((TableName) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyInt());
        // Now shove our HRI implementation into the spied-upon connection.
        Mockito.doReturn(implementation).when(connection).getClient(Mockito.any(ServerName.class));
        // Scan meta for user tables and verify we got back expected answer.
        NavigableMap<HRegionInfo, Result> hris = MetaTableAccessor.getServerUserRegions(connection, sn);
        assertEquals(1, hris.size());
        assertTrue(hris.firstEntry().getKey().equals(HRegionInfo.FIRST_META_REGIONINFO));
        assertTrue(Bytes.equals(rowToVerify, hris.firstEntry().getValue().getRow()));
        // Finally verify that scan was called four times -- three times
        // with exception and then on 4th attempt we succeed
        Mockito.verify(implementation, Mockito.times(4)).scan((RpcController) Mockito.any(), (ScanRequest) Mockito.any());
    } finally {
        if (connection != null && !connection.isClosed())
            connection.close();
        zkw.close();
    }
}