Examples with CellScanner org.apache.hadoop.hbase.CellScanner used on opensource projects

Example 36 with CellScanner

use of org.apache.hadoop.hbase.CellScanner in project hbase by apache.

the class TestVisibilityLabels method testSimpleVisibilityLabels.

@Test
public void testSimpleVisibilityLabels() throws Exception {
    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
    try (Table table = createTableAndWriteDataWithLabels(tableName, SECRET + "|" + CONFIDENTIAL, PRIVATE + "|" + CONFIDENTIAL)) {
        Scan s = new Scan();
        s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE));
        ResultScanner scanner = table.getScanner(s);
        Result[] next = scanner.next(3);
        assertTrue(next.length == 2);
        CellScanner cellScanner = next[0].cellScanner();
        cellScanner.advance();
        Cell current = cellScanner.current();
        assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(), row1, 0, row1.length));
        cellScanner = next[1].cellScanner();
        cellScanner.advance();
        current = cellScanner.current();
        assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(), row2, 0, row2.length));
    }
}

Example 37 with CellScanner

use of org.apache.hadoop.hbase.CellScanner in project hbase by apache.

the class TestRecoveredEdits method verifyAllEditsMadeItIn.

/**
   * @param fs
   * @param conf
   * @param edits
   * @param region
   * @return Return how many edits seen.
   * @throws IOException
   */
private int verifyAllEditsMadeItIn(final FileSystem fs, final Configuration conf, final Path edits, final HRegion region) throws IOException {
    int count = 0;
    // Based on HRegion#replayRecoveredEdits
    WAL.Reader reader = null;
    try {
        reader = WALFactory.createReader(fs, edits, conf);
        WAL.Entry entry;
        while ((entry = reader.next()) != null) {
            WALKey key = entry.getKey();
            WALEdit val = entry.getEdit();
            count++;
            // Check this edit is for this region.
            if (!Bytes.equals(key.getEncodedRegionName(), region.getRegionInfo().getEncodedNameAsBytes())) {
                continue;
            }
            Cell previous = null;
            for (Cell cell : val.getCells()) {
                if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY))
                    continue;
                if (previous != null && CellComparator.COMPARATOR.compareRows(previous, cell) == 0)
                    continue;
                previous = cell;
                Get g = new Get(CellUtil.cloneRow(cell));
                Result r = region.get(g);
                boolean found = false;
                for (CellScanner scanner = r.cellScanner(); scanner.advance(); ) {
                    Cell current = scanner.current();
                    if (CellComparator.COMPARATOR.compareKeyIgnoresMvcc(cell, current) == 0) {
                        found = true;
                        break;
                    }
                }
                assertTrue("Failed to find " + cell, found);
            }
        }
    } finally {
        if (reader != null)
            reader.close();
    }
    return count;
}

Example 38 with CellScanner

use of org.apache.hadoop.hbase.CellScanner in project hbase by apache.

the class TestClientScanner method testMoreResults.

@Test
@SuppressWarnings("unchecked")
public void testMoreResults() throws IOException {
    final Result[] results1 = new Result[1];
    KeyValue kv1 = new KeyValue("row".getBytes(), "cf".getBytes(), "cq".getBytes(), 1, Type.Maximum);
    results1[0] = Result.create(new Cell[] { kv1 });
    final Result[] results2 = new Result[1];
    KeyValue kv2 = new KeyValue("row2".getBytes(), "cf".getBytes(), "cq".getBytes(), 1, Type.Maximum);
    results2[0] = Result.create(new Cell[] { kv2 });
    RpcRetryingCaller<Result[]> caller = Mockito.mock(RpcRetryingCaller.class);
    Mockito.when(rpcFactory.<Result[]>newCaller()).thenReturn(caller);
    Mockito.when(caller.callWithoutRetries(Mockito.any(RetryingCallable.class), Mockito.anyInt())).thenAnswer(new Answer<Result[]>() {

        private int count = 0;

        @Override
        public Result[] answer(InvocationOnMock invocation) throws Throwable {
            ScannerCallableWithReplicas callable = invocation.getArgumentAt(0, ScannerCallableWithReplicas.class);
            switch(count) {
                case // initialize
                0:
                    count++;
                    callable.currentScannerCallable.setMoreResultsInRegion(MoreResults.YES);
                    return results1;
                case 1:
                    count++;
                    // The server reports back false WRT more results
                    callable.currentScannerCallable.setMoreResultsInRegion(MoreResults.NO);
                    return results2;
                case // close
                2:
                    count++;
                    return null;
                default:
                    throw new RuntimeException("Expected only 3 invocations");
            }
        }
    });
    // Set a much larger cache and buffer size than we'll provide
    scan.setCaching(100);
    scan.setMaxResultSize(1000 * 1000);
    try (MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf(name.getMethodName()), clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE)) {
        InOrder inOrder = Mockito.inOrder(caller);
        scanner.setRpcFinished(true);
        scanner.loadCache();
        inOrder.verify(caller, Mockito.times(2)).callWithoutRetries(Mockito.any(RetryingCallable.class), Mockito.anyInt());
        assertEquals(2, scanner.cache.size());
        Result r = scanner.cache.poll();
        assertNotNull(r);
        CellScanner cs = r.cellScanner();
        assertTrue(cs.advance());
        assertEquals(kv1, cs.current());
        assertFalse(cs.advance());
        r = scanner.cache.poll();
        assertNotNull(r);
        cs = r.cellScanner();
        assertTrue(cs.advance());
        assertEquals(kv2, cs.current());
        assertFalse(cs.advance());
    }
}

Example 39 with CellScanner

use of org.apache.hadoop.hbase.CellScanner in project hbase by apache.

the class TestClientScanner method testCacheLimit.

@Test
@SuppressWarnings("unchecked")
public void testCacheLimit() throws IOException {
    KeyValue kv1 = new KeyValue("row1".getBytes(), "cf".getBytes(), "cq".getBytes(), 1, Type.Maximum), kv2 = new KeyValue("row2".getBytes(), "cf".getBytes(), "cq".getBytes(), 1, Type.Maximum), kv3 = new KeyValue("row3".getBytes(), "cf".getBytes(), "cq".getBytes(), 1, Type.Maximum);
    final Result[] results = new Result[] { Result.create(new Cell[] { kv1 }), Result.create(new Cell[] { kv2 }), Result.create(new Cell[] { kv3 }) };
    RpcRetryingCaller<Result[]> caller = Mockito.mock(RpcRetryingCaller.class);
    Mockito.when(rpcFactory.<Result[]>newCaller()).thenReturn(caller);
    Mockito.when(caller.callWithoutRetries(Mockito.any(RetryingCallable.class), Mockito.anyInt())).thenAnswer(new Answer<Result[]>() {

        private int count = 0;

        @Override
        public Result[] answer(InvocationOnMock invocation) throws Throwable {
            ScannerCallableWithReplicas callable = invocation.getArgumentAt(0, ScannerCallableWithReplicas.class);
            switch(count) {
                case // initialize
                0:
                    count++;
                    // if we set no here the implementation will trigger a close
                    callable.currentScannerCallable.setMoreResultsInRegion(MoreResults.YES);
                    return results;
                case // close
                1:
                    count++;
                    return null;
                default:
                    throw new RuntimeException("Expected only 2 invocations");
            }
        }
    });
    Mockito.when(rpcFactory.<Result[]>newCaller()).thenReturn(caller);
    // Set a small cache
    scan.setCaching(1);
    // Set a very large size
    scan.setMaxResultSize(1000 * 1000);
    try (MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf(name.getMethodName()), clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE)) {
        InOrder inOrder = Mockito.inOrder(caller);
        scanner.loadCache();
        inOrder.verify(caller, Mockito.times(1)).callWithoutRetries(Mockito.any(RetryingCallable.class), Mockito.anyInt());
        assertEquals(3, scanner.cache.size());
        Result r = scanner.cache.poll();
        assertNotNull(r);
        CellScanner cs = r.cellScanner();
        assertTrue(cs.advance());
        assertEquals(kv1, cs.current());
        assertFalse(cs.advance());
        r = scanner.cache.poll();
        assertNotNull(r);
        cs = r.cellScanner();
        assertTrue(cs.advance());
        assertEquals(kv2, cs.current());
        assertFalse(cs.advance());
        r = scanner.cache.poll();
        assertNotNull(r);
        cs = r.cellScanner();
        assertTrue(cs.advance());
        assertEquals(kv3, cs.current());
        assertFalse(cs.advance());
    }
}

Example 40 with CellScanner

use of org.apache.hadoop.hbase.CellScanner in project hbase by apache.

the class BlockingRpcConnection method readResponse.

/*
   * Receive a response. Because only one receiver, so no synchronization on in.
   */
private void readResponse() {
    Call call = null;
    boolean expectedCall = false;
    try {
        // See HBaseServer.Call.setResponse for where we write out the response.
        // Total size of the response. Unused. But have to read it in anyways.
        int totalSize = in.readInt();
        // Read the header
        ResponseHeader responseHeader = ResponseHeader.parseDelimitedFrom(in);
        int id = responseHeader.getCallId();
        // call.done have to be set before leaving this method
        call = calls.remove(id);
        expectedCall = (call != null && !call.isDone());
        if (!expectedCall) {
            // So we got a response for which we have no corresponding 'call' here on the client-side.
            // We probably timed out waiting, cleaned up all references, and now the server decides
            // to return a response. There is nothing we can do w/ the response at this stage. Clean
            // out the wire of the response so its out of the way and we can get other responses on
            // this connection.
            int readSoFar = getTotalSizeWhenWrittenDelimited(responseHeader);
            int whatIsLeftToRead = totalSize - readSoFar;
            IOUtils.skipFully(in, whatIsLeftToRead);
            if (call != null) {
                call.callStats.setResponseSizeBytes(totalSize);
                call.callStats.setCallTimeMs(EnvironmentEdgeManager.currentTime() - call.callStats.getStartTime());
            }
            return;
        }
        if (responseHeader.hasException()) {
            ExceptionResponse exceptionResponse = responseHeader.getException();
            RemoteException re = createRemoteException(exceptionResponse);
            call.setException(re);
            call.callStats.setResponseSizeBytes(totalSize);
            call.callStats.setCallTimeMs(EnvironmentEdgeManager.currentTime() - call.callStats.getStartTime());
            if (isFatalConnectionException(exceptionResponse)) {
                synchronized (this) {
                    closeConn(re);
                }
            }
        } else {
            Message value = null;
            if (call.responseDefaultType != null) {
                Builder builder = call.responseDefaultType.newBuilderForType();
                ProtobufUtil.mergeDelimitedFrom(builder, in);
                value = builder.build();
            }
            CellScanner cellBlockScanner = null;
            if (responseHeader.hasCellBlockMeta()) {
                int size = responseHeader.getCellBlockMeta().getLength();
                byte[] cellBlock = new byte[size];
                IOUtils.readFully(this.in, cellBlock, 0, cellBlock.length);
                cellBlockScanner = this.rpcClient.cellBlockBuilder.createCellScanner(this.codec, this.compressor, cellBlock);
            }
            call.setResponse(value, cellBlockScanner);
            call.callStats.setResponseSizeBytes(totalSize);
            call.callStats.setCallTimeMs(EnvironmentEdgeManager.currentTime() - call.callStats.getStartTime());
        }
    } catch (IOException e) {
        if (expectedCall) {
            call.setException(e);
        }
        if (e instanceof SocketTimeoutException) {
            // {@link ConnectionId#rpcTimeout}.
            if (LOG.isTraceEnabled()) {
                LOG.trace("ignored", e);
            }
        } else {
            synchronized (this) {
                closeConn(e);
            }
        }
    }
}