Examples with RegionScanner org.apache.hadoop.hbase.regionserver.RegionScanner used on opensource projects

Example 1 with RegionScanner

use of org.apache.hadoop.hbase.regionserver.RegionScanner in project hbase by apache.

the class TestPrefixTree method testHBASE12817.

@Test
public void testHBASE12817() throws IOException {
    for (int i = 0; i < 100; i++) {
        region.put(new Put(Bytes.toBytes("obj" + (2900 + i))).addColumn(fam, qual1, Bytes.toBytes(i)));
    }
    region.put(new Put(Bytes.toBytes("obj299")).addColumn(fam, qual1, Bytes.toBytes("whatever")));
    region.put(new Put(Bytes.toBytes("obj29")).addColumn(fam, qual1, Bytes.toBytes("whatever")));
    region.put(new Put(Bytes.toBytes("obj2")).addColumn(fam, qual1, Bytes.toBytes("whatever")));
    region.put(new Put(Bytes.toBytes("obj3")).addColumn(fam, qual1, Bytes.toBytes("whatever")));
    region.flush(true);
    Scan scan = new Scan(Bytes.toBytes("obj29995"));
    RegionScanner scanner = region.getScanner(scan);
    List<Cell> cells = new ArrayList<>();
    assertFalse(scanner.next(cells));
    assertArrayEquals(Bytes.toBytes("obj3"), Result.create(cells).getRow());
}

Example 2 with RegionScanner

use of org.apache.hadoop.hbase.regionserver.RegionScanner in project hbase by apache.

the class TestPrefixTree method testHBASE11728.

@Test
public void testHBASE11728() throws Exception {
    Put put = new Put(Bytes.toBytes("a-b-0-0"));
    put.addColumn(fam, qual1, Bytes.toBytes("c1-value"));
    region.put(put);
    put = new Put(row1_bytes);
    put.addColumn(fam, qual1, Bytes.toBytes("c1-value"));
    region.put(put);
    put = new Put(row2_bytes);
    put.addColumn(fam, qual2, Bytes.toBytes("c2-value"));
    region.put(put);
    put = new Put(row3_bytes);
    put.addColumn(fam, qual2, Bytes.toBytes("c2-value-2"));
    region.put(put);
    put = new Put(row4_bytes);
    put.addColumn(fam, qual2, Bytes.toBytes("c2-value-3"));
    region.put(put);
    region.flush(true);
    String[] rows = new String[3];
    rows[0] = row1;
    rows[1] = row2;
    rows[2] = row3;
    byte[][] val = new byte[3][];
    val[0] = Bytes.toBytes("c1-value");
    val[1] = Bytes.toBytes("c2-value");
    val[2] = Bytes.toBytes("c2-value-2");
    Scan scan = new Scan();
    scan.setStartRow(row1_bytes);
    scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
    RegionScanner scanner = region.getScanner(scan);
    List<Cell> cells = new ArrayList<>();
    for (int i = 0; i < 3; i++) {
        assertEquals(i < 2, scanner.next(cells));
        CellScanner cellScanner = Result.create(cells).cellScanner();
        while (cellScanner.advance()) {
            assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner.current().getRowOffset(), cellScanner.current().getRowLength()));
            assertEquals(Bytes.toString(val[i]), Bytes.toString(cellScanner.current().getValueArray(), cellScanner.current().getValueOffset(), cellScanner.current().getValueLength()));
        }
        cells.clear();
    }
    scanner.close();
    // Add column
    scan = new Scan();
    scan.addColumn(fam, qual2);
    scan.setStartRow(row1_bytes);
    scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
    scanner = region.getScanner(scan);
    for (int i = 1; i < 3; i++) {
        assertEquals(i < 2, scanner.next(cells));
        CellScanner cellScanner = Result.create(cells).cellScanner();
        while (cellScanner.advance()) {
            assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner.current().getRowOffset(), cellScanner.current().getRowLength()));
        }
        cells.clear();
    }
    scanner.close();
    scan = new Scan();
    scan.addColumn(fam, qual2);
    scan.setStartRow(Bytes.toBytes("a-b-A-1-"));
    scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
    scanner = region.getScanner(scan);
    for (int i = 1; i < 3; i++) {
        assertEquals(i < 2, scanner.next(cells));
        CellScanner cellScanner = Result.create(cells).cellScanner();
        while (cellScanner.advance()) {
            assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner.current().getRowOffset(), cellScanner.current().getRowLength()));
        }
        cells.clear();
    }
    scanner.close();
    scan = new Scan();
    scan.addColumn(fam, qual2);
    scan.setStartRow(Bytes.toBytes("a-b-A-1-140239"));
    scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
    scanner = region.getScanner(scan);
    assertFalse(scanner.next(cells));
    assertFalse(cells.isEmpty());
    scanner.close();
}

Example 3 with RegionScanner

use of org.apache.hadoop.hbase.regionserver.RegionScanner in project hadoop by apache.

the class FlowRunCoprocessor method preGetOp.

/*
   * (non-Javadoc)
   *
   * Creates a {@link FlowScanner} Scan so that it can correctly process the
   * contents of {@link FlowRunTable}.
   *
   * @see
   * org.apache.hadoop.hbase.coprocessor.BaseRegionObserver#preGetOp(org.apache
   * .hadoop.hbase.coprocessor.ObserverContext,
   * org.apache.hadoop.hbase.client.Get, java.util.List)
   */
@Override
public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, List<Cell> results) throws IOException {
    if (!isFlowRunRegion) {
        return;
    }
    Scan scan = new Scan(get);
    scan.setMaxVersions();
    RegionScanner scanner = null;
    try {
        scanner = new FlowScanner(e.getEnvironment(), scan, region.getScanner(scan), FlowScannerOperation.READ);
        scanner.next(results);
        e.bypass();
    } finally {
        if (scanner != null) {
            scanner.close();
        }
    }
}

Example 4 with RegionScanner

use of org.apache.hadoop.hbase.regionserver.RegionScanner in project phoenix by apache.

the class RegionScannerFactory method getWrappedScanner.

/**
   * Return wrapped scanner that catches unexpected exceptions (i.e. Phoenix bugs) and
   * re-throws as DoNotRetryIOException to prevent needless retrying hanging the query
   * for 30 seconds. Unfortunately, until HBASE-7481 gets fixed, there's no way to do
   * the same from a custom filter.
   * @param arrayKVRefs
   * @param arrayFuncRefs
   * @param offset starting position in the rowkey.
   * @param scan
   * @param tupleProjector
   * @param dataRegion
   * @param indexMaintainer
   * @param tx current transaction
   * @param viewConstants
   */
public RegionScanner getWrappedScanner(final RegionCoprocessorEnvironment env, final RegionScanner s, final Set<KeyValueColumnExpression> arrayKVRefs, final Expression[] arrayFuncRefs, final int offset, final Scan scan, final ColumnReference[] dataColumns, final TupleProjector tupleProjector, final Region dataRegion, final IndexMaintainer indexMaintainer, Transaction tx, final byte[][] viewConstants, final KeyValueSchema kvSchema, final ValueBitSet kvSchemaBitSet, final TupleProjector projector, final ImmutableBytesWritable ptr, final boolean useQualifierAsListIndex) {
    return new RegionScanner() {

        private boolean hasReferences = checkForReferenceFiles();

        private HRegionInfo regionInfo = env.getRegionInfo();

        private byte[] actualStartKey = getActualStartKey();

        // If there are any reference files after local index region merge some cases we might
        // get the records less than scan start row key. This will happen when we replace the
        // actual region start key with merge region start key. This method gives whether are
        // there any reference files in the region or not.
        private boolean checkForReferenceFiles() {
            if (!ScanUtil.isLocalIndex(scan))
                return false;
            for (byte[] family : scan.getFamilies()) {
                if (getRegion().getStore(family).hasReferences()) {
                    return true;
                }
            }
            return false;
        }

        // Get the actual scan start row of local index. This will be used to compare the row
        // key of the results less than scan start row when there are references.
        public byte[] getActualStartKey() {
            return ScanUtil.isLocalIndex(scan) ? ScanUtil.getActualStartRow(scan, regionInfo) : null;
        }

        @Override
        public boolean next(List<Cell> results) throws IOException {
            try {
                return s.next(results);
            } catch (Throwable t) {
                ServerUtil.throwIOException(getRegion().getRegionInfo().getRegionNameAsString(), t);
                // impossible
                return false;
            }
        }

        @Override
        public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
            try {
                return s.next(result, scannerContext);
            } catch (Throwable t) {
                ServerUtil.throwIOException(getRegion().getRegionInfo().getRegionNameAsString(), t);
                // impossible
                return false;
            }
        }

        @Override
        public void close() throws IOException {
            s.close();
        }

        @Override
        public HRegionInfo getRegionInfo() {
            return s.getRegionInfo();
        }

        @Override
        public boolean isFilterDone() throws IOException {
            return s.isFilterDone();
        }

        @Override
        public boolean reseek(byte[] row) throws IOException {
            return s.reseek(row);
        }

        @Override
        public long getMvccReadPoint() {
            return s.getMvccReadPoint();
        }

        @Override
        public boolean nextRaw(List<Cell> result) throws IOException {
            try {
                boolean next = s.nextRaw(result);
                Cell arrayElementCell = null;
                if (result.size() == 0) {
                    return next;
                }
                if (arrayFuncRefs != null && arrayFuncRefs.length > 0 && arrayKVRefs.size() > 0) {
                    int arrayElementCellPosition = replaceArrayIndexElement(arrayKVRefs, arrayFuncRefs, result);
                    arrayElementCell = result.get(arrayElementCellPosition);
                }
                if (ScanUtil.isLocalIndex(scan) && !ScanUtil.isAnalyzeTable(scan)) {
                    if (hasReferences && actualStartKey != null) {
                        next = scanTillScanStartRow(s, arrayKVRefs, arrayFuncRefs, result, null, arrayElementCell);
                        if (result.isEmpty()) {
                            return next;
                        }
                    }
                    /* In the following, c is only used when data region is null.
            dataRegion will never be null in case of non-coprocessor call,
            therefore no need to refactor
             */
                    IndexUtil.wrapResultUsingOffset(env, result, offset, dataColumns, tupleProjector, dataRegion, indexMaintainer, viewConstants, ptr);
                }
                if (projector != null) {
                    Tuple toProject = useQualifierAsListIndex ? new PositionBasedResultTuple(result) : new ResultTuple(Result.create(result));
                    Tuple tuple = projector.projectResults(toProject, useNewValueColumnQualifier);
                    result.clear();
                    result.add(tuple.getValue(0));
                    if (arrayElementCell != null) {
                        result.add(arrayElementCell);
                    }
                }
                // There is a scanattribute set to retrieve the specific array element
                return next;
            } catch (Throwable t) {
                ServerUtil.throwIOException(getRegion().getRegionInfo().getRegionNameAsString(), t);
                // impossible
                return false;
            }
        }

        @Override
        public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
            try {
                boolean next = s.nextRaw(result, scannerContext);
                Cell arrayElementCell = null;
                if (result.size() == 0) {
                    return next;
                }
                if (arrayFuncRefs != null && arrayFuncRefs.length > 0 && arrayKVRefs.size() > 0) {
                    int arrayElementCellPosition = replaceArrayIndexElement(arrayKVRefs, arrayFuncRefs, result);
                    arrayElementCell = result.get(arrayElementCellPosition);
                }
                if ((offset > 0 || ScanUtil.isLocalIndex(scan)) && !ScanUtil.isAnalyzeTable(scan)) {
                    if (hasReferences && actualStartKey != null) {
                        next = scanTillScanStartRow(s, arrayKVRefs, arrayFuncRefs, result, scannerContext, arrayElementCell);
                        if (result.isEmpty()) {
                            return next;
                        }
                    }
                    /* In the following, c is only used when data region is null.
            dataRegion will never be null in case of non-coprocessor call,
            therefore no need to refactor
             */
                    IndexUtil.wrapResultUsingOffset(env, result, offset, dataColumns, tupleProjector, dataRegion, indexMaintainer, viewConstants, ptr);
                }
                if (projector != null) {
                    Tuple toProject = useQualifierAsListIndex ? new PositionBasedMultiKeyValueTuple(result) : new ResultTuple(Result.create(result));
                    Tuple tuple = projector.projectResults(toProject, useNewValueColumnQualifier);
                    result.clear();
                    result.add(tuple.getValue(0));
                    if (arrayElementCell != null)
                        result.add(arrayElementCell);
                }
                // There is a scanattribute set to retrieve the specific array element
                return next;
            } catch (Throwable t) {
                ServerUtil.throwIOException(getRegion().getRegionInfo().getRegionNameAsString(), t);
                // impossible
                return false;
            }
        }

        /**
       * When there is a merge in progress while scanning local indexes we might get the key values less than scan start row.
       * In that case we need to scan until get the row key more or  equal to scan start key.
       * TODO try to fix this case in LocalIndexStoreFileScanner when there is a merge.
       */
        private boolean scanTillScanStartRow(final RegionScanner s, final Set<KeyValueColumnExpression> arrayKVRefs, final Expression[] arrayFuncRefs, List<Cell> result, ScannerContext scannerContext, Cell arrayElementCell) throws IOException {
            boolean next = true;
            Cell firstCell = result.get(0);
            while (Bytes.compareTo(firstCell.getRowArray(), firstCell.getRowOffset(), firstCell.getRowLength(), actualStartKey, 0, actualStartKey.length) < 0) {
                result.clear();
                if (scannerContext == null) {
                    next = s.nextRaw(result);
                } else {
                    next = s.nextRaw(result, scannerContext);
                }
                if (result.isEmpty()) {
                    return next;
                }
                if (arrayFuncRefs != null && arrayFuncRefs.length > 0 && arrayKVRefs.size() > 0) {
                    int arrayElementCellPosition = replaceArrayIndexElement(arrayKVRefs, arrayFuncRefs, result);
                    arrayElementCell = result.get(arrayElementCellPosition);
                }
                firstCell = result.get(0);
            }
            return next;
        }

        private int replaceArrayIndexElement(final Set<KeyValueColumnExpression> arrayKVRefs, final Expression[] arrayFuncRefs, List<Cell> result) {
            // make a copy of the results array here, as we're modifying it below
            MultiKeyValueTuple tuple = new MultiKeyValueTuple(ImmutableList.copyOf(result));
            // The size of both the arrays would be same?
            // Using KeyValueSchema to set and retrieve the value
            // collect the first kv to get the row
            Cell rowKv = result.get(0);
            for (KeyValueColumnExpression kvExp : arrayKVRefs) {
                if (kvExp.evaluate(tuple, ptr)) {
                    ListIterator<Cell> itr = result.listIterator();
                    while (itr.hasNext()) {
                        Cell kv = itr.next();
                        if (Bytes.equals(kvExp.getColumnFamily(), 0, kvExp.getColumnFamily().length, kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength()) && Bytes.equals(kvExp.getColumnQualifier(), 0, kvExp.getColumnQualifier().length, kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength())) {
                            // remove the kv that has the full array values.
                            itr.remove();
                            break;
                        }
                    }
                }
            }
            byte[] value = kvSchema.toBytes(tuple, arrayFuncRefs, kvSchemaBitSet, ptr);
            // Add a dummy kv with the exact value of the array index
            result.add(new KeyValue(rowKv.getRowArray(), rowKv.getRowOffset(), rowKv.getRowLength(), QueryConstants.ARRAY_VALUE_COLUMN_FAMILY, 0, QueryConstants.ARRAY_VALUE_COLUMN_FAMILY.length, QueryConstants.ARRAY_VALUE_COLUMN_QUALIFIER, 0, QueryConstants.ARRAY_VALUE_COLUMN_QUALIFIER.length, HConstants.LATEST_TIMESTAMP, KeyValue.Type.codeToType(rowKv.getTypeByte()), value, 0, value.length));
            return result.size() - 1;
        }

        @Override
        public long getMaxResultSize() {
            return s.getMaxResultSize();
        }

        @Override
        public int getBatch() {
            return s.getBatch();
        }
    };
}

Example 5 with RegionScanner

use of org.apache.hadoop.hbase.regionserver.RegionScanner in project phoenix by apache.

the class TestLocalTableState method testCorrectRollback.

/**
   * Test that we correctly rollback the state of keyvalue
   * @throws Exception
   */
@Test
@SuppressWarnings("unchecked")
public void testCorrectRollback() throws Exception {
    Put m = new Put(row);
    m.add(fam, qual, ts, val);
    // setup mocks
    RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);
    Region region = Mockito.mock(Region.class);
    Mockito.when(env.getRegion()).thenReturn(region);
    RegionScanner scanner = Mockito.mock(RegionScanner.class);
    Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner);
    final byte[] stored = Bytes.toBytes("stored-value");
    final KeyValue storedKv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
    storedKv.setSequenceId(2);
    Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() {

        @Override
        public Boolean answer(InvocationOnMock invocation) throws Throwable {
            List<KeyValue> list = (List<KeyValue>) invocation.getArguments()[0];
            list.add(storedKv);
            return false;
        }
    });
    LocalHBaseState state = new LocalTable(env);
    LocalTableState table = new LocalTableState(env, state, m);
    // add the kvs from the mutation
    KeyValue kv = KeyValueUtil.ensureKeyValue(m.get(fam, qual).get(0));
    kv.setSequenceId(0);
    table.addPendingUpdates(kv);
    // setup the lookup
    ColumnReference col = new ColumnReference(fam, qual);
    table.setCurrentTimestamp(ts);
    // check that the value is there
    Pair<Scanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
    Scanner s = p.getFirst();
    assertEquals("Didn't get the pending mutation's value first", kv, s.next());
    // rollback that value
    table.rollback(Arrays.asList(kv));
    p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
    s = p.getFirst();
    assertEquals("Didn't correctly rollback the row - still found it!", null, s.next());
    Mockito.verify(env, Mockito.times(1)).getRegion();
    Mockito.verify(region, Mockito.times(1)).getScanner(Mockito.any(Scan.class));
}