Examples with RegionScanner org.apache.hadoop.hbase.regionserver.RegionScanner used on opensource projects

Example 51 with RegionScanner

use of org.apache.hadoop.hbase.regionserver.RegionScanner in project phoenix by apache.

the class GroupedAggregateRegionObserver method scanUnordered.

/**
     * Used for an aggregate query in which the key order does not necessarily match the group by
     * key order. In this case, we must collect all distinct groups within a region into a map,
     * aggregating as we go.
     * @param limit TODO
     */
private RegionScanner scanUnordered(ObserverContext<RegionCoprocessorEnvironment> c, Scan scan, final RegionScanner scanner, final List<Expression> expressions, final ServerAggregators aggregators, long limit) throws IOException {
    if (logger.isDebugEnabled()) {
        logger.debug(LogUtil.addCustomAnnotations("Grouped aggregation over unordered rows with scan " + scan + ", group by " + expressions + ", aggregators " + aggregators, ScanUtil.getCustomAnnotations(scan)));
    }
    RegionCoprocessorEnvironment env = c.getEnvironment();
    Configuration conf = env.getConfiguration();
    int estDistVals = conf.getInt(GROUPBY_ESTIMATED_DISTINCT_VALUES_ATTRIB, DEFAULT_GROUPBY_ESTIMATED_DISTINCT_VALUES);
    byte[] estDistValsBytes = scan.getAttribute(BaseScannerRegionObserver.ESTIMATED_DISTINCT_VALUES);
    if (estDistValsBytes != null) {
        // Allocate 1.5x estimation
        estDistVals = Math.max(MIN_DISTINCT_VALUES, (int) (Bytes.toInt(estDistValsBytes) * 1.5f));
    }
    Pair<Integer, Integer> minMaxQualifiers = EncodedColumnsUtil.getMinMaxQualifiersFromScan(scan);
    boolean useQualifierAsIndex = EncodedColumnsUtil.useQualifierAsIndex(EncodedColumnsUtil.getMinMaxQualifiersFromScan(scan));
    final boolean spillableEnabled = conf.getBoolean(GROUPBY_SPILLABLE_ATTRIB, DEFAULT_GROUPBY_SPILLABLE);
    GroupByCache groupByCache = GroupByCacheFactory.INSTANCE.newCache(env, ScanUtil.getTenantId(scan), ScanUtil.getCustomAnnotations(scan), aggregators, estDistVals);
    boolean success = false;
    try {
        boolean hasMore;
        Tuple result = useQualifierAsIndex ? new PositionBasedMultiKeyValueTuple() : new MultiKeyValueTuple();
        if (logger.isDebugEnabled()) {
            logger.debug(LogUtil.addCustomAnnotations("Spillable groupby enabled: " + spillableEnabled, ScanUtil.getCustomAnnotations(scan)));
        }
        Region region = c.getEnvironment().getRegion();
        boolean acquiredLock = false;
        try {
            region.startRegionOperation();
            acquiredLock = true;
            synchronized (scanner) {
                do {
                    List<Cell> results = useQualifierAsIndex ? new EncodedColumnQualiferCellsList(minMaxQualifiers.getFirst(), minMaxQualifiers.getSecond(), encodingScheme) : new ArrayList<Cell>();
                    // Results are potentially returned even when the return
                    // value of s.next is false
                    // since this is an indication of whether or not there are
                    // more values after the
                    // ones returned
                    hasMore = scanner.nextRaw(results);
                    if (!results.isEmpty()) {
                        result.setKeyValues(results);
                        ImmutableBytesPtr key = TupleUtil.getConcatenatedValue(result, expressions);
                        Aggregator[] rowAggregators = groupByCache.cache(key);
                        // Aggregate values here
                        aggregators.aggregate(rowAggregators, result);
                    }
                } while (hasMore && groupByCache.size() < limit);
            }
        } finally {
            if (acquiredLock)
                region.closeRegionOperation();
        }
        RegionScanner regionScanner = groupByCache.getScanner(scanner);
        // Do not sort here, but sort back on the client instead
        // The reason is that if the scan ever extends beyond a region
        // (which can happen if we're basing our parallelization split
        // points on old metadata), we'll get incorrect query results.
        success = true;
        return regionScanner;
    } finally {
        if (!success) {
            Closeables.closeQuietly(groupByCache);
        }
    }
}

Example 52 with RegionScanner

use of org.apache.hadoop.hbase.regionserver.RegionScanner in project phoenix by apache.

the class MetaDataEndpointImpl method doDropTable.

private MetaDataMutationResult doDropTable(byte[] key, byte[] tenantId, byte[] schemaName, byte[] tableName, byte[] parentTableName, PTableType tableType, List<Mutation> rowsToDelete, List<ImmutableBytesPtr> invalidateList, List<RowLock> locks, List<byte[]> tableNamesToDelete, List<SharedTableState> sharedTablesToDelete, boolean isCascade) throws IOException, SQLException {
    long clientTimeStamp = MetaDataUtil.getClientTimeStamp(rowsToDelete);
    Region region = env.getRegion();
    ImmutableBytesPtr cacheKey = new ImmutableBytesPtr(key);
    Cache<ImmutableBytesPtr, PMetaDataEntity> metaDataCache = GlobalCache.getInstance(this.env).getMetaDataCache();
    PTable table = (PTable) metaDataCache.getIfPresent(cacheKey);
    // We always cache the latest version - fault in if not in cache
    if (table != null || (table = buildTable(key, cacheKey, region, HConstants.LATEST_TIMESTAMP)) != null) {
        if (table.getTimeStamp() < clientTimeStamp) {
            if (isTableDeleted(table) || tableType != table.getType()) {
                return new MetaDataMutationResult(MutationCode.TABLE_NOT_FOUND, EnvironmentEdgeManager.currentTimeMillis(), null);
            }
        } else {
            return new MetaDataMutationResult(MutationCode.NEWER_TABLE_FOUND, EnvironmentEdgeManager.currentTimeMillis(), null);
        }
    }
    // there was a table, but it's been deleted. In either case we want to return.
    if (table == null) {
        if (buildDeletedTable(key, cacheKey, region, clientTimeStamp) != null) {
            return new MetaDataMutationResult(MutationCode.NEWER_TABLE_FOUND, EnvironmentEdgeManager.currentTimeMillis(), null);
        }
        return new MetaDataMutationResult(MutationCode.TABLE_NOT_FOUND, EnvironmentEdgeManager.currentTimeMillis(), null);
    }
    // Make sure we're not deleting the "wrong" child
    if (parentTableName != null && table.getParentTableName() != null && !Arrays.equals(parentTableName, table.getParentTableName().getBytes())) {
        return new MetaDataMutationResult(MutationCode.TABLE_NOT_FOUND, EnvironmentEdgeManager.currentTimeMillis(), null);
    }
    // Since we don't allow back in time DDL, we know if we have a table it's the one
    // we want to delete. FIXME: we shouldn't need a scan here, but should be able to
    // use the table to generate the Delete markers.
    Scan scan = MetaDataUtil.newTableRowsScan(key, MIN_TABLE_TIMESTAMP, clientTimeStamp);
    List<byte[]> indexNames = Lists.newArrayList();
    List<Cell> results = Lists.newArrayList();
    try (RegionScanner scanner = region.getScanner(scan)) {
        scanner.next(results);
        if (results.isEmpty()) {
            // Should not be possible
            return new MetaDataMutationResult(MutationCode.TABLE_NOT_FOUND, EnvironmentEdgeManager.currentTimeMillis(), null);
        }
        if (tableType == PTableType.TABLE || tableType == PTableType.SYSTEM) {
            // Handle any child views that exist
            TableViewFinder tableViewFinderResult = findChildViews(region, tenantId, table);
            if (tableViewFinderResult.hasViews()) {
                if (isCascade) {
                    if (tableViewFinderResult.allViewsInMultipleRegions()) {
                        // view metadata spans multiple regions
                        return new MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION, EnvironmentEdgeManager.currentTimeMillis(), null);
                    } else if (tableViewFinderResult.allViewsInSingleRegion()) {
                        // Recursively delete views - safe as all the views as all in the same region
                        for (ViewInfo viewInfo : tableViewFinderResult.getViewInfoList()) {
                            byte[] viewTenantId = viewInfo.getTenantId();
                            byte[] viewSchemaName = viewInfo.getSchemaName();
                            byte[] viewName = viewInfo.getViewName();
                            byte[] viewKey = SchemaUtil.getTableKey(viewTenantId, viewSchemaName, viewName);
                            Delete delete = new Delete(viewKey, clientTimeStamp);
                            rowsToDelete.add(delete);
                            acquireLock(region, viewKey, locks);
                            MetaDataMutationResult result = doDropTable(viewKey, viewTenantId, viewSchemaName, viewName, null, PTableType.VIEW, rowsToDelete, invalidateList, locks, tableNamesToDelete, sharedTablesToDelete, false);
                            if (result.getMutationCode() != MutationCode.TABLE_ALREADY_EXISTS) {
                                return result;
                            }
                        }
                    }
                } else {
                    // DROP without CASCADE on tables with child views is not permitted
                    return new MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION, EnvironmentEdgeManager.currentTimeMillis(), null);
                }
            }
        }
        // Add to list of HTables to delete, unless it's a view or its a shared index
        if (tableType != PTableType.VIEW && table.getViewIndexId() == null) {
            tableNamesToDelete.add(table.getPhysicalName().getBytes());
        } else {
            sharedTablesToDelete.add(new SharedTableState(table));
        }
        invalidateList.add(cacheKey);
        byte[][] rowKeyMetaData = new byte[5][];
        do {
            Cell kv = results.get(LINK_TYPE_INDEX);
            int nColumns = getVarChars(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), 0, rowKeyMetaData);
            if (nColumns == 5 && rowKeyMetaData[PhoenixDatabaseMetaData.FAMILY_NAME_INDEX].length > 0 && Bytes.compareTo(kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength(), LINK_TYPE_BYTES, 0, LINK_TYPE_BYTES.length) == 0) {
                LinkType linkType = LinkType.fromSerializedValue(kv.getValueArray()[kv.getValueOffset()]);
                if (rowKeyMetaData[PhoenixDatabaseMetaData.COLUMN_NAME_INDEX].length == 0 && linkType == LinkType.INDEX_TABLE) {
                    indexNames.add(rowKeyMetaData[PhoenixDatabaseMetaData.FAMILY_NAME_INDEX]);
                } else if (linkType == LinkType.PARENT_TABLE || linkType == LinkType.PHYSICAL_TABLE) {
                    // delete parent->child link for views
                    Cell parentTenantIdCell = MetaDataUtil.getCell(results, PhoenixDatabaseMetaData.PARENT_TENANT_ID_BYTES);
                    PName parentTenantId = parentTenantIdCell != null ? PNameFactory.newName(parentTenantIdCell.getValueArray(), parentTenantIdCell.getValueOffset(), parentTenantIdCell.getValueLength()) : null;
                    byte[] linkKey = MetaDataUtil.getChildLinkKey(parentTenantId, table.getParentSchemaName(), table.getParentTableName(), table.getTenantId(), table.getName());
                    Delete linkDelete = new Delete(linkKey, clientTimeStamp);
                    rowsToDelete.add(linkDelete);
                }
            }
            // FIXME: Remove when unintentionally deprecated method is fixed (HBASE-7870).
            // FIXME: the version of the Delete constructor without the lock args was introduced
            // in 0.94.4, thus if we try to use it here we can no longer use the 0.94.2 version
            // of the client.
            Delete delete = new Delete(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), clientTimeStamp);
            rowsToDelete.add(delete);
            results.clear();
            scanner.next(results);
        } while (!results.isEmpty());
    }
    // Recursively delete indexes
    for (byte[] indexName : indexNames) {
        byte[] indexKey = SchemaUtil.getTableKey(tenantId, schemaName, indexName);
        // FIXME: Remove when unintentionally deprecated method is fixed (HBASE-7870).
        // FIXME: the version of the Delete constructor without the lock args was introduced
        // in 0.94.4, thus if we try to use it here we can no longer use the 0.94.2 version
        // of the client.
        Delete delete = new Delete(indexKey, clientTimeStamp);
        rowsToDelete.add(delete);
        acquireLock(region, indexKey, locks);
        MetaDataMutationResult result = doDropTable(indexKey, tenantId, schemaName, indexName, tableName, PTableType.INDEX, rowsToDelete, invalidateList, locks, tableNamesToDelete, sharedTablesToDelete, false);
        if (result.getMutationCode() != MutationCode.TABLE_ALREADY_EXISTS) {
            return result;
        }
    }
    return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, EnvironmentEdgeManager.currentTimeMillis(), table, tableNamesToDelete);
}

Example 53 with RegionScanner

use of org.apache.hadoop.hbase.regionserver.RegionScanner in project phoenix by apache.

the class MetaDataEndpointImpl method buildTable.

private PTable buildTable(byte[] key, ImmutableBytesPtr cacheKey, Region region, long clientTimeStamp) throws IOException, SQLException {
    Scan scan = MetaDataUtil.newTableRowsScan(key, MIN_TABLE_TIMESTAMP, clientTimeStamp);
    RegionScanner scanner = region.getScanner(scan);
    Cache<ImmutableBytesPtr, PMetaDataEntity> metaDataCache = GlobalCache.getInstance(this.env).getMetaDataCache();
    try {
        PTable oldTable = (PTable) metaDataCache.getIfPresent(cacheKey);
        long tableTimeStamp = oldTable == null ? MIN_TABLE_TIMESTAMP - 1 : oldTable.getTimeStamp();
        PTable newTable;
        boolean blockWriteRebuildIndex = env.getConfiguration().getBoolean(QueryServices.INDEX_FAILURE_BLOCK_WRITE, QueryServicesOptions.DEFAULT_INDEX_FAILURE_BLOCK_WRITE);
        newTable = getTable(scanner, clientTimeStamp, tableTimeStamp);
        if (newTable == null) {
            return null;
        }
        if (oldTable == null || tableTimeStamp < newTable.getTimeStamp() || (blockWriteRebuildIndex && newTable.getIndexDisableTimestamp() > 0)) {
            if (logger.isDebugEnabled()) {
                logger.debug("Caching table " + Bytes.toStringBinary(cacheKey.get(), cacheKey.getOffset(), cacheKey.getLength()) + " at seqNum " + newTable.getSequenceNumber() + " with newer timestamp " + newTable.getTimeStamp() + " versus " + tableTimeStamp);
            }
            metaDataCache.put(cacheKey, newTable);
        }
        return newTable;
    } finally {
        scanner.close();
    }
}

Example 54 with RegionScanner

use of org.apache.hadoop.hbase.regionserver.RegionScanner in project phoenix by apache.

the class MetaDataEndpointImpl method buildDeletedTable.

private PTable buildDeletedTable(byte[] key, ImmutableBytesPtr cacheKey, Region region, long clientTimeStamp) throws IOException {
    if (clientTimeStamp == HConstants.LATEST_TIMESTAMP) {
        return null;
    }
    Scan scan = MetaDataUtil.newTableRowsScan(key, clientTimeStamp, HConstants.LATEST_TIMESTAMP);
    scan.setFilter(new FirstKeyOnlyFilter());
    scan.setRaw(true);
    List<Cell> results = Lists.<Cell>newArrayList();
    try (RegionScanner scanner = region.getScanner(scan)) {
        scanner.next(results);
    }
    for (Cell kv : results) {
        KeyValue.Type type = Type.codeToType(kv.getTypeByte());
        if (type == Type.DeleteFamily) {
            // Row was deleted
            Cache<ImmutableBytesPtr, PMetaDataEntity> metaDataCache = GlobalCache.getInstance(this.env).getMetaDataCache();
            PTable table = newDeletedTableMarker(kv.getTimestamp());
            metaDataCache.put(cacheKey, table);
            return table;
        }
    }
    return null;
}

Example 55 with RegionScanner

use of org.apache.hadoop.hbase.regionserver.RegionScanner in project phoenix by apache.

the class MetaDataEndpointImpl method buildSchemas.

private List<PSchema> buildSchemas(List<byte[]> keys, Region region, long clientTimeStamp, ImmutableBytesPtr cacheKey) throws IOException, SQLException {
    List<KeyRange> keyRanges = Lists.newArrayListWithExpectedSize(keys.size());
    for (byte[] key : keys) {
        byte[] stopKey = ByteUtil.concat(key, QueryConstants.SEPARATOR_BYTE_ARRAY);
        ByteUtil.nextKey(stopKey, stopKey.length);
        keyRanges.add(PVarbinary.INSTANCE.getKeyRange(key, true, stopKey, false));
    }
    Scan scan = new Scan();
    scan.setTimeRange(MIN_TABLE_TIMESTAMP, clientTimeStamp);
    ScanRanges scanRanges = ScanRanges.createPointLookup(keyRanges);
    scanRanges.initializeScan(scan);
    scan.setFilter(scanRanges.getSkipScanFilter());
    RegionScanner scanner = region.getScanner(scan);
    Cache<ImmutableBytesPtr, PMetaDataEntity> metaDataCache = GlobalCache.getInstance(this.env).getMetaDataCache();
    List<PSchema> schemas = new ArrayList<PSchema>();
    PSchema schema = null;
    try {
        for (int i = 0; i < keys.size(); i++) {
            schema = null;
            schema = getSchema(scanner, clientTimeStamp);
            if (schema == null) {
                return null;
            }
            metaDataCache.put(cacheKey, schema);
            schemas.add(schema);
        }
        return schemas;
    } finally {
        scanner.close();
    }
}