Search in sources :

Example 16 with RegexStringComparator

use of org.apache.hadoop.hbase.filter.RegexStringComparator in project hive by apache.

the class HBaseReadWrite method scanTables.

/**
   * Get a list of tables.
   * @param dbName Database these tables are in
   * @param regex Regular expression to use in searching for table names.  It is expected to
   *              be a Java regular expression.  If it is null then all tables in the indicated
   *              database will be returned.
   * @return list of tables matching the regular expression.
   * @throws IOException
   */
List<Table> scanTables(String dbName, String regex) throws IOException {
    // There's no way to know whether all the tables we are looking for are
    // in the cache, so we would need to scan one way or another.  Thus there's no value in hitting
    // the cache for this function.
    byte[] keyPrefix = null;
    if (dbName != null) {
        keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(dbName);
    }
    Filter filter = null;
    if (regex != null) {
        filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
    }
    Iterator<Result> iter = scan(TABLE_TABLE, keyPrefix, HBaseUtils.getEndPrefix(keyPrefix), CATALOG_CF, CATALOG_COL, filter);
    List<Table> tables = new ArrayList<>();
    while (iter.hasNext()) {
        Result result = iter.next();
        HBaseUtils.StorageDescriptorParts sdParts = HBaseUtils.deserializeTable(result.getRow(), result.getValue(CATALOG_CF, CATALOG_COL));
        StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
        HBaseUtils.assembleStorageDescriptor(sd, sdParts);
        tables.add(sdParts.containingTable);
    }
    return tables;
}
Also used : RegexStringComparator(org.apache.hadoop.hbase.filter.RegexStringComparator) RowFilter(org.apache.hadoop.hbase.filter.RowFilter) Table(org.apache.hadoop.hive.metastore.api.Table) RowFilter(org.apache.hadoop.hbase.filter.RowFilter) FirstKeyOnlyFilter(org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter) Filter(org.apache.hadoop.hbase.filter.Filter) CompareFilter(org.apache.hadoop.hbase.filter.CompareFilter) BloomFilter(org.apache.hive.common.util.BloomFilter) ArrayList(java.util.ArrayList) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) Result(org.apache.hadoop.hbase.client.Result)

Example 17 with RegexStringComparator

use of org.apache.hadoop.hbase.filter.RegexStringComparator in project hive by apache.

the class HBaseReadWrite method printFunctions.

/**
   * Print out functions
   * @param regex regular expression to use in matching functions
   * @return list of strings, one function each
   * @throws IOException
   * @throws TException
   */
List<String> printFunctions(String regex) throws IOException, TException {
    Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
    Iterator<Result> iter = scan(FUNC_TABLE, null, null, CATALOG_CF, CATALOG_COL, filter);
    List<String> lines = new ArrayList<>();
    while (iter.hasNext()) {
        Result result = iter.next();
        lines.add(dumpThriftObject(HBaseUtils.deserializeFunction(result.getRow(), result.getValue(CATALOG_CF, CATALOG_COL))));
    }
    if (lines.size() == 0)
        lines = noMatch(regex, "function");
    return lines;
}
Also used : RegexStringComparator(org.apache.hadoop.hbase.filter.RegexStringComparator) RowFilter(org.apache.hadoop.hbase.filter.RowFilter) RowFilter(org.apache.hadoop.hbase.filter.RowFilter) FirstKeyOnlyFilter(org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter) Filter(org.apache.hadoop.hbase.filter.Filter) CompareFilter(org.apache.hadoop.hbase.filter.CompareFilter) BloomFilter(org.apache.hive.common.util.BloomFilter) ArrayList(java.util.ArrayList) Result(org.apache.hadoop.hbase.client.Result)

Example 18 with RegexStringComparator

use of org.apache.hadoop.hbase.filter.RegexStringComparator in project hbase by apache.

the class Export method getExportFilter.

private static Filter getExportFilter(String[] args) {
    Filter exportFilter = null;
    String filterCriteria = (args.length > 5) ? args[5] : null;
    if (filterCriteria == null)
        return null;
    if (filterCriteria.startsWith("^")) {
        String regexPattern = filterCriteria.substring(1, filterCriteria.length());
        exportFilter = new RowFilter(CompareOp.EQUAL, new RegexStringComparator(regexPattern));
    } else {
        exportFilter = new PrefixFilter(Bytes.toBytesBinary(filterCriteria));
    }
    return exportFilter;
}
Also used : RegexStringComparator(org.apache.hadoop.hbase.filter.RegexStringComparator) RowFilter(org.apache.hadoop.hbase.filter.RowFilter) PrefixFilter(org.apache.hadoop.hbase.filter.PrefixFilter) PrefixFilter(org.apache.hadoop.hbase.filter.PrefixFilter) RowFilter(org.apache.hadoop.hbase.filter.RowFilter) Filter(org.apache.hadoop.hbase.filter.Filter)

Example 19 with RegexStringComparator

use of org.apache.hadoop.hbase.filter.RegexStringComparator in project hbase by apache.

the class AccessControlLists method removeTablePermissions.

private static void removeTablePermissions(TableName tableName, byte[] column, Table table, boolean closeTable) throws IOException {
    Scan scan = new Scan();
    scan.addFamily(ACL_LIST_FAMILY);
    String columnName = Bytes.toString(column);
    scan.setFilter(new QualifierFilter(CompareOp.EQUAL, new RegexStringComparator(String.format("(%s%s%s)|(%s%s)$", ACL_KEY_DELIMITER, columnName, ACL_KEY_DELIMITER, ACL_KEY_DELIMITER, columnName))));
    Set<byte[]> qualifierSet = new TreeSet<>(Bytes.BYTES_COMPARATOR);
    ResultScanner scanner = null;
    try {
        scanner = table.getScanner(scan);
        for (Result res : scanner) {
            for (byte[] q : res.getFamilyMap(ACL_LIST_FAMILY).navigableKeySet()) {
                qualifierSet.add(q);
            }
        }
        if (qualifierSet.size() > 0) {
            Delete d = new Delete(tableName.getName());
            for (byte[] qualifier : qualifierSet) {
                d.addColumns(ACL_LIST_FAMILY, qualifier);
            }
            table.delete(d);
        }
    } finally {
        if (scanner != null)
            scanner.close();
        if (closeTable)
            table.close();
    }
}
Also used : RegexStringComparator(org.apache.hadoop.hbase.filter.RegexStringComparator) Delete(org.apache.hadoop.hbase.client.Delete) ResultScanner(org.apache.hadoop.hbase.client.ResultScanner) TreeSet(java.util.TreeSet) Scan(org.apache.hadoop.hbase.client.Scan) QualifierFilter(org.apache.hadoop.hbase.filter.QualifierFilter) Result(org.apache.hadoop.hbase.client.Result)

Example 20 with RegexStringComparator

use of org.apache.hadoop.hbase.filter.RegexStringComparator in project drill by apache.

the class MapRDBFilterBuilder method createHBaseScanSpec.

private HBaseScanSpec createHBaseScanSpec(FunctionCall call, CompareFunctionsProcessor processor) {
    String functionName = processor.getFunctionName();
    SchemaPath field = processor.getPath();
    byte[] fieldValue = processor.getValue();
    boolean sortOrderAscending = processor.isSortOrderAscending();
    boolean isRowKey = field.getAsUnescapedPath().equals(ROW_KEY);
    if (!(isRowKey || (!field.getRootSegment().isLastPath() && field.getRootSegment().getChild().isLastPath() && field.getRootSegment().getChild().isNamed()))) {
        /*
       * if the field in this function is neither the row_key nor a qualified HBase column, return.
       */
        return null;
    }
    if (processor.isRowKeyPrefixComparison()) {
        return createRowKeyPrefixScanSpec(call, processor);
    }
    CompareOp compareOp = null;
    boolean isNullTest = false;
    ByteArrayComparable comparator = new BinaryComparator(fieldValue);
    byte[] startRow = HConstants.EMPTY_START_ROW;
    byte[] stopRow = HConstants.EMPTY_END_ROW;
    switch(functionName) {
        case "equal":
            compareOp = CompareOp.EQUAL;
            if (isRowKey) {
                startRow = fieldValue;
                /* stopRow should be just greater than 'value'*/
                stopRow = Arrays.copyOf(fieldValue, fieldValue.length + 1);
                compareOp = CompareOp.EQUAL;
            }
            break;
        case "not_equal":
            compareOp = CompareOp.NOT_EQUAL;
            break;
        case "greater_than_or_equal_to":
            if (sortOrderAscending) {
                compareOp = CompareOp.GREATER_OR_EQUAL;
                if (isRowKey) {
                    startRow = fieldValue;
                }
            } else {
                compareOp = CompareOp.LESS_OR_EQUAL;
                if (isRowKey) {
                    // stopRow should be just greater than 'value'
                    stopRow = Arrays.copyOf(fieldValue, fieldValue.length + 1);
                }
            }
            break;
        case "greater_than":
            if (sortOrderAscending) {
                compareOp = CompareOp.GREATER;
                if (isRowKey) {
                    // startRow should be just greater than 'value'
                    startRow = Arrays.copyOf(fieldValue, fieldValue.length + 1);
                }
            } else {
                compareOp = CompareOp.LESS;
                if (isRowKey) {
                    stopRow = fieldValue;
                }
            }
            break;
        case "less_than_or_equal_to":
            if (sortOrderAscending) {
                compareOp = CompareOp.LESS_OR_EQUAL;
                if (isRowKey) {
                    // stopRow should be just greater than 'value'
                    stopRow = Arrays.copyOf(fieldValue, fieldValue.length + 1);
                }
            } else {
                compareOp = CompareOp.GREATER_OR_EQUAL;
                if (isRowKey) {
                    startRow = fieldValue;
                }
            }
            break;
        case "less_than":
            if (sortOrderAscending) {
                compareOp = CompareOp.LESS;
                if (isRowKey) {
                    stopRow = fieldValue;
                }
            } else {
                compareOp = CompareOp.GREATER;
                if (isRowKey) {
                    // startRow should be just greater than 'value'
                    startRow = Arrays.copyOf(fieldValue, fieldValue.length + 1);
                }
            }
            break;
        case "isnull":
        case "isNull":
        case "is null":
            if (isRowKey) {
                return null;
            }
            isNullTest = true;
            compareOp = CompareOp.EQUAL;
            comparator = new NullComparator();
            break;
        case "isnotnull":
        case "isNotNull":
        case "is not null":
            if (isRowKey) {
                return null;
            }
            compareOp = CompareOp.NOT_EQUAL;
            comparator = new NullComparator();
            break;
        case "like":
            /*
       * Convert the LIKE operand to Regular Expression pattern so that we can
       * apply RegexStringComparator()
       */
            HBaseRegexParser parser = new HBaseRegexParser(call).parse();
            compareOp = CompareOp.EQUAL;
            comparator = new RegexStringComparator(parser.getRegexString());
            /*
       * We can possibly do better if the LIKE operator is on the row_key
       */
            if (isRowKey) {
                String prefix = parser.getPrefixString();
                if (prefix != null) {
                    /*
           * If there is a literal prefix, it can help us prune the scan to a sub range
           */
                    if (prefix.equals(parser.getLikeString())) {
                        /* The operand value is literal. This turns the LIKE operator to EQUAL operator */
                        startRow = stopRow = fieldValue;
                        compareOp = null;
                    } else {
                        startRow = prefix.getBytes(Charsets.UTF_8);
                        stopRow = startRow.clone();
                        boolean isMaxVal = true;
                        for (int i = stopRow.length - 1; i >= 0; --i) {
                            int nextByteValue = (0xff & stopRow[i]) + 1;
                            if (nextByteValue < 0xff) {
                                stopRow[i] = (byte) nextByteValue;
                                isMaxVal = false;
                                break;
                            } else {
                                stopRow[i] = 0;
                            }
                        }
                        if (isMaxVal) {
                            stopRow = HConstants.EMPTY_END_ROW;
                        }
                    }
                }
            }
            break;
    }
    if (compareOp != null || startRow != HConstants.EMPTY_START_ROW || stopRow != HConstants.EMPTY_END_ROW) {
        Filter filter = null;
        if (isRowKey) {
            if (compareOp != null) {
                filter = new RowFilter(compareOp, comparator);
            }
        } else {
            byte[] family = HBaseUtils.getBytes(field.getRootSegment().getPath());
            byte[] qualifier = HBaseUtils.getBytes(field.getRootSegment().getChild().getNameSegment().getPath());
            filter = new SingleColumnValueFilter(family, qualifier, compareOp, comparator);
            ((SingleColumnValueFilter) filter).setLatestVersionOnly(true);
            if (!isNullTest) {
                ((SingleColumnValueFilter) filter).setFilterIfMissing(true);
            }
        }
        return new HBaseScanSpec(groupScan.getTableName(), startRow, stopRow, filter);
    }
    // else
    return null;
}
Also used : HBaseScanSpec(org.apache.drill.exec.store.hbase.HBaseScanSpec) SingleColumnValueFilter(org.apache.hadoop.hbase.filter.SingleColumnValueFilter) HBaseRegexParser(org.apache.drill.exec.store.hbase.HBaseRegexParser) NullComparator(org.apache.hadoop.hbase.filter.NullComparator) BinaryComparator(org.apache.hadoop.hbase.filter.BinaryComparator) RegexStringComparator(org.apache.hadoop.hbase.filter.RegexStringComparator) ByteArrayComparable(org.apache.hadoop.hbase.filter.ByteArrayComparable) RowFilter(org.apache.hadoop.hbase.filter.RowFilter) SchemaPath(org.apache.drill.common.expression.SchemaPath) RowFilter(org.apache.hadoop.hbase.filter.RowFilter) Filter(org.apache.hadoop.hbase.filter.Filter) SingleColumnValueFilter(org.apache.hadoop.hbase.filter.SingleColumnValueFilter) CompareOp(org.apache.hadoop.hbase.filter.CompareFilter.CompareOp)

Aggregations

RegexStringComparator (org.apache.hadoop.hbase.filter.RegexStringComparator)28 Filter (org.apache.hadoop.hbase.filter.Filter)24 RowFilter (org.apache.hadoop.hbase.filter.RowFilter)24 FirstKeyOnlyFilter (org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter)13 CompareFilter (org.apache.hadoop.hbase.filter.CompareFilter)10 QualifierFilter (org.apache.hadoop.hbase.filter.QualifierFilter)10 Result (org.apache.hadoop.hbase.client.Result)9 BinaryComparator (org.apache.hadoop.hbase.filter.BinaryComparator)9 SingleColumnValueFilter (org.apache.hadoop.hbase.filter.SingleColumnValueFilter)9 Test (org.junit.Test)9 ArrayList (java.util.ArrayList)8 PrefixFilter (org.apache.hadoop.hbase.filter.PrefixFilter)8 Scan (org.apache.hadoop.hbase.client.Scan)7 BloomFilter (org.apache.hive.common.util.BloomFilter)7 ByteArrayComparable (org.apache.hadoop.hbase.filter.ByteArrayComparable)6 CompareOp (org.apache.hadoop.hbase.filter.CompareFilter.CompareOp)6 InclusiveStopFilter (org.apache.hadoop.hbase.filter.InclusiveStopFilter)6 SchemaPath (org.apache.drill.common.expression.SchemaPath)5 NullComparator (org.apache.hadoop.hbase.filter.NullComparator)5 ValueFilter (org.apache.hadoop.hbase.filter.ValueFilter)5