Examples with ScannerBase org.apache.accumulo.core.client.ScannerBase used on opensource projects

Example 6 with ScannerBase

use of org.apache.accumulo.core.client.ScannerBase in project incubator-rya by apache.

the class AccumuloTemporalIndexer method queryInstantAfterInstant.

/**
 * get statements where the date object is after the given queryInstant.
 */
@Override
public CloseableIteration<Statement, QueryEvaluationException> queryInstantAfterInstant(final TemporalInstant queryInstant, final StatementConstraints constraints) throws QueryEvaluationException {
    final Query query = new Query() {

        @Override
        public Range getRange(final KeyParts keyParts) {
            // <-- specific logic
            final Text start = Range.followingPrefix(keyParts.getQueryKey());
            // no constraints                            // <-- specific logic
            Text endAt = null;
            if (keyParts.constraintPrefix != null) {
                endAt = Range.followingPrefix(keyParts.constraintPrefix);
            }
            // System.out.println("Scanning queryInstantAfterInstant from after:" + KeyParts.toHumanString(start) + " up to:" + KeyParts.toHumanString(endAt));
            return new Range(start, true, endAt, false);
        }
    };
    final ScannerBase scanner = query.doQuery(queryInstant, constraints);
    return getContextIteratorWrapper(scanner, constraints.getContext());
}

Example 7 with ScannerBase

use of org.apache.accumulo.core.client.ScannerBase in project incubator-rya by apache.

the class AccumuloRyaQueryEngine method queryWithBindingSet.

@Override
public CloseableIteration<? extends Map.Entry<RyaStatement, BindingSet>, RyaDAOException> queryWithBindingSet(Collection<Map.Entry<RyaStatement, BindingSet>> stmts, AccumuloRdfConfiguration conf) throws RyaDAOException {
    if (conf == null) {
        conf = configuration;
    }
    // query configuration
    Authorizations authorizations = conf.getAuthorizations();
    Long ttl = conf.getTtl();
    Long maxResults = conf.getLimit();
    Integer maxRanges = conf.getMaxRangesForScanner();
    Integer numThreads = conf.getNumThreads();
    // TODO: cannot span multiple tables here
    try {
        Collection<Range> ranges = new HashSet<Range>();
        RangeBindingSetEntries rangeMap = new RangeBindingSetEntries();
        TABLE_LAYOUT layout = null;
        RyaURI context = null;
        TriplePatternStrategy strategy = null;
        RyaURI columnFamily = null;
        boolean columnFamilySet = false;
        for (Map.Entry<RyaStatement, BindingSet> stmtbs : stmts) {
            RyaStatement stmt = stmtbs.getKey();
            context = stmt.getContext();
            // Scanner will fetch all ColumnFamilies.
            if (!columnFamilySet) {
                columnFamily = context;
                columnFamilySet = true;
            } else if (columnFamily != null && !columnFamily.equals(context)) {
                columnFamily = null;
            }
            BindingSet bs = stmtbs.getValue();
            strategy = ryaContext.retrieveStrategy(stmt);
            if (strategy == null) {
                throw new IllegalArgumentException("TriplePattern[" + stmt + "] not supported");
            }
            Map.Entry<RdfCloudTripleStoreConstants.TABLE_LAYOUT, ByteRange> entry = strategy.defineRange(stmt.getSubject(), stmt.getPredicate(), stmt.getObject(), stmt.getContext(), conf);
            // use range to set scanner
            // populate scanner based on authorizations, ttl
            layout = entry.getKey();
            ByteRange byteRange = entry.getValue();
            Range range = new Range(new Text(byteRange.getStart()), new Text(byteRange.getEnd()));
            Range rangeMapRange = range;
            // as the Value specified in the BindingSet
            if (context != null) {
                byte[] contextBytes = context.getData().getBytes("UTF-8");
                rangeMapRange = range.bound(new Column(contextBytes, new byte[] { (byte) 0x00 }, new byte[] { (byte) 0x00 }), new Column(contextBytes, new byte[] { (byte) 0xff }, new byte[] { (byte) 0xff }));
            }
            // ranges gets a Range that has no Column bounds, but
            // rangeMap gets a Range that does have Column bounds
            // If we inserted multiple Ranges with the same Row (but
            // distinct Column bounds) into the Set ranges, we would get
            // duplicate
            // results when the Row is not exact. So RyaStatements that
            // differ only in their context are all mapped to the same
            // Range (with no Column bounds) for scanning purposes.
            // However, context information is included in a Column that
            // bounds the Range inserted into rangeMap. This is because
            // in the class {@link RyaStatementBindingSetKeyValueIterator},
            // the rangeMap is
            // used to join the scan results with the BindingSets to produce
            // the query results. The additional ColumnFamily info is
            // required in this join
            // process to allow for the Statement contexts to be compared
            // with the BindingSet contexts
            // See {@link RangeBindingSetEntries#containsKey}.
            ranges.add(range);
            rangeMap.put(rangeMapRange, bs);
        }
        // no ranges. if strategy alone is null, it would be thrown in the loop above.
        if (layout == null || strategy == null) {
            return null;
        }
        String regexSubject = conf.getRegexSubject();
        String regexPredicate = conf.getRegexPredicate();
        String regexObject = conf.getRegexObject();
        TripleRowRegex tripleRowRegex = strategy.buildRegex(regexSubject, regexPredicate, regexObject, null, null);
        String table = layoutToTable(layout, conf);
        boolean useBatchScanner = ranges.size() > maxRanges;
        RyaStatementBindingSetKeyValueIterator iterator = null;
        if (useBatchScanner) {
            ScannerBase scanner = connector.createBatchScanner(table, authorizations, numThreads);
            ((BatchScanner) scanner).setRanges(ranges);
            fillScanner(scanner, columnFamily, null, ttl, null, tripleRowRegex, conf);
            iterator = new RyaStatementBindingSetKeyValueIterator(layout, ryaContext, scanner, rangeMap);
        } else {
            Scanner scannerBase = null;
            Iterator<Map.Entry<Key, Value>>[] iters = new Iterator[ranges.size()];
            int i = 0;
            for (Range range : ranges) {
                scannerBase = connector.createScanner(table, authorizations);
                scannerBase.setRange(range);
                fillScanner(scannerBase, columnFamily, null, ttl, null, tripleRowRegex, conf);
                iters[i] = scannerBase.iterator();
                i++;
            }
            iterator = new RyaStatementBindingSetKeyValueIterator(layout, Iterators.concat(iters), rangeMap, ryaContext);
        }
        if (maxResults != null) {
            iterator.setMaxResults(maxResults);
        }
        return iterator;
    } catch (Exception e) {
        throw new RyaDAOException(e);
    }
}

Example 8 with ScannerBase

use of org.apache.accumulo.core.client.ScannerBase in project vertexium by visallo.

the class OverflowIntoHdfsStreamingPropertyValueStorageStrategy method streamingPropertyValueTableDatas.

private Map<String, byte[]> streamingPropertyValueTableDatas(List<String> dataRowKeys) {
    try {
        if (dataRowKeys.size() == 0) {
            return Collections.emptyMap();
        }
        List<org.apache.accumulo.core.data.Range> ranges = dataRowKeys.stream().map(RangeUtils::createRangeFromString).collect(Collectors.toList());
        final long timerStartTime = System.currentTimeMillis();
        ScannerBase scanner = graph.createBatchScanner(graph.getDataTableName(), ranges, new org.apache.accumulo.core.security.Authorizations());
        graph.getGraphLogger().logStartIterator(scanner);
        Span trace = Trace.start("streamingPropertyValueTableData");
        trace.data("dataRowKeyCount", Integer.toString(dataRowKeys.size()));
        try {
            Map<String, byte[]> results = new HashMap<>();
            for (Map.Entry<Key, Value> col : scanner) {
                results.put(col.getKey().getRow().toString(), col.getValue().get());
            }
            return results;
        } finally {
            scanner.close();
            trace.stop();
            graph.getGraphLogger().logEndIterator(System.currentTimeMillis() - timerStartTime);
        }
    } catch (Exception ex) {
        throw new VertexiumException(ex);
    }
}

Example 9 with ScannerBase

use of org.apache.accumulo.core.client.ScannerBase in project vertexium by visallo.

the class StreamingPropertyValueTable method streamingPropertyValueTableData.

public byte[] streamingPropertyValueTableData(String dataRowKey, Long timestamp) {
    try {
        List<Range> ranges = Lists.newArrayList(RangeUtils.createRangeFromString(dataRowKey));
        long timerStartTime = System.currentTimeMillis();
        ScannerBase scanner = graph.createBatchScanner(graph.getDataTableName(), ranges, new org.apache.accumulo.core.security.Authorizations());
        if (timestamp != null && !DataTableRowKey.isLegacy(dataRowKey)) {
            IteratorSetting iteratorSetting = new IteratorSetting(80, TimestampFilter.class.getSimpleName(), TimestampFilter.class);
            TimestampFilter.setStart(iteratorSetting, timestamp, true);
            TimestampFilter.setEnd(iteratorSetting, timestamp, true);
            scanner.addScanIterator(iteratorSetting);
        }
        GRAPH_LOGGER.logStartIterator(scanner);
        Span trace = Trace.start("streamingPropertyValueTableData");
        trace.data("dataRowKeyCount", Integer.toString(1));
        try {
            byte[] result = null;
            for (Map.Entry<Key, Value> col : scanner) {
                String foundKey = col.getKey().getRow().toString();
                byte[] value = col.getValue().get();
                if (foundKey.equals(dataRowKey)) {
                    result = value;
                }
            }
            if (result == null) {
                throw new VertexiumException("Could not find data with key: " + dataRowKey);
            }
            return result;
        } finally {
            scanner.close();
            trace.stop();
            GRAPH_LOGGER.logEndIterator(System.currentTimeMillis() - timerStartTime);
        }
    } catch (Exception ex) {
        throw new VertexiumException(ex);
    }
}

Example 10 with ScannerBase

use of org.apache.accumulo.core.client.ScannerBase in project incubator-rya by apache.

the class AccumuloTemporalIndexer method queryInstantEqualsInstant.

/**
 * statements where the datetime is exactly the same as the queryInstant.
 */
@Override
public CloseableIteration<Statement, QueryEvaluationException> queryInstantEqualsInstant(final TemporalInstant queryInstant, final StatementConstraints constraints) throws QueryEvaluationException {
    // get rows where the repository time is equal to the given time in queryInstant.
    final Query query = new Query() {

        @Override
        public Range getRange(final KeyParts keyParts) {
            // <-- specific logic
            return Range.prefix(keyParts.getQueryKey());
        }
    };
    final ScannerBase scanner = query.doQuery(queryInstant, constraints);
    // TODO currently context constraints are filtered on the client.
    return getContextIteratorWrapper(scanner, constraints.getContext());
}