Examples with CloseableIteration org.eclipse.rdf4j.common.iteration.CloseableIteration used on opensource projects

Example 1 with CloseableIteration

use of org.eclipse.rdf4j.common.iteration.CloseableIteration in project incubator-rya by apache.

the class InMemoryQueryRepository method updateCache.

/**
 * Updates the {@link #queriesCache} to reflect the latest position within the {@link #changeLog}.
 */
private void updateCache() {
    log.trace("updateCache() - Enter");
    CloseableIteration<ChangeLogEntry<QueryChange>, QueryChangeLogException> it = null;
    try {
        // Iterate over everything since the last position that was handled within the change log.
        log.debug("Starting cache position:" + cachePosition);
        if (cachePosition.isPresent()) {
            it = changeLog.readFromPosition(cachePosition.get() + 1);
        } else {
            it = changeLog.readFromStart();
        }
        // Apply each change to the cache.
        while (it.hasNext()) {
            final ChangeLogEntry<QueryChange> entry = it.next();
            final QueryChange change = entry.getEntry();
            final UUID queryId = change.getQueryId();
            log.debug("Updating the cache to reflect:\n" + change);
            switch(change.getChangeType()) {
                case CREATE:
                    final StreamsQuery query = new StreamsQuery(queryId, change.getSparql().get(), change.getIsActive().get(), change.getIsInsert().get());
                    queriesCache.put(queryId, query);
                    break;
                case UPDATE:
                    if (queriesCache.containsKey(queryId)) {
                        final StreamsQuery old = queriesCache.get(queryId);
                        final StreamsQuery updated = new StreamsQuery(old.getQueryId(), old.getSparql(), change.getIsActive().get(), old.isInsert());
                        queriesCache.put(queryId, updated);
                    }
                    break;
                case DELETE:
                    queriesCache.remove(queryId);
                    break;
            }
            log.debug("Notifying listeners with the updated state.");
            final Optional<StreamsQuery> newQueryState = Optional.ofNullable(queriesCache.get(queryId));
            listeners.forEach(listener -> listener.notify(entry, newQueryState));
            cachePosition = Optional.of(entry.getPosition());
            log.debug("New cache position: " + cachePosition);
        }
    } catch (final QueryChangeLogException e) {
        // Rethrow the exception because the object the supplier tried to create could not be created.
        throw new RuntimeException("Could not update the cache of " + InMemoryQueryRepository.class.getName(), e);
    } finally {
        // Try to close the iteration if it was opened.
        try {
            if (it != null) {
                it.close();
            }
        } catch (final QueryChangeLogException e) {
            log.error("Could not close the " + CloseableIteration.class.getName(), e);
        }
        log.trace("updateCache() - Exit");
    }
}

Example 2 with CloseableIteration

use of org.eclipse.rdf4j.common.iteration.CloseableIteration in project incubator-rya by apache.

the class RdfCloudTripleStoreConnection method evaluateInternal.

@Override
protected CloseableIteration<? extends BindingSet, QueryEvaluationException> evaluateInternal(TupleExpr tupleExpr, final Dataset dataset, BindingSet bindings, final boolean flag) throws SailException {
    verifyIsOpen();
    logger.trace("Incoming query model:\n{}", tupleExpr.toString());
    if (provenanceCollector != null) {
        try {
            provenanceCollector.recordQuery(tupleExpr.toString());
        } catch (final ProvenanceCollectionException e) {
            logger.trace("Provenance failed to record query.", e);
        }
    }
    tupleExpr = tupleExpr.clone();
    final C queryConf = (C) store.getConf().clone();
    if (queryConf == null) {
        // Should not happen, but this is better than a null dereference error.
        throw new SailException("Cloning store.getConf() returned null, aborting.");
    }
    if (bindings != null) {
        final Binding dispPlan = bindings.getBinding(RdfCloudTripleStoreConfiguration.CONF_QUERYPLAN_FLAG);
        if (dispPlan != null) {
            queryConf.setDisplayQueryPlan(Boolean.parseBoolean(dispPlan.getValue().stringValue()));
        }
        final Binding authBinding = bindings.getBinding(RdfCloudTripleStoreConfiguration.CONF_QUERY_AUTH);
        if (authBinding != null) {
            queryConf.setAuths(authBinding.getValue().stringValue().split(","));
        }
        final Binding ttlBinding = bindings.getBinding(RdfCloudTripleStoreConfiguration.CONF_TTL);
        if (ttlBinding != null) {
            queryConf.setTtl(Long.valueOf(ttlBinding.getValue().stringValue()));
        }
        final Binding startTimeBinding = bindings.getBinding(RdfCloudTripleStoreConfiguration.CONF_STARTTIME);
        if (startTimeBinding != null) {
            queryConf.setStartTime(Long.valueOf(startTimeBinding.getValue().stringValue()));
        }
        final Binding performantBinding = bindings.getBinding(RdfCloudTripleStoreConfiguration.CONF_PERFORMANT);
        if (performantBinding != null) {
            queryConf.setBoolean(RdfCloudTripleStoreConfiguration.CONF_PERFORMANT, Boolean.parseBoolean(performantBinding.getValue().stringValue()));
        }
        final Binding inferBinding = bindings.getBinding(RdfCloudTripleStoreConfiguration.CONF_INFER);
        if (inferBinding != null) {
            queryConf.setInfer(Boolean.parseBoolean(inferBinding.getValue().stringValue()));
        }
        final Binding useStatsBinding = bindings.getBinding(RdfCloudTripleStoreConfiguration.CONF_USE_STATS);
        if (useStatsBinding != null) {
            queryConf.setUseStats(Boolean.parseBoolean(useStatsBinding.getValue().stringValue()));
        }
        final Binding offsetBinding = bindings.getBinding(RdfCloudTripleStoreConfiguration.CONF_OFFSET);
        if (offsetBinding != null) {
            queryConf.setOffset(Long.parseLong(offsetBinding.getValue().stringValue()));
        }
        final Binding limitBinding = bindings.getBinding(RdfCloudTripleStoreConfiguration.CONF_LIMIT);
        if (limitBinding != null) {
            queryConf.setLimit(Long.parseLong(limitBinding.getValue().stringValue()));
        }
    } else {
        bindings = new QueryBindingSet();
    }
    if (!(tupleExpr instanceof QueryRoot)) {
        tupleExpr = new QueryRoot(tupleExpr);
    }
    try {
        final List<Class<QueryOptimizer>> optimizers = queryConf.getOptimizers();
        final Class<QueryOptimizer> pcjOptimizer = queryConf.getPcjOptimizer();
        if (pcjOptimizer != null) {
            QueryOptimizer opt = null;
            try {
                final Constructor<QueryOptimizer> construct = pcjOptimizer.getDeclaredConstructor();
                opt = construct.newInstance();
            } catch (final Exception e) {
            }
            if (opt == null) {
                throw new NoSuchMethodException("Could not find valid constructor for " + pcjOptimizer.getName());
            }
            if (opt instanceof Configurable) {
                ((Configurable) opt).setConf(conf);
            }
            opt.optimize(tupleExpr, dataset, bindings);
        }
        final ParallelEvaluationStrategyImpl strategy = new ParallelEvaluationStrategyImpl(new StoreTripleSource<C>(queryConf, ryaDAO), inferenceEngine, dataset, queryConf);
        (new BindingAssigner()).optimize(tupleExpr, dataset, bindings);
        (new ConstantOptimizer(strategy)).optimize(tupleExpr, dataset, bindings);
        (new CompareOptimizer()).optimize(tupleExpr, dataset, bindings);
        (new ConjunctiveConstraintSplitter()).optimize(tupleExpr, dataset, bindings);
        (new DisjunctiveConstraintOptimizer()).optimize(tupleExpr, dataset, bindings);
        (new SameTermFilterOptimizer()).optimize(tupleExpr, dataset, bindings);
        (new QueryModelNormalizer()).optimize(tupleExpr, dataset, bindings);
        (new IterativeEvaluationOptimizer()).optimize(tupleExpr, dataset, bindings);
        if (!optimizers.isEmpty()) {
            for (final Class<QueryOptimizer> optclz : optimizers) {
                QueryOptimizer result = null;
                try {
                    final Constructor<QueryOptimizer> meth = optclz.getDeclaredConstructor();
                    result = meth.newInstance();
                } catch (final Exception e) {
                }
                try {
                    final Constructor<QueryOptimizer> meth = optclz.getDeclaredConstructor(EvaluationStrategy.class);
                    result = meth.newInstance(strategy);
                } catch (final Exception e) {
                }
                if (result == null) {
                    throw new NoSuchMethodException("Could not find valid constructor for " + optclz.getName());
                }
                if (result instanceof Configurable) {
                    ((Configurable) result).setConf(conf);
                }
                result.optimize(tupleExpr, dataset, bindings);
            }
        }
        (new FilterOptimizer()).optimize(tupleExpr, dataset, bindings);
        (new OrderLimitOptimizer()).optimize(tupleExpr, dataset, bindings);
        logger.trace("Optimized query model:\n{}", tupleExpr.toString());
        if (queryConf.isInfer() && this.inferenceEngine != null) {
            try {
                tupleExpr.visit(new DomainRangeVisitor(queryConf, inferenceEngine));
                tupleExpr.visit(new SomeValuesFromVisitor(queryConf, inferenceEngine));
                tupleExpr.visit(new AllValuesFromVisitor(queryConf, inferenceEngine));
                tupleExpr.visit(new HasValueVisitor(queryConf, inferenceEngine));
                tupleExpr.visit(new IntersectionOfVisitor(queryConf, inferenceEngine));
                tupleExpr.visit(new ReflexivePropertyVisitor(queryConf, inferenceEngine));
                tupleExpr.visit(new PropertyChainVisitor(queryConf, inferenceEngine));
                tupleExpr.visit(new TransitivePropertyVisitor(queryConf, inferenceEngine));
                tupleExpr.visit(new SymmetricPropertyVisitor(queryConf, inferenceEngine));
                tupleExpr.visit(new InverseOfVisitor(queryConf, inferenceEngine));
                tupleExpr.visit(new SubPropertyOfVisitor(queryConf, inferenceEngine));
                tupleExpr.visit(new SubClassOfVisitor(queryConf, inferenceEngine));
                tupleExpr.visit(new SameAsVisitor(queryConf, inferenceEngine));
                tupleExpr.visit(new OneOfVisitor(queryConf, inferenceEngine));
                tupleExpr.visit(new HasSelfVisitor(queryConf, inferenceEngine));
            } catch (final Exception e) {
                logger.error("Error encountered while visiting query node.", e);
            }
        }
        if (queryConf.isPerformant()) {
            tupleExpr.visit(new SeparateFilterJoinsVisitor());
        // tupleExpr.visit(new FilterTimeIndexVisitor(queryConf));
        // tupleExpr.visit(new PartitionFilterTimeIndexVisitor(queryConf));
        }
        final FilterRangeVisitor rangeVisitor = new FilterRangeVisitor(queryConf);
        tupleExpr.visit(rangeVisitor);
        // this has to be done twice to get replace the statementpatterns with the right ranges
        tupleExpr.visit(rangeVisitor);
        EvaluationStatistics stats = null;
        if (!queryConf.isUseStats() && queryConf.isPerformant() || rdfEvalStatsDAO == null) {
            stats = new DefaultStatistics();
        } else if (queryConf.isUseStats()) {
            if (queryConf.isUseSelectivity()) {
                stats = new RdfCloudTripleStoreSelectivityEvaluationStatistics<C>(queryConf, rdfEvalStatsDAO, selectEvalDAO);
            } else {
                stats = new RdfCloudTripleStoreEvaluationStatistics<C>(queryConf, rdfEvalStatsDAO);
            }
        }
        if (stats != null) {
            if (stats instanceof RdfCloudTripleStoreSelectivityEvaluationStatistics) {
                final QueryJoinSelectOptimizer qjso = new QueryJoinSelectOptimizer(stats, selectEvalDAO);
                qjso.optimize(tupleExpr, dataset, bindings);
            } else {
                final QueryJoinOptimizer qjo = new QueryJoinOptimizer(stats);
                // TODO: Make pluggable
                qjo.optimize(tupleExpr, dataset, bindings);
            }
        }
        final CloseableIteration<BindingSet, QueryEvaluationException> iter = strategy.evaluate(tupleExpr, EmptyBindingSet.getInstance());
        final CloseableIteration<BindingSet, QueryEvaluationException> iterWrap = new CloseableIteration<BindingSet, QueryEvaluationException>() {

            @Override
            public void remove() throws QueryEvaluationException {
                iter.remove();
            }

            @Override
            public BindingSet next() throws QueryEvaluationException {
                return iter.next();
            }

            @Override
            public boolean hasNext() throws QueryEvaluationException {
                return iter.hasNext();
            }

            @Override
            public void close() throws QueryEvaluationException {
                iter.close();
                strategy.shutdown();
            }
        };
        return iterWrap;
    } catch (final QueryEvaluationException e) {
        throw new SailException(e);
    } catch (final Exception e) {
        throw new SailException(e);
    }
}

Example 3 with CloseableIteration

use of org.eclipse.rdf4j.common.iteration.CloseableIteration in project incubator-rya by apache.

the class HashJoin method join.

@Override
public CloseableIteration<RyaStatement, RyaDAOException> join(C conf, RyaIRI... preds) throws RyaDAOException {
    ConcurrentHashMap<Map.Entry<RyaIRI, RyaType>, Integer> ht = new ConcurrentHashMap<Map.Entry<RyaIRI, RyaType>, Integer>();
    int count = 0;
    boolean first = true;
    for (RyaIRI pred : preds) {
        count++;
        // query
        CloseableIteration<RyaStatement, RyaDAOException> results = ryaQueryEngine.query(new RyaStatement(null, pred, null), null);
        // add to hashtable
        while (results.hasNext()) {
            RyaStatement next = results.next();
            RyaIRI subject = next.getSubject();
            RyaType object = next.getObject();
            Map.Entry<RyaIRI, RyaType> entry = new RdfCloudTripleStoreUtils.CustomEntry<RyaIRI, RyaType>(subject, object);
            if (!first) {
                if (!ht.containsKey(entry)) {
                    // not in join
                    continue;
                }
            }
            ht.put(entry, count);
        }
        // remove from hashtable values that are under count
        if (first) {
            first = false;
        } else {
            for (Map.Entry<Map.Entry<RyaIRI, RyaType>, Integer> entry : ht.entrySet()) {
                if (entry.getValue() < count) {
                    ht.remove(entry.getKey());
                }
            }
        }
    }
    final Enumeration<Map.Entry<RyaIRI, RyaType>> keys = ht.keys();
    return new CloseableIteration<RyaStatement, RyaDAOException>() {

        @Override
        public void close() throws RyaDAOException {
        }

        @Override
        public boolean hasNext() throws RyaDAOException {
            return keys.hasMoreElements();
        }

        @Override
        public RyaStatement next() throws RyaDAOException {
            Map.Entry<RyaIRI, RyaType> subjObj = keys.nextElement();
            return new RyaStatement(subjObj.getKey(), null, subjObj.getValue());
        }

        @Override
        public void remove() throws RyaDAOException {
            keys.nextElement();
        }
    };
}

Example 4 with CloseableIteration

use of org.eclipse.rdf4j.common.iteration.CloseableIteration in project incubator-rya by apache.

the class IterativeJoin method join.

protected CloseableIteration<RyaStatement, RyaDAOException> join(final CloseableIteration<RyaStatement, RyaDAOException> iteration, final RyaIRI pred) {
    // TODO: configure batch
    // TODO: batch = 1, does not work
    final int batch = 100;
    return new CloseableIteration<RyaStatement, RyaDAOException>() {

        private CloseableIteration<Map.Entry<RyaStatement, BindingSet>, RyaDAOException> query;

        @Override
        public void close() throws RyaDAOException {
            iteration.close();
            if (query != null) {
                query.close();
            }
        }

        @Override
        public boolean hasNext() throws RyaDAOException {
            return !(query == null || !query.hasNext()) || batchNext();
        }

        @Override
        public RyaStatement next() throws RyaDAOException {
            if (query == null || !query.hasNext()) {
                if (!batchNext())
                    return null;
            }
            if (query != null && query.hasNext()) {
                return query.next().getKey();
            } else {
                return null;
            }
        }

        private boolean batchNext() throws RyaDAOException {
            if (!iteration.hasNext()) {
                return false;
            }
            Collection<Map.Entry<RyaStatement, BindingSet>> batchedResults = new ArrayList<Map.Entry<RyaStatement, BindingSet>>();
            for (int i = 0; i < batch && iteration.hasNext(); i++) {
                RyaStatement next = iteration.next();
                batchedResults.add(new RdfCloudTripleStoreUtils.CustomEntry<RyaStatement, BindingSet>(new RyaStatement(next.getSubject(), pred, next.getObject()), null));
            }
            query = ryaQueryEngine.queryWithBindingSet(batchedResults, null);
            return query.hasNext();
        }

        @Override
        public void remove() throws RyaDAOException {
            this.next();
        }
    };
}

Example 5 with CloseableIteration

use of org.eclipse.rdf4j.common.iteration.CloseableIteration in project incubator-rya by apache.

the class EntityTupleSet method evaluate.

@Override
public CloseableIteration<BindingSet, QueryEvaluationException> evaluate(BindingSet bindings) throws QueryEvaluationException {
    // into the remainder of the star query to be evaluated
    if (minCard < 1000 && starQuery.size() > 2 && numberOfSpVars(minSp) == 1 && !starQuery.commonVarConstant()) {
        try {
            RdfCloudTripleStoreConnection conn = getRyaSailConnection();
            CloseableIteration<BindingSet, QueryEvaluationException> sol = (CloseableIteration<BindingSet, QueryEvaluationException>) conn.evaluate(minSp, null, bindings, false);
            Set<BindingSet> bSet = Sets.newHashSet();
            while (sol.hasNext()) {
                // TODO this is not optimal - should check if bindings variables intersect minSp variables
                // creating the following QueryBindingSet is only necessary if no intersection occurs
                QueryBindingSet bs = new QueryBindingSet();
                bs.addAll(sol.next());
                bs.addAll(bindings);
                bSet.add(bs);
            }
            List<StatementPattern> spList = starQuery.getNodes();
            spList.remove(minSp);
            StarQuery sq = new StarQuery(spList);
            conn.close();
            return new EntityTupleSet(sq, conf, true).evaluate(bSet);
        } catch (Exception e) {
            throw new QueryEvaluationException(e);
        }
    } else {
        this.evalOptUsed = true;
        return this.evaluate(Collections.singleton(bindings));
    }
}