Examples with CloseableIterator org.apache.geode.internal.cache.persistence.query.CloseableIterator used on opensource projects

Example 1 with CloseableIterator

use of org.apache.geode.internal.cache.persistence.query.CloseableIterator in project geode by apache.

the class RangeIndex method queryEquijoinCondition.

// Asif TODO: Provide explanation of the method. Test this method
@Override
public List queryEquijoinCondition(IndexProtocol indx, ExecutionContext context) throws TypeMismatchException, FunctionDomainException, NameResolutionException, QueryInvocationTargetException {
    // get a read lock when doing a lookup
    long start = updateIndexUseStats();
    ((AbstractIndex) indx).updateIndexUseStats();
    List data = new ArrayList();
    Iterator inner = null;
    try {
        // We will iterate over each of the valueToEntries Map to obatin the keys
        // & its correspodning
        // Entry to ResultSet Map
        Iterator outer = this.valueToEntriesMap.entrySet().iterator();
        if (indx instanceof CompactRangeIndex) {
            inner = ((CompactRangeIndex) indx).getIndexStorage().iterator(null);
        } else {
            inner = ((RangeIndex) indx).getValueToEntriesMap().entrySet().iterator();
        }
        Map.Entry outerEntry = null;
        Object innerEntry = null;
        Object outerKey = null;
        Object innerKey = null;
        // boolean incrementOuter = true;
        boolean incrementInner = true;
        outer: while (outer.hasNext()) {
            // if (incrementOuter) {
            outerEntry = (Map.Entry) outer.next();
            // }
            outerKey = outerEntry.getKey();
            while (!incrementInner || inner.hasNext()) {
                if (incrementInner) {
                    innerEntry = inner.next();
                    if (innerEntry instanceof IndexStoreEntry) {
                        innerKey = ((IndexStoreEntry) innerEntry).getDeserializedKey();
                    } else {
                        innerKey = ((Map.Entry) innerEntry).getKey();
                    }
                }
                int compare = ((Comparable) outerKey).compareTo(innerKey);
                if (compare == 0) {
                    // Asif :Select the data
                    // incrementOuter = true;
                    Object innerValue = null;
                    if (innerEntry instanceof IndexStoreEntry) {
                        innerValue = ((CompactRangeIndex) indx).getIndexStorage().get(outerKey);
                    } else {
                        innerValue = ((Map.Entry) innerEntry).getValue();
                    }
                    populateListForEquiJoin(data, outerEntry.getValue(), innerValue, context, null);
                    incrementInner = true;
                    continue outer;
                } else if (compare < 0) {
                    // Asif :The outer key is smaller than the inner key. That means
                    // that we need to increment the outer loop without moving inner loop.
                    // incrementOuter = true;
                    incrementInner = false;
                    continue outer;
                } else {
                    // Asif : The outer key is greater than inner key , so increment the
                    // inner loop without changing outer
                    incrementInner = true;
                }
            }
            break;
        }
        return data;
    } finally {
        ((AbstractIndex) indx).updateIndexUseEndStats(start);
        updateIndexUseEndStats(start);
        if (inner != null && indx instanceof CompactRangeIndex) {
            ((CloseableIterator<IndexStoreEntry>) inner).close();
        }
    }
}

Example 2 with CloseableIterator

use of org.apache.geode.internal.cache.persistence.query.CloseableIterator in project geode by apache.

the class CompactRangeIndex method addToResultsFromEntries.

/*
   * 
   * @param lowerBoundKey the index key to match on for a lower bound on a ranged query, otherwise
   * the key to match on
   * 
   * @param upperBoundKey the index key to match on for an upper bound on a ranged query, otherwise
   * null
   * 
   * @param lowerBoundOperator the operator to use to determine a match against the lower bound
   * 
   * @param upperBoundOperator the operator to use to determine a match against the upper bound
   */
private void addToResultsFromEntries(Object lowerBoundKey, Object upperBoundKey, int lowerBoundOperator, int upperBoundOperator, CloseableIterator<IndexStoreEntry> entriesIter, Collection result, CompiledValue iterOps, RuntimeIterator runtimeItr, ExecutionContext context, List projAttrib, SelectResults intermediateResults, boolean isIntersection, int limit) throws FunctionDomainException, TypeMismatchException, NameResolutionException, QueryInvocationTargetException {
    QueryObserver observer = QueryObserverHolder.getInstance();
    boolean limitApplied = false;
    if (entriesIter == null || (limitApplied = verifyLimit(result, limit))) {
        if (limitApplied) {
            if (observer != null) {
                observer.limitAppliedAtIndexLevel(this, limit, result);
            }
        }
        return;
    }
    Set seenKey = null;
    if (IndexManager.IS_TEST_EXPANSION) {
        seenKey = new HashSet();
    }
    while (entriesIter.hasNext()) {
        try {
            // Check if query execution on this thread is canceled.
            QueryMonitor.isQueryExecutionCanceled();
            if (IndexManager.testHook != null) {
                if (this.region.getCache().getLogger().fineEnabled()) {
                    this.region.getCache().getLogger().fine("IndexManager TestHook is set in addToResultsFromEntries.");
                }
                IndexManager.testHook.hook(11);
            }
            IndexStoreEntry indexEntry = null;
            try {
                indexEntry = entriesIter.next();
            } catch (NoSuchElementException ignore) {
                // Continue from while.
                continue;
            }
            Object value = indexEntry.getDeserializedValue();
            if (IndexManager.IS_TEST_EXPANSION) {
                Object rk = indexEntry.getDeserializedRegionKey();
                if (seenKey.contains(rk)) {
                    continue;
                }
                seenKey.add(rk);
                List expandedResults = expandValue(context, lowerBoundKey, upperBoundKey, lowerBoundOperator, upperBoundOperator, value);
                Iterator iterator = ((Collection) expandedResults).iterator();
                while (iterator.hasNext()) {
                    value = iterator.next();
                    if (value != null) {
                        boolean ok = true;
                        if (runtimeItr != null) {
                            runtimeItr.setCurrent(value);
                        }
                        if (ok && runtimeItr != null && iterOps != null) {
                            ok = QueryUtils.applyCondition(iterOps, context);
                        }
                        if (ok) {
                            if (context != null && context.isCqQueryContext()) {
                                result.add(new CqEntry(indexEntry.getDeserializedRegionKey(), value));
                            } else {
                                applyProjection(projAttrib, context, result, value, intermediateResults, isIntersection);
                            }
                            if (verifyLimit(result, limit)) {
                                observer.limitAppliedAtIndexLevel(this, limit, result);
                                return;
                            }
                        }
                    }
                }
            } else {
                if (value != null) {
                    boolean ok = true;
                    if (indexEntry.isUpdateInProgress() || TEST_ALWAYS_UPDATE_IN_PROGRESS) {
                        IndexInfo indexInfo = (IndexInfo) context.cacheGet(CompiledValue.INDEX_INFO);
                        if (runtimeItr == null) {
                            runtimeItr = getRuntimeIteratorForThisIndex(context, indexInfo);
                            if (runtimeItr == null) {
                                // could not match index with iterator
                                throw new QueryInvocationTargetException("Query alias's must be used consistently");
                            }
                        }
                        runtimeItr.setCurrent(value);
                        // Verify index key in region entry value.
                        ok = evaluateEntry((IndexInfo) indexInfo, context, null);
                    }
                    if (runtimeItr != null) {
                        runtimeItr.setCurrent(value);
                    }
                    if (ok && runtimeItr != null && iterOps != null) {
                        ok = QueryUtils.applyCondition(iterOps, context);
                    }
                    if (ok) {
                        if (context != null && context.isCqQueryContext()) {
                            result.add(new CqEntry(indexEntry.getDeserializedRegionKey(), value));
                        } else {
                            if (IndexManager.testHook != null) {
                                IndexManager.testHook.hook(200);
                            }
                            applyProjection(projAttrib, context, result, value, intermediateResults, isIntersection);
                        }
                        if (verifyLimit(result, limit)) {
                            observer.limitAppliedAtIndexLevel(this, limit, result);
                            return;
                        }
                    }
                }
            }
        } catch (ClassCastException | EntryDestroyedException ignore) {
        // ignore it
        }
    }
}

Example 3 with CloseableIterator

use of org.apache.geode.internal.cache.persistence.query.CloseableIterator in project geode by apache.

the class AbstractIndex method populateListForEquiJoin.

/**
   * This will populate resultSet from both type of indexes, {@link CompactRangeIndex} and
   * {@link RangeIndex}.
   */
void populateListForEquiJoin(List list, Object outerEntries, Object innerEntries, ExecutionContext context, Object key) throws FunctionDomainException, TypeMismatchException, NameResolutionException, QueryInvocationTargetException {
    Assert.assertTrue(outerEntries != null && innerEntries != null, "OuterEntries or InnerEntries must not be null");
    Object[][] values = new Object[2][];
    Iterator itr = null;
    int j = 0;
    while (j < 2) {
        boolean isRangeIndex = false;
        if (j == 0) {
            if (outerEntries instanceof RegionEntryToValuesMap) {
                itr = ((RegionEntryToValuesMap) outerEntries).map.entrySet().iterator();
                isRangeIndex = true;
            } else if (outerEntries instanceof CloseableIterator) {
                itr = (Iterator) outerEntries;
            }
        } else {
            if (innerEntries instanceof RegionEntryToValuesMap) {
                itr = ((RegionEntryToValuesMap) innerEntries).map.entrySet().iterator();
                isRangeIndex = true;
            } else if (innerEntries instanceof CloseableIterator) {
                itr = (Iterator) innerEntries;
            }
        }
        // extract the values from the RegionEntries
        List dummy = new ArrayList();
        RegionEntry re = null;
        IndexStoreEntry ie = null;
        Object val = null;
        Object entryVal = null;
        IndexInfo[] indexInfo = (IndexInfo[]) context.cacheGet(CompiledValue.INDEX_INFO);
        IndexInfo indInfo = indexInfo[j];
        while (itr.hasNext()) {
            if (isRangeIndex) {
                Map.Entry entry = (Map.Entry) itr.next();
                val = entry.getValue();
                if (val instanceof Collection) {
                    entryVal = ((Iterable) val).iterator().next();
                } else {
                    entryVal = val;
                }
                re = (RegionEntry) entry.getKey();
            } else {
                ie = (IndexStoreEntry) itr.next();
            }
            // Bug#41010: We need to verify if Inner and Outer Entries
            // are consistent with index key values.
            boolean ok = true;
            if (isRangeIndex) {
                if (re.isUpdateInProgress()) {
                    ok = ((RangeIndex) indInfo._getIndex()).verifyEntryAndIndexValue(re, entryVal, context);
                }
            } else if (ie.isUpdateInProgress()) {
                ok = ((CompactRangeIndex) indInfo._getIndex()).verifyInnerAndOuterEntryValues(ie, context, indInfo, key);
            }
            if (ok) {
                if (isRangeIndex) {
                    if (val instanceof Collection) {
                        dummy.addAll((Collection) val);
                    } else {
                        dummy.add(val);
                    }
                } else {
                    if (IndexManager.IS_TEST_EXPANSION) {
                        dummy.addAll(((CompactRangeIndex) indInfo._getIndex()).expandValue(context, key, null, OQLLexerTokenTypes.TOK_EQ, -1, ie.getDeserializedValue()));
                    } else {
                        dummy.add(ie.getDeserializedValue());
                    }
                }
            }
        }
        Object[] newValues = new Object[dummy.size()];
        dummy.toArray(newValues);
        values[j++] = newValues;
    }
    list.add(values);
}

Example 4 with CloseableIterator

use of org.apache.geode.internal.cache.persistence.query.CloseableIterator in project geode by apache.

the class CompactRangeIndexJUnitTest method testInvalidTokens.

@Test
public void testInvalidTokens() throws Exception {
    final Region r = utils.getCache().getRegion("/exampleRegion");
    r.put("0", new Portfolio(0));
    r.invalidate("0");
    index = utils.createIndex("compact range index", "p.status", "/exampleRegion p");
    QueryService qs = utils.getCache().getQueryService();
    SelectResults results = (SelectResults) qs.newQuery("Select * from /exampleRegion r where r.status='active'").execute();
    // the remove should have happened
    assertEquals(0, results.size());
    results = (SelectResults) qs.newQuery("Select * from /exampleRegion r where r.status!='inactive'").execute();
    assertEquals(0, results.size());
    CompactRangeIndex cindex = (CompactRangeIndex) index;
    MemoryIndexStore indexStore = (MemoryIndexStore) cindex.getIndexStorage();
    CloseableIterator iterator = indexStore.get(QueryService.UNDEFINED);
    int count = 0;
    while (iterator.hasNext()) {
        count++;
        iterator.next();
    }
    assertEquals("incorrect number of entries in collection", 0, count);
}

Example 5 with CloseableIterator

use of org.apache.geode.internal.cache.persistence.query.CloseableIterator in project geode by apache.

the class PdxStringQueryDUnitTest method testPartitionRegionCompactRangeIndex.

@Test
public void testPartitionRegionCompactRangeIndex() throws CacheException {
    final Host host = Host.getHost(0);
    VM server0 = host.getVM(0);
    VM server1 = host.getVM(1);
    VM server2 = host.getVM(2);
    VM client = host.getVM(3);
    final int numberOfEntries = 10;
    final boolean isPr = true;
    // Start server1 and create index
    server0.invoke(new CacheSerializableRunnable("Create Server1") {

        public void run2() throws CacheException {
            configAndStartBridgeServer(isPr, false, false);
            // create a local query service
            QueryService localQueryService = null;
            try {
                localQueryService = getCache().getQueryService();
            } catch (Exception e) {
                Assert.fail("Failed to get QueryService.", e);
            }
            // Verify the type of index created
            Index index = null;
            try {
                index = localQueryService.createIndex("statusIndex", "status", regName);
                if (index instanceof PartitionedIndex) {
                    for (Object o : ((PartitionedIndex) index).getBucketIndexes()) {
                        if (!(o instanceof CompactRangeIndex)) {
                            fail("CompactRangeIndex Index should have been created instead of " + index.getClass());
                        }
                    }
                } else {
                    fail("Partitioned index expected");
                }
            } catch (Exception ex) {
                fail("Failed to create index." + ex.getMessage());
            }
        }
    });
    // Start server2
    server1.invoke(new CacheSerializableRunnable("Create Server2") {

        public void run2() throws CacheException {
            configAndStartBridgeServer(isPr, false, false);
            Region region = getRootRegion().getSubregion(regionName);
        }
    });
    // Start server3
    server2.invoke(new CacheSerializableRunnable("Create Server3") {

        public void run2() throws CacheException {
            configAndStartBridgeServer(isPr, false, false);
            Region region = getRootRegion().getSubregion(regionName);
        }
    });
    // Client pool.
    final int port0 = server0.invoke(() -> PdxStringQueryDUnitTest.getCacheServerPort());
    final int port1 = server1.invoke(() -> PdxStringQueryDUnitTest.getCacheServerPort());
    final int port2 = server2.invoke(() -> PdxStringQueryDUnitTest.getCacheServerPort());
    final String host0 = NetworkUtils.getServerHostName(server0.getHost());
    // Create client pool.
    final String poolName = "testClientServerQueryPool";
    createPool(client, poolName, new String[] { host0 }, new int[] { port0, port1, port2 }, true);
    // Create client region and put PortfolioPdx objects (PdxInstances)
    client.invoke(new CacheSerializableRunnable("Create client") {

        public void run2() throws CacheException {
            AttributesFactory factory = new AttributesFactory();
            factory.setScope(Scope.LOCAL);
            ClientServerTestCase.configureConnectionPool(factory, host0, port1, -1, true, -1, -1, null);
            Region region = createRegion(regionName, rootRegionName, factory.create());
            LogWriterUtils.getLogWriter().info("Put PortfolioPdx");
            for (int i = 0; i < numberOfEntries; i++) {
                region.put("key-" + i, new PortfolioPdx(i));
            }
        }
    });
    // Verify if all the index keys are PdxStrings
    server0.invoke(new CacheSerializableRunnable("Create Server") {

        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            QueryService localQueryService = getCache().getQueryService();
            Index index = localQueryService.getIndex(region, "statusIndex");
            if (index instanceof PartitionedIndex) {
                for (Object o : ((PartitionedIndex) index).getBucketIndexes()) {
                    CloseableIterator<IndexStoreEntry> iter = ((CompactRangeIndex) o).getIndexStorage().iterator(null);
                    while (iter.hasNext()) {
                        Object key = iter.next().getDeserializedKey();
                        if (!(key instanceof PdxString)) {
                            fail("All keys of the CompactRangeIndex in the Partitioned index should be PdxStrings and not " + key.getClass());
                        }
                    }
                }
            } else {
                fail("Partitioned index expected");
            }
        }
    });
    // Execute queries from client to server and locally on client
    SerializableRunnable executeQueries = new CacheSerializableRunnable("Execute queries") {

        public void run2() throws CacheException {
            QueryService remoteQueryService = null;
            QueryService localQueryService = null;
            SelectResults[][] rs = new SelectResults[1][2];
            try {
                remoteQueryService = (PoolManager.find(poolName)).getQueryService();
                localQueryService = getCache().getQueryService();
            } catch (Exception e) {
                Assert.fail("Failed to get QueryService.", e);
            }
            for (int i = 0; i < queryString.length; i++) {
                try {
                    LogWriterUtils.getLogWriter().info("### Executing Query on remote server:" + queryString[i]);
                    Query query = remoteQueryService.newQuery(queryString[i]);
                    rs[0][0] = (SelectResults) query.execute();
                    LogWriterUtils.getLogWriter().info("RR remote indexType:CompactRange size of resultset: " + rs[0][0].size() + " for query: " + queryString[i]);
                    ;
                    checkForPdxString(rs[0][0].asList(), queryString[i]);
                    LogWriterUtils.getLogWriter().info("### Executing Query locally on client:" + queryString[i]);
                    query = localQueryService.newQuery(queryString[i]);
                    rs[0][1] = (SelectResults) query.execute();
                    LogWriterUtils.getLogWriter().info("isPR: " + isPr + "  client local indexType:CompactRange size of resultset: " + rs[0][1].size() + " for query: " + queryString[i]);
                    ;
                    checkForPdxString(rs[0][1].asList(), queryString[i]);
                    if (i < orderByQueryIndex) {
                        // Compare local and remote query results.
                        if (!compareResultsOfWithAndWithoutIndex(rs)) {
                            fail("Local and Remote Query Results are not matching for query :" + queryString[i]);
                        }
                    } else {
                        // compare the order of results returned
                        compareResultsOrder(rs, isPr);
                    }
                } catch (Exception e) {
                    Assert.fail("Failed executing " + queryString[i], e);
                }
            }
        }
    };
    client.invoke(executeQueries);
    // Put Non Pdx objects on server execute queries locally
    server0.invoke(new CacheSerializableRunnable("Create Bridge Server") {

        public void run2() throws CacheException {
            Region region = getRootRegion().getSubregion(regionName);
            LogWriterUtils.getLogWriter().info("Put Objects locally on server");
            for (int i = numberOfEntries; i < numberOfEntries * 2; i++) {
                region.put("key-" + i, new Portfolio(i));
            }
            QueryService localQueryService = getCache().getQueryService();
            // Query server1 locally to check if PdxString is not being returned
            for (int i = 0; i < queryString.length; i++) {
                try {
                    LogWriterUtils.getLogWriter().info("### Executing Query locally on server:" + queryString[i]);
                    SelectResults rs = (SelectResults) localQueryService.newQuery(queryString[i]).execute();
                    LogWriterUtils.getLogWriter().info("RR server local indexType:Range  size of resultset: " + rs.size() + " for query: " + queryString[i]);
                    // The results should not be PdxString
                    checkForPdxString(rs.asList(), queryString[i]);
                } catch (Exception e) {
                    Assert.fail("Failed executing " + queryString[i], e);
                }
            }
        }
    });
    // test for readSerialized flag
    server0.invoke(new CacheSerializableRunnable("Create Bridge Server") {

        public void run2() throws CacheException {
            GemFireCacheImpl cache = (GemFireCacheImpl) getCache();
            cache.setReadSerialized(true);
            QueryService localQueryService = getCache().getQueryService();
            // Query server1 locally to check if PdxString is not being returned
            for (int i = 0; i < queryString.length; i++) {
                try {
                    LogWriterUtils.getLogWriter().info("### Executing Query locally on server:" + queryString[i]);
                    SelectResults rs = (SelectResults) localQueryService.newQuery(queryString[i]).execute();
                    LogWriterUtils.getLogWriter().info("isPR: " + isPr + " server local readSerializedTrue: indexType:CompactRange size of resultset: " + rs.size() + " for query: " + queryString[i]);
                    // The results should not be PdxString
                    checkForPdxString(rs.asList(), queryString[i]);
                } catch (Exception e) {
                    Assert.fail("Failed executing " + queryString[i], e);
                }
            }
        }
    });
    // test for readSerialized flag on client
    client.invoke(new CacheSerializableRunnable("Create client") {

        public void run2() throws CacheException {
            GemFireCacheImpl cache = (GemFireCacheImpl) getCache();
            cache.setReadSerialized(true);
            QueryService remoteQueryService = (PoolManager.find(poolName)).getQueryService();
            // Query server1 remotely to check if PdxString is not being returned
            for (int i = 0; i < queryString.length; i++) {
                try {
                    LogWriterUtils.getLogWriter().info("### Executing Query locally on server:" + queryString[i]);
                    SelectResults rs = (SelectResults) remoteQueryService.newQuery(queryString[i]).execute();
                    LogWriterUtils.getLogWriter().info("RR server remote readSerializedTrue: indexType: indexType:CompactRange size of resultset: " + rs.size() + " for query: " + queryString[i]);
                    // The results should not be PdxString
                    checkForPdxString(rs.asList(), queryString[i]);
                } catch (Exception e) {
                    Assert.fail("Failed executing " + queryString[i], e);
                }
            }
        }
    });
    closeClient(server2);
    closeClient(client);
    closeClient(server1);
    closeClient(server0);
}