use of org.apache.geode.cache.query.internal.types.StructTypeImpl in project geode by apache.
the class QueryUtils method getRelationshipIndexResultsMergedWithIntermediateResults.
/**
* This function is used to evaluate a filter evaluatable CompositeCondition(ie Range Indexes
* available on both LHS & RHS operands).This function is invoked from AND junction evaluation of
* CompositeGroupJunction. It expands the intermediate resultset passed , to the level of groups
* determined by the LHS & RHS operand, using the range indexes. It is possible that the group of
* iterators for an operand of condition already exists in the intermediate resultset passed. In
* such situation, the intermediate resultset is iterated & the operand ( whose group of iterators
* are available in the intermediate resultset ) is evaluated. For each such evaluated value , the
* other operand's Range Index is queried & the Range Index's results are appropriately expanded &
* cut down & a final tuple obtained( which includes the previously existing fields of
* intermediate resultset). The array of independent iterators passed from the Composite Group
* junction will be null, except for the final condition ( subject to the fact that complete
* expansion flag is false. Otherwise even for final condition , the array will be null) as that
* array will be used to get the final position of iterators in the resultant StructBag
*
* TODO: break this method up
*
* @param intermediateResults SelectResults object containing the intermediate resultset obtained
* by evaluation of previous filter evaluatable composite conditions of the
* CompositeGroupJunction
* @param indxInfo Array of IndexInfo objects ( size 2), representing the range index for the two
* operands of the condition
* @param context ExecutionContext object
* @param completeExpansionNeeded A boolean when true indicates that the final result from
* Composite GroupJunction needs to be evaluated to the query from clause ( top ) level.
* @param iterOperands CompiledValue representing the conditions which are to be iter evaluated.
* This can exist only if instead of AllGroupJunction we have a single
* CompositeGroupJunction
* @param indpdntItrs Array of RuntimeIterators representing the independent iterators of their
* representative groups forming the CompositeGroupJunction *
* @return SelectResults The Result object created by evaluating the filter evaluatable condition
* merged with the intermediate results
*/
static SelectResults getRelationshipIndexResultsMergedWithIntermediateResults(SelectResults intermediateResults, IndexInfo[] indxInfo, ExecutionContext context, boolean completeExpansionNeeded, CompiledValue iterOperands, RuntimeIterator[] indpdntItrs) throws FunctionDomainException, TypeMismatchException, NameResolutionException, QueryInvocationTargetException {
ObjectType resultType1 = indxInfo[0]._index.getResultSetType();
int indexFieldsSize1 = resultType1 instanceof StructType ? ((StructTypeImpl) resultType1).getFieldNames().length : 1;
ObjectType resultType2 = indxInfo[1]._index.getResultSetType();
int indexFieldsSize2 = resultType2 instanceof StructType ? ((StructTypeImpl) resultType2).getFieldNames().length : 1;
/*
* even if the complete expansion is needed pass the flag of complete expansion as false. Thus
* for LHS & RHS we will get the expansionList for that individual group.
*/
// NOTE: use false for completeExpansion irrespective of actual value
IndexConditioningHelper ich1 = new IndexConditioningHelper(indxInfo[0], context, indexFieldsSize1, false, iterOperands, null);
// NOTE: use false for completeExpansion irrespective of actual value
IndexConditioningHelper ich2 = new IndexConditioningHelper(indxInfo[1], context, indexFieldsSize2, false, iterOperands, null);
// We cannot have a condition where in intermediateResultset is empty
// or null & complete
// expansion flag true because in that case instead of this function we should
// have called
int noOfIndexesToUse = intermediateResults == null || intermediateResults.isEmpty() ? 2 : 0;
RuntimeIterator[] resultFieldsItrMapping = null;
List allItrs = context.getCurrentIterators();
IndexConditioningHelper singleUsableICH = null;
IndexConditioningHelper nonUsableICH = null;
List finalList = completeExpansionNeeded ? allItrs : indpdntItrs == null ? new ArrayList() : null;
// the set will contain those iterators which we don't have to expand to either because they are
// already present ( because of intermediate results or because index result already contains
// them
Set expnItrsToIgnore = null;
if (noOfIndexesToUse == 0) {
// If the intermediate Resultset is not empty then check if the resultset
// fields of intermediate
// resultset contains any independent iterator of the current condition
noOfIndexesToUse = 2;
StructType stype = (StructType) intermediateResults.getCollectionType().getElementType();
String[] fieldNames = stype.getFieldNames();
int len = fieldNames.length;
resultFieldsItrMapping = new RuntimeIterator[len];
String fieldName = null;
String lhsID = ich1.indpndntItr.getInternalId();
String rhsID = ich2.indpndntItr.getInternalId();
for (int i = 0; i < len; ++i) {
fieldName = fieldNames[i];
if (noOfIndexesToUse != 0) {
if (fieldName.equals(lhsID)) {
--noOfIndexesToUse;
singleUsableICH = ich2;
nonUsableICH = ich1;
} else if (fieldName.equals(rhsID)) {
--noOfIndexesToUse;
singleUsableICH = ich1;
nonUsableICH = ich2;
}
}
int pos = Integer.parseInt(fieldName.substring(4));
RuntimeIterator itrPrsntInIntermdtRes = (RuntimeIterator) allItrs.get(pos - 1);
resultFieldsItrMapping[i] = itrPrsntInIntermdtRes;
// the iterator below is already present in resultset so needs to be ignored for expansion
if (completeExpansionNeeded) {
if (expnItrsToIgnore == null) {
expnItrsToIgnore = new HashSet();
}
expnItrsToIgnore.add(itrPrsntInIntermdtRes);
} else if (indpdntItrs == null) {
// We will need to know the intermediate iterators so as to know
// the final list which will be used to obtain the correct structset.
// But if the independent group of iterators is passed, the final list needs
// to be calculated
// on its basis
finalList.add(itrPrsntInIntermdtRes);
}
}
if (noOfIndexesToUse == 0) {
singleUsableICH = null;
}
}
QueryObserver observer = QueryObserverHolder.getInstance();
if (noOfIndexesToUse == 2) {
List data = null;
try {
ArrayList resultData = new ArrayList();
observer.beforeIndexLookup(indxInfo[0]._index, OQLLexerTokenTypes.TOK_EQ, null);
observer.beforeIndexLookup(indxInfo[1]._index, OQLLexerTokenTypes.TOK_EQ, null);
if (context.getBucketList() != null) {
data = queryEquijoinConditionBucketIndexes(indxInfo, context);
} else {
data = indxInfo[0]._index.queryEquijoinCondition(indxInfo[1]._index, context);
}
} finally {
observer.afterIndexLookup(data);
}
// For sure we need to evaluate both the conditions & expand it only to
// its own respective
// Ignore the boolean of reshuffling needed etc for this case
List totalExpList = new ArrayList();
totalExpList.addAll(ich1.expansionList);
totalExpList.addAll(ich2.expansionList);
if (completeExpansionNeeded) {
if (expnItrsToIgnore == null) {
// The expnItrsToIgnore set being null at this point implies that though complete
// expansion flag is true but intermediate result set is empty
Support.Assert(intermediateResults == null || intermediateResults.isEmpty(), "expnItrsToIgnore should not have been null if the intermediate result set is not empty");
expnItrsToIgnore = new HashSet();
}
expnItrsToIgnore.addAll(ich1.finalList);
expnItrsToIgnore.addAll(ich2.finalList);
// identify the iterators which we need to expand to
// TODO: Make the code compact by using a common function to take care of this
int size = finalList.size();
for (int i = 0; i < size; ++i) {
RuntimeIterator currItr = (RuntimeIterator) finalList.get(i);
// If the runtimeIterators of scope not present in CheckSet add it to the expansion list
if (!expnItrsToIgnore.contains(currItr)) {
totalExpList.add(currItr);
}
}
} else {
// struct set mismatch while doing intersection with GroupJunction results
if (indpdntItrs != null) {
finalList = getDependentItrChainForIndpndntItrs(indpdntItrs, context);
} else {
finalList.addAll(ich1.finalList);
finalList.addAll(ich2.finalList);
}
}
List[] checkList = new List[] { ich1.checkList, ich2.checkList };
StructType stype = createStructTypeForRuntimeIterators(finalList);
SelectResults returnSet = QueryUtils.createStructCollection(context, stype);
RuntimeIterator[][] mappings = new RuntimeIterator[2][];
mappings[0] = ich1.indexFieldToItrsMapping;
mappings[1] = ich2.indexFieldToItrsMapping;
List[] totalCheckList = new List[] { ich1.checkList, ich2.checkList };
RuntimeIterator[][] resultMappings = new RuntimeIterator[1][];
resultMappings[0] = resultFieldsItrMapping;
Iterator dataItr = data.iterator();
IndexCutDownExpansionHelper[] icdeh = new IndexCutDownExpansionHelper[] { new IndexCutDownExpansionHelper(ich1.checkList, context), new IndexCutDownExpansionHelper(ich2.checkList, context) };
ListIterator expansionListIterator = totalExpList.listIterator();
if (dataItr.hasNext()) {
observer = QueryObserverHolder.getInstance();
try {
observer.beforeMergeJoinOfDoubleIndexResults(indxInfo[0]._index, indxInfo[1]._index, data);
boolean doMergeWithIntermediateResults = intermediateResults != null && !intermediateResults.isEmpty();
int maxCartesianDepth = totalExpList.size() + (doMergeWithIntermediateResults ? 1 : 0);
while (dataItr.hasNext()) {
// TODO: Change the code in range Index so that while collecting data instead of
// creating two dimensional object array , we create one dimensional Object array of
// size 2, & each elemnt stores an Object array
Object[][] values = (Object[][]) dataItr.next();
// made by different data in the other set)
if (doMergeWithIntermediateResults) {
mergeRelationshipIndexResultsWithIntermediateResults(returnSet, new SelectResults[] { intermediateResults }, resultMappings, values, mappings, expansionListIterator, finalList, context, checkList, iterOperands, icdeh, 0, maxCartesianDepth);
} else {
mergeAndExpandCutDownRelationshipIndexResults(values, returnSet, mappings, expansionListIterator, finalList, context, totalCheckList, iterOperands, icdeh, 0);
}
if (icdeh[0].cutDownNeeded)
icdeh[0].checkSet.clear();
}
} finally {
observer.afterMergeJoinOfDoubleIndexResults(returnSet);
}
}
return returnSet;
} else if (noOfIndexesToUse == 1) {
// There exists one independent iterator in the current condition which is also a part of the
// intermediate resultset Identify the final List which will depend upon the complete
// expansion flag Identify the iterators to be expanded to, which will also depend upon
// complete expansion flag..
List totalExpList = new ArrayList();
totalExpList.addAll(singleUsableICH.expansionList);
if (completeExpansionNeeded) {
Support.Assert(expnItrsToIgnore != null, "expnItrsToIgnore should not have been null as we are in this block itself indicates that intermediate results was not null");
expnItrsToIgnore.addAll(singleUsableICH.finalList);
// identify the iterators which we need to expand to
// TODO: Make the code compact by using a common function to take care of this
int size = finalList.size();
for (int i = 0; i < size; ++i) {
RuntimeIterator currItr = (RuntimeIterator) finalList.get(i);
// If the runtimeIterators of scope not present in CheckSet add it to the expansion list
if (!expnItrsToIgnore.contains(currItr)) {
totalExpList.add(currItr);
}
}
} else {
// struct set mismatch while doing intersection with GroupJunction results
if (indpdntItrs != null) {
finalList = getDependentItrChainForIndpndntItrs(indpdntItrs, context);
} else {
finalList.addAll(singleUsableICH.finalList);
}
}
StructType stype = createStructTypeForRuntimeIterators(finalList);
SelectResults returnSet = QueryUtils.createStructCollection(context, stype);
// Obtain the empty resultset for the single usable index
IndexProtocol singleUsblIndex = singleUsableICH.indxInfo._index;
CompiledValue nonUsblIndxPath = nonUsableICH.indxInfo._path;
ObjectType singlUsblIndxResType = singleUsblIndex.getResultSetType();
SelectResults singlUsblIndxRes = null;
if (singlUsblIndxResType instanceof StructType) {
singlUsblIndxRes = QueryUtils.createStructCollection(context, (StructTypeImpl) singlUsblIndxResType);
} else {
singlUsblIndxRes = QueryUtils.createResultCollection(context, singlUsblIndxResType);
}
// iterate over the intermediate structset
Iterator intrmdtRsItr = intermediateResults.iterator();
observer = QueryObserverHolder.getInstance();
try {
observer.beforeIndexLookup(singleUsblIndex, OQLLexerTokenTypes.TOK_EQ, null);
observer.beforeIterJoinOfSingleIndexResults(singleUsblIndex, nonUsableICH.indxInfo._index);
while (intrmdtRsItr.hasNext()) {
Struct strc = (Struct) intrmdtRsItr.next();
Object[] val = strc.getFieldValues();
int len = val.length;
for (int i = 0; i < len; ++i) {
resultFieldsItrMapping[i].setCurrent(val[i]);
}
// TODO: Issue relevant index use callbacks to QueryObserver
Object key = nonUsblIndxPath.evaluate(context);
// TODO: Check this logic out
if (key != null && key.equals(QueryService.UNDEFINED)) {
continue;
}
singleUsblIndex.query(key, OQLLexerTokenTypes.TOK_EQ, singlUsblIndxRes, context);
cutDownAndExpandIndexResults(returnSet, singlUsblIndxRes, singleUsableICH.indexFieldToItrsMapping, totalExpList, finalList, context, singleUsableICH.checkList, iterOperands, singleUsableICH.indxInfo);
singlUsblIndxRes.clear();
}
} finally {
observer.afterIterJoinOfSingleIndexResults(returnSet);
observer.afterIndexLookup(returnSet);
}
return returnSet;
} else {
// PART OF ITER OPERANDS
if (logger.isDebugEnabled()) {
StringBuilder tempBuffLhs = new StringBuilder();
StringBuilder tempBuffRhs = new StringBuilder();
ich1.indxInfo._path.generateCanonicalizedExpression(tempBuffLhs, context);
ich2.indxInfo._path.generateCanonicalizedExpression(tempBuffRhs, context);
logger.debug("For better performance indexes are not used for the condition {} = {}", tempBuffLhs, tempBuffRhs);
}
CompiledValue reconstructedVal = new CompiledComparison(ich1.indxInfo._path, ich2.indxInfo._path, OQLLexerTokenTypes.TOK_EQ);
// Add this reconstructed value to the iter operand if any
CompiledValue finalVal = reconstructedVal;
if (iterOperands != null) {
// The type of CompiledJunction has to be AND junction as this function gets invoked only
// for AND . Also it is OK if we have iterOperands which itself is a CompiledJunction. We
// can have a tree of CompiledJunction with its operands being a CompiledComparison & a
// CompiledJunction. We can live without creating a flat structure
finalVal = new CompiledJunction(new CompiledValue[] { iterOperands, reconstructedVal }, OQLLexerTokenTypes.LITERAL_and);
}
RuntimeIterator[][] resultMappings = new RuntimeIterator[1][];
resultMappings[0] = resultFieldsItrMapping;
return cartesian(new SelectResults[] { intermediateResults }, resultMappings, Collections.emptyList(), finalList, context, finalVal);
}
}
use of org.apache.geode.cache.query.internal.types.StructTypeImpl in project geode by apache.
the class JavaSerializationJUnitTest method testStructImplSerialization.
@Test
public void testStructImplSerialization() throws Exception {
String[] fieldNames = { "col1", "col2" };
ObjectType[] fieldTypes = { new ObjectTypeImpl(Integer.class), new ObjectTypeImpl(String.class) };
StructTypeImpl type = new StructTypeImpl(fieldNames, fieldTypes);
Object[] values = { new Integer(123), new String("456") };
StructImpl si = new StructImpl(type, values);
verifyJavaSerialization(si);
}
use of org.apache.geode.cache.query.internal.types.StructTypeImpl in project geode by apache.
the class StructSetJUnitTest method testIntersectionAndRetainAll.
@Test
public void testIntersectionAndRetainAll() {
String[] names = { "p", "pos" };
ObjectType[] types = { TypeUtils.OBJECT_TYPE, TypeUtils.OBJECT_TYPE };
StructTypeImpl sType = new StructTypeImpl(names, types);
StructSet set1 = new StructSet(sType);
Portfolio ptf = new Portfolio(0);
Iterator pIter = ptf.positions.values().iterator();
while (pIter.hasNext()) {
Object[] arr = { ptf, pIter.next() };
set1.addFieldValues(arr);
}
StructSet set2 = new StructSet(sType);
pIter = ptf.positions.values().iterator();
while (pIter.hasNext()) {
Object[] arr = { ptf, pIter.next() };
set2.addFieldValues(arr);
}
assertEquals(2, set1.size());
assertEquals(2, set2.size());
// tests that retainAll does not modify set1
assertTrue(!set1.retainAll(set2));
assertEquals(2, set1.size());
assertEquals(2, set2.size());
SelectResults sr = QueryUtils.intersection(set1, set2, null);
assertEquals(2, sr.size());
}
use of org.apache.geode.cache.query.internal.types.StructTypeImpl in project geode by apache.
the class CustomerOptimizationsJUnitTest method testUnionDuringIndexEvaluationWithMultipleFilters.
@Test
public void testUnionDuringIndexEvaluationWithMultipleFilters() throws QueryException {
QueryService qs = CacheUtils.getQueryService();
Region rgn = CacheUtils.getRegion("/pos");
for (int i = 100; i < 200; ++i) {
Portfolio pf = new Portfolio(i);
pf.setCreateTime(10l);
rgn.put("" + i, pf);
}
String[] queries = new String[] { "select p.status as sts, p as pos from /pos p where p.ID IN SET( 0,1,2,3,4,5,101,102,103,104,105) AND p.createTime > 9l" };
SelectResults[][] sr = new SelectResults[queries.length][2];
for (int i = 0; i < queries.length; ++i) {
Query q = qs.newQuery(queries[i]);
sr[i][0] = (SelectResults) q.execute();
}
qs.createIndex("PortFolioID", IndexType.FUNCTIONAL, "ID", "/pos");
qs.createIndex("CreateTime", IndexType.FUNCTIONAL, "createTime", "/pos");
final boolean[] expectedIndexUsed = new boolean[] { true };
final boolean[] actualIndexUsed = new boolean[] { false };
final boolean[] expectedProjectionCallabck = new boolean[] { false };
final boolean[] actualProjectionCallback = new boolean[] { false };
final boolean[] expectedUnionCallback = { false };
final boolean[] actualUnionCallback = new boolean[queries.length];
final boolean[] expectedIntersectionCallback = { false };
final boolean[] actualIntersectionCallback = new boolean[queries.length];
ObjectType[] expectedTypes = new ObjectType[] { new StructTypeImpl(new String[] { "sts", "pos" }, new ObjectType[] { new ObjectTypeImpl(String.class), new ObjectTypeImpl(Portfolio.class) }) };
QueryObserverHolder.setInstance(new QueryObserverAdapter() {
private int i = 0;
public void invokedQueryUtilsUnion(SelectResults r1, SelectResults r2) {
actualUnionCallback[i] = true;
}
public void invokedQueryUtilsIntersection(SelectResults r1, SelectResults r2) {
actualIntersectionCallback[i] = true;
}
public void beforeIndexLookup(Index index, int oper, Object key) {
actualIndexUsed[i] = true;
}
public void beforeApplyingProjectionOnFilterEvaluatedResults(Object preProjectionApplied) {
actualProjectionCallback[i] = true;
}
public void afterQueryEvaluation(Object result) {
++i;
}
});
for (int i = 0; i < queries.length; ++i) {
Query q = qs.newQuery(queries[i]);
sr[i][1] = (SelectResults) q.execute();
assertEquals(expectedUnionCallback[i], actualUnionCallback[i]);
assertEquals(expectedTypes[i], sr[i][1].getCollectionType().getElementType());
assertEquals(expectedIndexUsed[i], actualIndexUsed[i]);
assertEquals(expectedProjectionCallabck[i], actualProjectionCallback[i]);
assertEquals(expectedIntersectionCallback[i], actualIntersectionCallback[i]);
}
CacheUtils.compareResultsOfWithAndWithoutIndex(sr, this);
}
use of org.apache.geode.cache.query.internal.types.StructTypeImpl in project geode by apache.
the class CustomerOptimizationsJUnitTest method testUnionDuringIndexEvaluationForIN.
@Test
public void testUnionDuringIndexEvaluationForIN() throws QueryException {
QueryService qs = CacheUtils.getQueryService();
Region rgn = CacheUtils.getRegion("/pos");
for (int i = 100; i < 200; ++i) {
Portfolio pf = new Portfolio(i);
pf.setCreateTime(10l);
rgn.put("" + i, pf);
}
String[] queries = new String[] { "select p.status as sts, p as pos from /pos p where p.ID IN SET( 0,1,2,3,4,5) ", "select p.status as sts, p as pos from /pos p where p.ID IN SET( 0,1,2,3,4,5,101,102,103,104,105) AND p.createTime > 9l" };
SelectResults[][] sr = new SelectResults[queries.length][2];
for (int i = 0; i < queries.length; ++i) {
Query q = qs.newQuery(queries[i]);
sr[i][0] = (SelectResults) q.execute();
}
qs.createIndex("PortFolioID", IndexType.FUNCTIONAL, "ID", "/pos");
final boolean[] expectedIndexUsed = new boolean[] { true, true };
final boolean[] actualIndexUsed = new boolean[] { false, false };
final boolean[] expectedProjectionCallabck = new boolean[] { false, false };
final boolean[] actualProjectionCallback = new boolean[] { false, false };
final boolean[] expectedUnionCallback = { false, false };
final boolean[] actualUnionCallback = new boolean[queries.length];
final boolean[] expectedIntersectionCallback = { false, false };
final boolean[] actualIntersectionCallback = new boolean[queries.length];
ObjectType[] expectedTypes = new ObjectType[] { new StructTypeImpl(new String[] { "sts", "pos" }, new ObjectType[] { new ObjectTypeImpl(String.class), new ObjectTypeImpl(Portfolio.class) }), new StructTypeImpl(new String[] { "sts", "pos" }, new ObjectType[] { new ObjectTypeImpl(String.class), new ObjectTypeImpl(Portfolio.class) }) };
QueryObserverHolder.setInstance(new QueryObserverAdapter() {
private int i = 0;
public void invokedQueryUtilsUnion(SelectResults r1, SelectResults r2) {
actualUnionCallback[i] = true;
}
public void invokedQueryUtilsIntersection(SelectResults r1, SelectResults r2) {
actualIntersectionCallback[i] = true;
}
public void beforeIndexLookup(Index index, int oper, Object key) {
actualIndexUsed[i] = true;
}
public void beforeApplyingProjectionOnFilterEvaluatedResults(Object preProjectionApplied) {
actualProjectionCallback[i] = true;
}
public void afterQueryEvaluation(Object result) {
++i;
}
});
for (int i = 0; i < queries.length; ++i) {
Query q = qs.newQuery(queries[i]);
sr[i][1] = (SelectResults) q.execute();
assertEquals(expectedUnionCallback[i], actualUnionCallback[i]);
assertEquals(expectedTypes[i], sr[i][1].getCollectionType().getElementType());
assertEquals(expectedIndexUsed[i], actualIndexUsed[i]);
assertEquals(expectedProjectionCallabck[i], actualProjectionCallback[i]);
assertEquals(expectedIntersectionCallback[i], actualIntersectionCallback[i]);
}
CacheUtils.compareResultsOfWithAndWithoutIndex(sr, this);
}
Aggregations