use of org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.graph.GraphFetchTree in project legend-engine by finos.
the class RelationalExecutionNodeExecutor method executeRelationalCrossRootQueryTempTableGraphFetchExecutionNode.
private Result executeRelationalCrossRootQueryTempTableGraphFetchExecutionNode(RelationalCrossRootQueryTempTableGraphFetchExecutionNode node) {
boolean isLeaf = node.children == null || node.children.isEmpty();
List<Object> childObjects = new ArrayList<>();
Result childResult = null;
RelationalStoreExecutionState relationalStoreExecutionState = (RelationalStoreExecutionState) this.executionState.getStoreExecutionState(StoreType.Relational);
BlockConnectionContext oldBlockConnectionContext = relationalStoreExecutionState.getBlockConnectionContext();
boolean oldRetainConnectionFlag = relationalStoreExecutionState.retainConnection();
relationalStoreExecutionState.setBlockConnectionContext(new BlockConnectionContext());
relationalStoreExecutionState.setRetainConnection(true);
try (Scope ignored = GlobalTracer.get().buildSpan("cross store property graph fetch").withTag("storeType", "relational").withTag("property", ((PropertyGraphFetchTree) node.graphFetchTree).property).startActive(true)) {
IRelationalCrossRootQueryTempTableGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.profiles);
RelationalGraphObjectsBatch relationalGraphObjectsBatch = new RelationalGraphObjectsBatch(this.executionState.graphObjectsBatch);
List<?> parentObjects = relationalGraphObjectsBatch.getObjectsForNodeIndex(node.parentIndex);
if ((parentObjects != null) && !parentObjects.isEmpty()) {
GraphFetchTree nodeSubTree = node.graphFetchTree;
boolean cachingEnabled = false;
ExecutionCache<GraphFetchCacheKey, List<Object>> crossCache = relationalGraphObjectsBatch.getXStorePropertyCacheForNodeIndex(node.nodeIndex);
List<Method> parentCrossKeyGettersOrderedPerTargetProperties = null;
if (crossCache != null) {
cachingEnabled = true;
parentCrossKeyGettersOrderedPerTargetProperties = nodeSpecifics.parentCrossKeyGettersOrderedByTargetProperties();
}
List<Object> parentsToDeepFetch = new ArrayList<>();
for (Object parent : parentObjects) {
if (cachingEnabled) {
List<Object> children = crossCache.getIfPresent(new RelationalGraphFetchUtils.RelationalCrossObjectGraphFetchCacheKey(parent, parentCrossKeyGettersOrderedPerTargetProperties));
if (children == null) {
parentsToDeepFetch.add(parent);
} else {
for (Object child : children) {
childObjects.add(child);
nodeSpecifics.addChildToParent(parent, child, DefaultExecutionNodeContext.factory().create(this.executionState, null));
}
}
} else {
parentsToDeepFetch.add(parent);
}
}
if (!parentsToDeepFetch.isEmpty()) {
Map<Object, List<Object>> parentToChildMap = new HashMap<>();
RealizedRelationalResult parentRealizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.parentTempTableColumns);
List<Method> crossKeyGetters = nodeSpecifics.parentCrossKeyGetters();
for (Object parentObject : parentsToDeepFetch) {
this.addKeyRowToRealizedRelationalResult(parentObject, crossKeyGetters, parentRealizedRelationalResult);
parentToChildMap.put(parentObject, new ArrayList<>());
}
this.executionState.addResult(node.parentTempTableName, parentRealizedRelationalResult);
childResult = node.executionNodes.get(0).accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
SQLExecutionResult childSqlResult = (SQLExecutionResult) childResult;
DatabaseConnection databaseConnection = childSqlResult.getSQLExecutionNode().connection;
ResultSet childResultSet = childSqlResult.getResultSet();
List<String> parentSQLKeyColumns = nodeSpecifics.parentCrossKeyColumns(childSqlResult.getResultColumns().stream().map(ResultColumn::getNonQuotedLabel).collect(Collectors.toList()));
List<Integer> parentCrossKeyIndices = parentSQLKeyColumns.stream().map(FunctionHelper.unchecked(childResultSet::findColumn)).collect(Collectors.toList());
@SuppressWarnings("MismatchedQueryAndUpdateOfCollection") DoubleStrategyHashMap<Object, List<Object>, SQLExecutionResult> parentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategy(crossKeyGetters, parentCrossKeyIndices));
parentsToDeepFetch.forEach((o) -> parentMap.getIfAbsentPut(o, ArrayList::new).add(o));
RealizedRelationalResult realizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.columns);
List<Method> primaryKeyGetters = nodeSpecifics.primaryKeyGetters();
DoubleStrategyHashMap<Object, Object, SQLExecutionResult> currentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
/* Prepare for reading */
nodeSpecifics.prepare(childResultSet, childSqlResult.getDatabaseTimeZone(), ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(databaseConnection));
while (childResultSet.next()) {
relationalGraphObjectsBatch.incrementRowCount();
List<Object> parents = parentMap.getWithSecondKey(childSqlResult);
if (parents == null) {
throw new RuntimeException("Cannot find the parent for child");
}
IGraphInstance<? extends IReferencedObject> childGraphInstance = nodeSpecifics.nextGraphInstance();
Object child = childGraphInstance.getValue();
Object mapObject = currentMap.putIfAbsent(child, child);
if (mapObject == null) {
mapObject = child;
childObjects.add(mapObject);
relationalGraphObjectsBatch.addObjectMemoryUtilization(childGraphInstance.instanceSize());
if (!isLeaf) {
this.addKeyRowToRealizedRelationalResult(child, primaryKeyGetters, realizedRelationalResult);
}
}
for (Object parent : parents) {
if (parentToChildMap.containsKey(parent)) {
parentToChildMap.get(parent).add(mapObject);
} else {
parentToChildMap.put(parent, new ArrayList<>());
parentToChildMap.get(parent).add(mapObject);
}
nodeSpecifics.addChildToParent(parent, mapObject, DefaultExecutionNodeContext.factory().create(this.executionState, null));
}
}
relationalGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, childObjects);
childResult.close();
childResult = null;
/* Execute store local children */
if (!isLeaf) {
ExecutionState newState = new ExecutionState(executionState);
newState.graphObjectsBatch = relationalGraphObjectsBatch;
this.executeTempTableNodeChildren(node, realizedRelationalResult, databaseConnection, childSqlResult.getDatabaseType(), childSqlResult.getDatabaseTimeZone(), currentMap, primaryKeyGetters, newState);
}
if (cachingEnabled) {
List<Method> getters = parentCrossKeyGettersOrderedPerTargetProperties;
parentToChildMap.forEach((p, cs) -> {
crossCache.put(new RelationalGraphFetchUtils.RelationalCrossObjectGraphFetchCacheKey(p, getters), cs);
});
}
}
}
return new ConstantResult(childObjects);
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (childResult != null) {
childResult.close();
}
relationalStoreExecutionState.getBlockConnectionContext().unlockAllBlockConnections();
relationalStoreExecutionState.getBlockConnectionContext().closeAllBlockConnectionsAsync();
relationalStoreExecutionState.setBlockConnectionContext(oldBlockConnectionContext);
relationalStoreExecutionState.setRetainConnection(oldRetainConnectionFlag);
}
}
use of org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.graph.GraphFetchTree in project legend-engine by finos.
the class RelationalGraphFetchUtils method subTreeValidForCaching.
static boolean subTreeValidForCaching(GraphFetchTree graphFetchTree) {
boolean currentValidity = true;
if (graphFetchTree instanceof PropertyGraphFetchTree) {
PropertyGraphFetchTree propertyGraphFetchTree = (PropertyGraphFetchTree) graphFetchTree;
if (propertyGraphFetchTree.parameters != null && !propertyGraphFetchTree.parameters.isEmpty()) {
currentValidity = propertyGraphFetchTree.parameters.stream().allMatch(param -> param instanceof CBoolean || param instanceof CInteger || param instanceof CFloat || param instanceof CDecimal || param instanceof CString || param instanceof CStrictDate || param instanceof CDateTime || param instanceof EnumValue || (param instanceof Collection && ((Collection) param).values.stream().allMatch(x -> x instanceof EnumValue)));
}
}
boolean childrenValidity = graphFetchTree.subTrees == null || graphFetchTree.subTrees.isEmpty() || graphFetchTree.subTrees.stream().allMatch(RelationalGraphFetchUtils::subTreeValidForCaching);
return currentValidity && childrenValidity;
}
use of org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.graph.GraphFetchTree in project legend-engine by finos.
the class HelperValueSpecificationBuilder method buildPropertyGraphFetchTree.
private static GraphFetchTree buildPropertyGraphFetchTree(PropertyGraphFetchTree propertyGraphFetchTree, CompileContext context, Class<?> parentClass, MutableList<String> openVariables, ProcessingContext processingContext) {
AbstractProperty<?> property;
MutableList<org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.ValueSpecification> pureParameters = Lists.mutable.empty();
if (!propertyGraphFetchTree.parameters.isEmpty()) {
Variable thisVariable = new Variable("this", HelperModelBuilder.getElementFullPath(parentClass, context.pureModel.getExecutionSupport()), new Multiplicity(1, 1));
property = HelperModelBuilder.getAppliedProperty(context, parentClass, Optional.of(Lists.mutable.<org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification>with(thisVariable).withAll(propertyGraphFetchTree.parameters)), propertyGraphFetchTree.property);
processingContext.push("PropertyTree");
processingContext.addInferredVariables("this", HelperModelBuilder.createThisVariableForClass(context, HelperModelBuilder.getElementFullPath(parentClass, context.pureModel.getExecutionSupport())));
pureParameters = ListIterate.collect(propertyGraphFetchTree.parameters, x -> x.accept(new ValueSpecificationBuilder(context, openVariables, processingContext)));
processingContext.flushVariable("this");
processingContext.pop();
} else {
property = HelperModelBuilder.getAppliedProperty(context, parentClass, Optional.empty(), propertyGraphFetchTree.property, propertyGraphFetchTree.sourceInformation);
}
Class<?> subType = propertyGraphFetchTree.subType == null ? null : context.resolveClass(propertyGraphFetchTree.subType, propertyGraphFetchTree.sourceInformation);
Type returnType = subType == null ? property._genericType()._rawType() : subType;
ListIterable<GraphFetchTree> children = ListIterate.collect(propertyGraphFetchTree.subTrees, subTree -> buildGraphFetchTree(subTree, context, (Class<?>) returnType, openVariables, processingContext));
return new Root_meta_pure_graphFetch_PropertyGraphFetchTree_Impl("")._property(property)._parameters(pureParameters)._alias(propertyGraphFetchTree.alias)._subType(subType)._subTrees(children);
}
use of org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.graph.GraphFetchTree in project legend-engine by finos.
the class RelationalGraphFetchUtils method findCacheByEqualityKeys.
static GraphFetchCacheByEqualityKeys findCacheByEqualityKeys(GraphFetchTree graphFetchTree, String mappingId, String instanceSetId, List<GraphFetchCache> graphFetchCaches) {
if (!subTreeValidForCaching(graphFetchTree)) {
return null;
}
String subTree = getSubTreeString(graphFetchTree);
GraphFetchCacheByEqualityKeys matchingUtilizedCache = null;
for (GraphFetchCache c : graphFetchCaches) {
if (c instanceof GraphFetchCacheByEqualityKeys) {
GraphFetchCacheByEqualityKeys ce = (GraphFetchCacheByEqualityKeys) c;
if (ce.isCacheUtilized() && mappingId.equals(ce.getMappingId()) && instanceSetId.equals(ce.getInstanceSetId()) && subTree.equals(ce.getSubTree())) {
matchingUtilizedCache = ce;
break;
}
}
}
if (matchingUtilizedCache == null) {
GraphFetchCacheByEqualityKeys unUtilizedCache = null;
for (GraphFetchCache c : graphFetchCaches) {
if (c instanceof GraphFetchCacheByEqualityKeys) {
GraphFetchCacheByEqualityKeys ce = (GraphFetchCacheByEqualityKeys) c;
if (!ce.isCacheUtilized() && mappingId.equals(ce.getMappingId()) && instanceSetId.equals(ce.getInstanceSetId())) {
unUtilizedCache = ce;
break;
}
}
}
if (unUtilizedCache != null) {
unUtilizedCache.setSubTree(subTree);
return unUtilizedCache;
}
return null;
}
return matchingUtilizedCache;
}
use of org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.graph.GraphFetchTree in project legend-engine by finos.
the class GraphFetchTreeParseTreeWalker method visitRootGraphDefinition.
private RootGraphFetchTree visitRootGraphDefinition(GraphFetchTreeParserGrammar.GraphDefinitionContext graphDefinitionContext, GraphFetchTreeParserGrammar.DefinitionContext definitionContext) {
List<GraphFetchTree> subTrees = new ArrayList<>();
for (GraphFetchTreeParserGrammar.GraphPathContext graphPathContext : graphDefinitionContext.graphPaths().graphPath()) {
subTrees.add(this.visitGraphPathContext(graphPathContext));
}
RootGraphFetchTree result = new RootGraphFetchTree();
result._class = PureGrammarParserUtility.fromQualifiedName(definitionContext.qualifiedName().packagePath() == null ? Collections.emptyList() : definitionContext.qualifiedName().packagePath().identifier(), definitionContext.qualifiedName().identifier());
result.sourceInformation = walkerSourceInformation.getSourceInformation(definitionContext.qualifiedName());
result.subTrees = subTrees;
return result;
}
Aggregations