use of org.finos.legend.engine.plan.execution.stores.relational.result.SQLExecutionResult in project legend-engine by finos.
the class RelationalExecutionNodeExecutor method visit.
@Override
public Result visit(ExecutionNode executionNode) {
if (executionNode instanceof RelationalBlockExecutionNode) {
RelationalBlockExecutionNode relationalBlockExecutionNode = (RelationalBlockExecutionNode) executionNode;
ExecutionState connectionAwareState = new ExecutionState(this.executionState);
((RelationalStoreExecutionState) connectionAwareState.getStoreExecutionState(StoreType.Relational)).setRetainConnection(true);
try {
Result res = new ExecutionNodeExecutor(this.profiles, connectionAwareState).visit((SequenceExecutionNode) relationalBlockExecutionNode);
((RelationalStoreExecutionState) connectionAwareState.getStoreExecutionState(StoreType.Relational)).getBlockConnectionContext().unlockAllBlockConnections();
return res;
} catch (Exception e) {
((RelationalStoreExecutionState) connectionAwareState.getStoreExecutionState(StoreType.Relational)).getBlockConnectionContext().unlockAllBlockConnections();
((RelationalStoreExecutionState) connectionAwareState.getStoreExecutionState(StoreType.Relational)).getBlockConnectionContext().closeAllBlockConnections();
throw e;
}
} else if (executionNode instanceof CreateAndPopulateTempTableExecutionNode) {
CreateAndPopulateTempTableExecutionNode createAndPopulateTempTableExecutionNode = (CreateAndPopulateTempTableExecutionNode) executionNode;
Stream<Result> results = createAndPopulateTempTableExecutionNode.inputVarNames.stream().map(this.executionState::getResult);
Stream<?> inputStream = results.flatMap(result -> {
if (result instanceof ConstantResult) {
Object value = ((ConstantResult) result).getValue();
if (value instanceof Map && ((Map<?, ?>) value).get("values") instanceof List) {
return ((List<?>) ((Map<?, ?>) value).get("values")).stream().map(val -> ((List<?>) ((Map<?, ?>) val).get("values")).get(0));
}
if (value instanceof List) {
return ((List<?>) value).stream();
}
if (ClassUtils.isPrimitiveOrWrapper(value.getClass()) || (value instanceof String)) {
return Stream.of(value);
}
if (value instanceof Stream) {
return (Stream<?>) value;
}
throw new IllegalArgumentException("Result passed into CreateAndPopulateTempTableExecutionNode should be a stream");
}
if (result instanceof StreamingObjectResult) {
return ((StreamingObjectResult<?>) result).getObjectStream();
}
throw new IllegalArgumentException("Unexpected Result Type : " + result.getClass().getName());
});
if (!(createAndPopulateTempTableExecutionNode.implementation instanceof JavaPlatformImplementation)) {
throw new RuntimeException("Only Java implementations are currently supported, found: " + createAndPopulateTempTableExecutionNode.implementation);
}
JavaPlatformImplementation javaPlatformImpl = (JavaPlatformImplementation) createAndPopulateTempTableExecutionNode.implementation;
String executionClassName = JavaHelper.getExecutionClassFullName(javaPlatformImpl);
Class<?> clazz = ExecutionNodeJavaPlatformHelper.getClassToExecute(createAndPopulateTempTableExecutionNode, executionClassName, this.executionState, this.profiles);
if (Arrays.asList(clazz.getInterfaces()).contains(IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics.class)) {
try {
IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics nodeSpecifics = (IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics) clazz.newInstance();
createAndPopulateTempTableExecutionNode.tempTableColumnMetaData.forEach(t -> t.identifierForGetter = nodeSpecifics.getGetterNameForProperty(t.identifierForGetter));
} catch (InstantiationException | IllegalAccessException e) {
throw new RuntimeException(e);
}
} else {
// TODO Remove once platform version supports above and existing plans mitigated
String executionMethodName = JavaHelper.getExecutionMethodName(javaPlatformImpl);
createAndPopulateTempTableExecutionNode.tempTableColumnMetaData.forEach(t -> t.identifierForGetter = ExecutionNodeJavaPlatformHelper.executeStaticJavaMethod(createAndPopulateTempTableExecutionNode, executionClassName, executionMethodName, Collections.singletonList(Result.class), Collections.singletonList(new ConstantResult(t.identifierForGetter)), this.executionState, this.profiles));
}
RelationalDatabaseCommands databaseCommands = DatabaseManager.fromString(createAndPopulateTempTableExecutionNode.connection.type.name()).relationalDatabaseSupport();
try (Connection connectionManagerConnection = this.getConnection(createAndPopulateTempTableExecutionNode, databaseCommands, this.profiles, this.executionState)) {
TempTableStreamingResult tempTableStreamingResult = new TempTableStreamingResult(inputStream, createAndPopulateTempTableExecutionNode);
String databaseTimeZone = createAndPopulateTempTableExecutionNode.connection.timeZone == null ? RelationalExecutor.DEFAULT_DB_TIME_ZONE : createAndPopulateTempTableExecutionNode.connection.timeZone;
databaseCommands.accept(RelationalDatabaseCommandsVisitorBuilder.getStreamResultToTempTableVisitor(((RelationalStoreExecutionState) this.executionState.getStoreExecutionState(StoreType.Relational)).getRelationalExecutor().getRelationalExecutionConfiguration(), connectionManagerConnection, tempTableStreamingResult, createAndPopulateTempTableExecutionNode.tempTableName, databaseTimeZone));
} catch (SQLException e) {
throw new RuntimeException(e);
}
return new ConstantResult("success");
} else if (executionNode instanceof RelationalExecutionNode) {
RelationalExecutionNode relationalExecutionNode = (RelationalExecutionNode) executionNode;
Span topSpan = GlobalTracer.get().activeSpan();
this.executionState.topSpan = topSpan;
try (Scope scope = GlobalTracer.get().buildSpan("Relational DB Execution").startActive(true)) {
scope.span().setTag("databaseType", relationalExecutionNode.getDatabaseTypeName());
scope.span().setTag("sql", relationalExecutionNode.sqlQuery());
Result result = ((RelationalStoreExecutionState) executionState.getStoreExecutionState(StoreType.Relational)).getRelationalExecutor().execute(relationalExecutionNode, this.profiles, this.executionState);
if (result instanceof RelationalResult) {
scope.span().setTag("executedSql", ((RelationalResult) result).executedSQl);
}
if (relationalExecutionNode.implementation != null && !(ExecutionNodeResultHelper.isResultSizeRangeSet(relationalExecutionNode) && ExecutionNodeResultHelper.isSingleRecordResult(relationalExecutionNode))) {
return executeImplementation(relationalExecutionNode, result, this.executionState, this.profiles);
}
return result;
}
} else if (executionNode instanceof SQLExecutionNode) {
SQLExecutionNode SQLExecutionNode = (SQLExecutionNode) executionNode;
this.executionState.topSpan = GlobalTracer.get().activeSpan();
try (Scope scope = GlobalTracer.get().buildSpan("Relational DB Execution").startActive(true)) {
scope.span().setTag("databaseType", SQLExecutionNode.getDatabaseTypeName());
scope.span().setTag("sql", SQLExecutionNode.sqlQuery());
Result result = ((RelationalStoreExecutionState) executionState.getStoreExecutionState(StoreType.Relational)).getRelationalExecutor().execute(SQLExecutionNode, profiles, executionState);
if (result instanceof SQLExecutionResult) {
scope.span().setTag("executedSql", ((SQLExecutionResult) result).getExecutedSql());
}
return result;
}
} else if (executionNode instanceof RelationalTdsInstantiationExecutionNode) {
RelationalTdsInstantiationExecutionNode relationalTdsInstantiationExecutionNode = (RelationalTdsInstantiationExecutionNode) executionNode;
SQLExecutionResult sqlExecutionResult = null;
try {
sqlExecutionResult = (SQLExecutionResult) this.visit((SQLExecutionNode) relationalTdsInstantiationExecutionNode.executionNodes.get(0));
RelationalResult relationalTdsResult = new RelationalResult(sqlExecutionResult, relationalTdsInstantiationExecutionNode);
if (this.executionState.inAllocation) {
if (!this.executionState.transformAllocation) {
return relationalTdsResult;
}
RealizedRelationalResult realizedRelationalResult = (RealizedRelationalResult) relationalTdsResult.realizeInMemory();
List<Map<String, Object>> rowValueMaps = realizedRelationalResult.getRowValueMaps(false);
Result res = RelationalExecutor.evaluateAdditionalExtractors(this.resultInterpreterExtensions, this.executionState, rowValueMaps);
if (res != null) {
return res;
} else {
return new ConstantResult(rowValueMaps);
}
}
return relationalTdsResult;
} catch (Exception e) {
if (sqlExecutionResult != null) {
sqlExecutionResult.close();
}
throw e;
}
} else if (executionNode instanceof RelationalClassInstantiationExecutionNode) {
RelationalClassInstantiationExecutionNode node = (RelationalClassInstantiationExecutionNode) executionNode;
SQLExecutionResult sqlExecutionResult = null;
try {
SQLExecutionNode innerNode = (SQLExecutionNode) node.executionNodes.get(0);
sqlExecutionResult = (SQLExecutionResult) this.visit(innerNode);
RelationalResult relationalResult = new RelationalResult(sqlExecutionResult, node);
boolean realizeAsConstant = this.executionState.inAllocation && ExecutionNodeResultHelper.isResultSizeRangeSet(node) && ExecutionNodeResultHelper.isSingleRecordResult(node);
if (realizeAsConstant) {
RealizedRelationalResult realizedRelationalResult = (RealizedRelationalResult) relationalResult.realizeInMemory();
List<Map<String, Object>> rowValueMaps = realizedRelationalResult.getRowValueMaps(false);
if (rowValueMaps.size() == 1) {
return new ConstantResult(rowValueMaps.get(0));
} else {
return new ConstantResult(rowValueMaps);
}
}
return this.getStreamingObjectResultFromRelationalResult(node, relationalResult, innerNode.connection);
} catch (Exception e) {
if (sqlExecutionResult != null) {
sqlExecutionResult.close();
}
throw (e instanceof RuntimeException) ? (RuntimeException) e : new RuntimeException(e);
}
} else if (executionNode instanceof RelationalRelationDataInstantiationExecutionNode) {
RelationalRelationDataInstantiationExecutionNode node = (RelationalRelationDataInstantiationExecutionNode) executionNode;
SQLExecutionResult sqlExecutionResult = null;
try {
sqlExecutionResult = (SQLExecutionResult) this.visit((SQLExecutionNode) node.executionNodes.get(0));
return new RelationalResult(sqlExecutionResult, node);
} catch (Exception e) {
if (sqlExecutionResult != null) {
sqlExecutionResult.close();
}
throw e;
}
} else if (executionNode instanceof RelationalDataTypeInstantiationExecutionNode) {
RelationalDataTypeInstantiationExecutionNode node = (RelationalDataTypeInstantiationExecutionNode) executionNode;
SQLExecutionResult sqlExecutionResult = null;
try {
sqlExecutionResult = (SQLExecutionResult) this.visit((SQLExecutionNode) node.executionNodes.get(0));
RelationalResult relationalPrimitiveResult = new RelationalResult(sqlExecutionResult, node);
if (this.executionState.inAllocation) {
if ((ExecutionNodeResultHelper.isResultSizeRangeSet(node) && !ExecutionNodeResultHelper.isSingleRecordResult(node)) && !this.executionState.transformAllocation) {
return relationalPrimitiveResult;
}
if (relationalPrimitiveResult.getResultSet().next()) {
List<org.eclipse.collections.api.block.function.Function<Object, Object>> transformers = relationalPrimitiveResult.getTransformers();
Object convertedValue = transformers.get(0).valueOf(relationalPrimitiveResult.getResultSet().getObject(1));
return new ConstantResult(convertedValue);
} else {
throw new RuntimeException("Result set is empty for allocation node");
}
}
return relationalPrimitiveResult;
} catch (Exception e) {
if (sqlExecutionResult != null) {
sqlExecutionResult.close();
}
throw (e instanceof RuntimeException) ? (RuntimeException) e : new RuntimeException(e);
}
} else if (executionNode instanceof RelationalRootQueryTempTableGraphFetchExecutionNode) {
return this.executeRelationalRootQueryTempTableGraphFetchExecutionNode((RelationalRootQueryTempTableGraphFetchExecutionNode) executionNode);
} else if (executionNode instanceof RelationalCrossRootQueryTempTableGraphFetchExecutionNode) {
return this.executeRelationalCrossRootQueryTempTableGraphFetchExecutionNode((RelationalCrossRootQueryTempTableGraphFetchExecutionNode) executionNode);
} else if (executionNode instanceof RelationalPrimitiveQueryGraphFetchExecutionNode) {
return this.executeRelationalPrimitiveQueryGraphFetchExecutionNode((RelationalPrimitiveQueryGraphFetchExecutionNode) executionNode);
} else if (executionNode instanceof RelationalClassQueryTempTableGraphFetchExecutionNode) {
return this.executeRelationalClassQueryTempTableGraphFetchExecutionNode((RelationalClassQueryTempTableGraphFetchExecutionNode) executionNode);
} else if (executionNode instanceof RelationalRootGraphFetchExecutionNode) {
RelationalRootGraphFetchExecutionNode node = (RelationalRootGraphFetchExecutionNode) executionNode;
/* Fetch info from execution state */
GraphExecutionState graphExecutionState = (GraphExecutionState) executionState;
int batchSize = graphExecutionState.getBatchSize();
SQLExecutionResult rootResult = (SQLExecutionResult) graphExecutionState.getRootResult();
ResultSet rootResultSet = rootResult.getResultSet();
/* Ensure all children run in the same connection */
RelationalStoreExecutionState relationalStoreExecutionState = (RelationalStoreExecutionState) graphExecutionState.getStoreExecutionState(StoreType.Relational);
BlockConnectionContext oldBlockConnectionContext = relationalStoreExecutionState.getBlockConnectionContext();
boolean oldRetainConnectionFlag = relationalStoreExecutionState.retainConnection();
relationalStoreExecutionState.setBlockConnectionContext(new BlockConnectionContext());
relationalStoreExecutionState.setRetainConnection(true);
try (Scope ignored1 = GlobalTracer.get().buildSpan("Graph Query Relational: Execute Relational Root").startActive(true)) {
String databaseTimeZone = rootResult.getDatabaseTimeZone();
String databaseConnectionString = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(rootResult.getSQLExecutionNode().connection);
/* Get Java executor */
Class<?> executeClass = this.getExecuteClass(node);
if (Arrays.asList(executeClass.getInterfaces()).contains(IRelationalRootGraphNodeExecutor.class)) {
IRelationalRootGraphNodeExecutor executor = (IRelationalRootGraphNodeExecutor) executeClass.getConstructor().newInstance();
List<Method> primaryKeyGetters = executor.primaryKeyGetters();
int primaryKeyCount = primaryKeyGetters.size();
/* Check if caching is enabled and fetch the cache if required */
boolean cachingEnabledForNode = false;
ExecutionCache<GraphFetchCacheKey, Object> graphCache = null;
RelationalGraphFetchUtils.RelationalSQLResultGraphFetchCacheKey rootResultCacheKey = null;
if ((this.executionState.graphFetchCaches != null) && executor.supportsCaching()) {
GraphFetchCacheByEqualityKeys graphFetchCacheByEqualityKeys = RelationalGraphFetchUtils.findCacheByEqualityKeys(node.graphFetchTree, executor.getMappingId(rootResultSet, databaseTimeZone, databaseConnectionString), executor.getInstanceSetId(rootResultSet, databaseTimeZone, databaseConnectionString), this.executionState.graphFetchCaches);
if (graphFetchCacheByEqualityKeys != null) {
List<String> parentSQLKeyColumns = executor.primaryKeyColumns();
List<Integer> parentPrimaryKeyIndices = FastList.newList();
for (String pkCol : parentSQLKeyColumns) {
parentPrimaryKeyIndices.add(rootResultSet.findColumn(pkCol));
}
cachingEnabledForNode = true;
graphCache = graphFetchCacheByEqualityKeys.getExecutionCache();
rootResultCacheKey = new RelationalGraphFetchUtils.RelationalSQLResultGraphFetchCacheKey(rootResult, parentPrimaryKeyIndices);
}
}
/* Get the next batch of root records */
List<Object> resultObjectsBatch = new ArrayList<>();
List<org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?>> instancesToDeepFetch = new ArrayList<>();
int objectCount = 0;
try (Scope ignored2 = GlobalTracer.get().buildSpan("Graph Query Relational: Read Next Batch").startActive(true)) {
while (rootResultSet.next()) {
graphExecutionState.incrementRowCount();
boolean shouldDeepFetchOnThisInstance = true;
if (cachingEnabledForNode) {
Object cachedObject = graphCache.getIfPresent(rootResultCacheKey);
if (cachedObject != null) {
resultObjectsBatch.add(executor.deepCopy(cachedObject));
shouldDeepFetchOnThisInstance = false;
}
}
if (shouldDeepFetchOnThisInstance) {
org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?> wrappedObject = executor.getObjectFromResultSet(rootResultSet, databaseTimeZone, databaseConnectionString);
instancesToDeepFetch.add(wrappedObject);
resultObjectsBatch.add(wrappedObject.getValue());
}
objectCount += 1;
if (objectCount >= batchSize) {
break;
}
}
}
if (!instancesToDeepFetch.isEmpty()) {
boolean childrenExist = node.children != null && !node.children.isEmpty();
String tempTableName = node.tempTableName;
RealizedRelationalResult realizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.columns);
/* Create and populate double strategy map with key being object with its PK getters */
DoubleStrategyHashMap<Object, Object, SQLExecutionResult> rootMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
for (org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?> rootGraphInstance : instancesToDeepFetch) {
Object rootObject = rootGraphInstance.getValue();
rootMap.put(rootObject, rootObject);
graphExecutionState.addObjectMemoryUtilization(rootGraphInstance.instanceSize());
if (childrenExist) {
this.addKeyRowToRealizedRelationalResult(rootObject, primaryKeyGetters, realizedRelationalResult);
}
}
/* Execute store local children */
if (childrenExist) {
this.executeRelationalChildren(node, tempTableName, realizedRelationalResult, rootResult.getSQLExecutionNode().connection, rootResult.getDatabaseType(), databaseTimeZone, rootMap, primaryKeyGetters);
}
}
graphExecutionState.setObjectsForNodeIndex(node.nodeIndex, resultObjectsBatch);
if (cachingEnabledForNode) {
for (org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?> deepFetchedInstance : instancesToDeepFetch) {
Object objectClone = executor.deepCopy(deepFetchedInstance.getValue());
graphCache.put(new RelationalGraphFetchUtils.RelationalObjectGraphFetchCacheKey(objectClone, primaryKeyGetters), objectClone);
}
}
return new ConstantResult(resultObjectsBatch);
} else {
throw new RuntimeException("Unknown execute class " + executeClass.getCanonicalName());
}
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
relationalStoreExecutionState.getBlockConnectionContext().unlockAllBlockConnections();
relationalStoreExecutionState.getBlockConnectionContext().closeAllBlockConnectionsAsync();
relationalStoreExecutionState.setBlockConnectionContext(oldBlockConnectionContext);
relationalStoreExecutionState.setRetainConnection(oldRetainConnectionFlag);
}
} else if (executionNode instanceof RelationalCrossRootGraphFetchExecutionNode) {
RelationalCrossRootGraphFetchExecutionNode node = (RelationalCrossRootGraphFetchExecutionNode) executionNode;
GraphExecutionState graphExecutionState = (GraphExecutionState) executionState;
List<?> parentObjects = graphExecutionState.getObjectsToGraphFetch();
List<Object> childObjects = FastList.newList();
graphExecutionState.setObjectsForNodeIndex(node.nodeIndex, childObjects);
RelationalStoreExecutionState relationalStoreExecutionState = (RelationalStoreExecutionState) graphExecutionState.getStoreExecutionState(StoreType.Relational);
BlockConnectionContext oldBlockConnectionContext = relationalStoreExecutionState.getBlockConnectionContext();
boolean oldRetainConnectionFlag = relationalStoreExecutionState.retainConnection();
relationalStoreExecutionState.setBlockConnectionContext(new BlockConnectionContext());
relationalStoreExecutionState.setRetainConnection(true);
SQLExecutionResult childResult = null;
try (Scope ignored1 = GlobalTracer.get().buildSpan("Graph Query Relational: Execute Relational Cross Root").startActive(true)) {
/* Get Java executor */
Class<?> executeClass = this.getExecuteClass(node);
if (Arrays.asList(executeClass.getInterfaces()).contains(IRelationalCrossRootGraphNodeExecutor.class)) {
IRelationalCrossRootGraphNodeExecutor executor = (IRelationalCrossRootGraphNodeExecutor) executeClass.getConstructor().newInstance();
if (!parentObjects.isEmpty()) {
String parentTempTableName = node.parentTempTableName;
RealizedRelationalResult parentRealizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.parentTempTableColumns);
List<Method> crossKeyGetters = executor.parentCrossKeyGetters();
int parentKeyCount = crossKeyGetters.size();
for (Object parentObject : parentObjects) {
this.addKeyRowToRealizedRelationalResult(parentObject, crossKeyGetters, parentRealizedRelationalResult);
}
graphExecutionState.addResult(parentTempTableName, parentRealizedRelationalResult);
/* Execute relational node corresponding to the cross root */
childResult = (SQLExecutionResult) node.relationalNode.accept(new ExecutionNodeExecutor(this.profiles, graphExecutionState));
ResultSet childResultSet = childResult.getResultSet();
boolean childrenExist = node.children != null && !node.children.isEmpty();
String tempTableName = childrenExist ? node.tempTableName : null;
RealizedRelationalResult realizedRelationalResult = childrenExist ? RealizedRelationalResult.emptyRealizedRelationalResult(node.columns) : null;
DatabaseConnection databaseConnection = childResult.getSQLExecutionNode().connection;
String databaseType = childResult.getDatabaseType();
String databaseTimeZone = childResult.getDatabaseTimeZone();
List<String> parentSQLKeyColumns = executor.parentSQLColumnsInResultSet(childResult.getResultColumns().stream().map(ResultColumn::getNonQuotedLabel).collect(Collectors.toList()));
List<Integer> parentCrossKeyIndices = FastList.newList();
for (String pkCol : parentSQLKeyColumns) {
parentCrossKeyIndices.add(childResultSet.findColumn(pkCol));
}
DoubleStrategyHashMap<Object, List<Object>, SQLExecutionResult> parentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategy(crossKeyGetters, parentCrossKeyIndices));
for (Object parentObject : parentObjects) {
List<Object> mapped = parentMap.get(parentObject);
if (mapped == null) {
parentMap.put(parentObject, FastList.newListWith(parentObject));
} else {
mapped.add(parentObject);
}
}
List<Method> primaryKeyGetters = executor.primaryKeyGetters();
final int primaryKeyCount = primaryKeyGetters.size();
DoubleStrategyHashMap<Object, Object, SQLExecutionResult> currentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
String databaseConnectionString = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(childResult.getSQLExecutionNode().connection);
while (childResultSet.next()) {
graphExecutionState.incrementRowCount();
List<Object> parents = parentMap.getWithSecondKey(childResult);
if (parents == null) {
throw new RuntimeException("No parent");
}
org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?> childGraphInstance = executor.getObjectFromResultSet(childResultSet, childResult.getDatabaseTimeZone(), databaseConnectionString);
Object child = childGraphInstance.getValue();
Object mapObject = currentMap.putIfAbsent(child, child);
if (mapObject == null) {
mapObject = child;
childObjects.add(mapObject);
graphExecutionState.addObjectMemoryUtilization(childGraphInstance.instanceSize());
if (childrenExist) {
this.addKeyRowToRealizedRelationalResult(child, primaryKeyGetters, realizedRelationalResult);
}
}
for (Object parent : parents) {
executor.addChildToParent(parent, mapObject, DefaultExecutionNodeContext.factory().create(graphExecutionState, null));
}
}
childResult.close();
childResult = null;
if (childrenExist) {
this.executeRelationalChildren(node, tempTableName, realizedRelationalResult, databaseConnection, databaseType, databaseTimeZone, currentMap, primaryKeyGetters);
}
}
} else {
throw new RuntimeException("Unknown execute class " + executeClass.getCanonicalName());
}
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (childResult != null) {
childResult.close();
}
relationalStoreExecutionState.getBlockConnectionContext().unlockAllBlockConnections();
relationalStoreExecutionState.getBlockConnectionContext().closeAllBlockConnectionsAsync();
relationalStoreExecutionState.setBlockConnectionContext(oldBlockConnectionContext);
relationalStoreExecutionState.setRetainConnection(oldRetainConnectionFlag);
}
return new ConstantResult(childObjects);
}
throw new RuntimeException("Not implemented!");
}
use of org.finos.legend.engine.plan.execution.stores.relational.result.SQLExecutionResult in project legend-engine by finos.
the class RelationalExecutionNodeExecutor method executeRelationalCrossRootQueryTempTableGraphFetchExecutionNode.
private Result executeRelationalCrossRootQueryTempTableGraphFetchExecutionNode(RelationalCrossRootQueryTempTableGraphFetchExecutionNode node) {
boolean isLeaf = node.children == null || node.children.isEmpty();
List<Object> childObjects = new ArrayList<>();
Result childResult = null;
RelationalStoreExecutionState relationalStoreExecutionState = (RelationalStoreExecutionState) this.executionState.getStoreExecutionState(StoreType.Relational);
BlockConnectionContext oldBlockConnectionContext = relationalStoreExecutionState.getBlockConnectionContext();
boolean oldRetainConnectionFlag = relationalStoreExecutionState.retainConnection();
relationalStoreExecutionState.setBlockConnectionContext(new BlockConnectionContext());
relationalStoreExecutionState.setRetainConnection(true);
try (Scope ignored = GlobalTracer.get().buildSpan("cross store property graph fetch").withTag("storeType", "relational").withTag("property", ((PropertyGraphFetchTree) node.graphFetchTree).property).startActive(true)) {
IRelationalCrossRootQueryTempTableGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.profiles);
RelationalGraphObjectsBatch relationalGraphObjectsBatch = new RelationalGraphObjectsBatch(this.executionState.graphObjectsBatch);
List<?> parentObjects = relationalGraphObjectsBatch.getObjectsForNodeIndex(node.parentIndex);
if ((parentObjects != null) && !parentObjects.isEmpty()) {
GraphFetchTree nodeSubTree = node.graphFetchTree;
boolean cachingEnabled = false;
ExecutionCache<GraphFetchCacheKey, List<Object>> crossCache = relationalGraphObjectsBatch.getXStorePropertyCacheForNodeIndex(node.nodeIndex);
List<Method> parentCrossKeyGettersOrderedPerTargetProperties = null;
if (crossCache != null) {
cachingEnabled = true;
parentCrossKeyGettersOrderedPerTargetProperties = nodeSpecifics.parentCrossKeyGettersOrderedByTargetProperties();
}
List<Object> parentsToDeepFetch = new ArrayList<>();
for (Object parent : parentObjects) {
if (cachingEnabled) {
List<Object> children = crossCache.getIfPresent(new RelationalGraphFetchUtils.RelationalCrossObjectGraphFetchCacheKey(parent, parentCrossKeyGettersOrderedPerTargetProperties));
if (children == null) {
parentsToDeepFetch.add(parent);
} else {
for (Object child : children) {
childObjects.add(child);
nodeSpecifics.addChildToParent(parent, child, DefaultExecutionNodeContext.factory().create(this.executionState, null));
}
}
} else {
parentsToDeepFetch.add(parent);
}
}
if (!parentsToDeepFetch.isEmpty()) {
Map<Object, List<Object>> parentToChildMap = new HashMap<>();
RealizedRelationalResult parentRealizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.parentTempTableColumns);
List<Method> crossKeyGetters = nodeSpecifics.parentCrossKeyGetters();
for (Object parentObject : parentsToDeepFetch) {
this.addKeyRowToRealizedRelationalResult(parentObject, crossKeyGetters, parentRealizedRelationalResult);
parentToChildMap.put(parentObject, new ArrayList<>());
}
this.executionState.addResult(node.parentTempTableName, parentRealizedRelationalResult);
childResult = node.executionNodes.get(0).accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
SQLExecutionResult childSqlResult = (SQLExecutionResult) childResult;
DatabaseConnection databaseConnection = childSqlResult.getSQLExecutionNode().connection;
ResultSet childResultSet = childSqlResult.getResultSet();
List<String> parentSQLKeyColumns = nodeSpecifics.parentCrossKeyColumns(childSqlResult.getResultColumns().stream().map(ResultColumn::getNonQuotedLabel).collect(Collectors.toList()));
List<Integer> parentCrossKeyIndices = parentSQLKeyColumns.stream().map(FunctionHelper.unchecked(childResultSet::findColumn)).collect(Collectors.toList());
@SuppressWarnings("MismatchedQueryAndUpdateOfCollection") DoubleStrategyHashMap<Object, List<Object>, SQLExecutionResult> parentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategy(crossKeyGetters, parentCrossKeyIndices));
parentsToDeepFetch.forEach((o) -> parentMap.getIfAbsentPut(o, ArrayList::new).add(o));
RealizedRelationalResult realizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.columns);
List<Method> primaryKeyGetters = nodeSpecifics.primaryKeyGetters();
DoubleStrategyHashMap<Object, Object, SQLExecutionResult> currentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
/* Prepare for reading */
nodeSpecifics.prepare(childResultSet, childSqlResult.getDatabaseTimeZone(), ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(databaseConnection));
while (childResultSet.next()) {
relationalGraphObjectsBatch.incrementRowCount();
List<Object> parents = parentMap.getWithSecondKey(childSqlResult);
if (parents == null) {
throw new RuntimeException("Cannot find the parent for child");
}
IGraphInstance<? extends IReferencedObject> childGraphInstance = nodeSpecifics.nextGraphInstance();
Object child = childGraphInstance.getValue();
Object mapObject = currentMap.putIfAbsent(child, child);
if (mapObject == null) {
mapObject = child;
childObjects.add(mapObject);
relationalGraphObjectsBatch.addObjectMemoryUtilization(childGraphInstance.instanceSize());
if (!isLeaf) {
this.addKeyRowToRealizedRelationalResult(child, primaryKeyGetters, realizedRelationalResult);
}
}
for (Object parent : parents) {
if (parentToChildMap.containsKey(parent)) {
parentToChildMap.get(parent).add(mapObject);
} else {
parentToChildMap.put(parent, new ArrayList<>());
parentToChildMap.get(parent).add(mapObject);
}
nodeSpecifics.addChildToParent(parent, mapObject, DefaultExecutionNodeContext.factory().create(this.executionState, null));
}
}
relationalGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, childObjects);
childResult.close();
childResult = null;
/* Execute store local children */
if (!isLeaf) {
ExecutionState newState = new ExecutionState(executionState);
newState.graphObjectsBatch = relationalGraphObjectsBatch;
this.executeTempTableNodeChildren(node, realizedRelationalResult, databaseConnection, childSqlResult.getDatabaseType(), childSqlResult.getDatabaseTimeZone(), currentMap, primaryKeyGetters, newState);
}
if (cachingEnabled) {
List<Method> getters = parentCrossKeyGettersOrderedPerTargetProperties;
parentToChildMap.forEach((p, cs) -> {
crossCache.put(new RelationalGraphFetchUtils.RelationalCrossObjectGraphFetchCacheKey(p, getters), cs);
});
}
}
}
return new ConstantResult(childObjects);
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (childResult != null) {
childResult.close();
}
relationalStoreExecutionState.getBlockConnectionContext().unlockAllBlockConnections();
relationalStoreExecutionState.getBlockConnectionContext().closeAllBlockConnectionsAsync();
relationalStoreExecutionState.setBlockConnectionContext(oldBlockConnectionContext);
relationalStoreExecutionState.setRetainConnection(oldRetainConnectionFlag);
}
}
use of org.finos.legend.engine.plan.execution.stores.relational.result.SQLExecutionResult in project legend-engine by finos.
the class RelationalExecutionNodeExecutor method executeLocalRelationalGraphOperation.
private void executeLocalRelationalGraphOperation(RelationalGraphFetchExecutionNode node, DoubleStrategyHashMap<Object, Object, SQLExecutionResult> parentMap, List<Method> parentKeyGetters) {
GraphExecutionState graphExecutionState = (GraphExecutionState) executionState;
List<Object> childObjects = FastList.newList();
graphExecutionState.setObjectsForNodeIndex(node.nodeIndex, childObjects);
SQLExecutionResult childResult = null;
try {
/* Get Java executor */
Class<?> executeClass = this.getExecuteClass(node);
if (Arrays.asList(executeClass.getInterfaces()).contains(IRelationalChildGraphNodeExecutor.class)) {
IRelationalChildGraphNodeExecutor executor = (IRelationalChildGraphNodeExecutor) executeClass.getConstructor().newInstance();
/* Execute relational node corresponding to the child */
childResult = (SQLExecutionResult) node.relationalNode.accept(new ExecutionNodeExecutor(profiles, executionState));
boolean nonPrimitiveNode = node.resultType instanceof ClassResultType;
boolean childrenExist = node.children != null && !node.children.isEmpty();
/* Change the second strategy to suit the primary key indices of parent PKs in the child result set*/
List<String> parentSQLKeyColumns = executor.parentSQLColumnsInResultSet(childResult.getResultColumns().stream().map(ResultColumn::getNonQuotedLabel).collect(Collectors.toList()));
List<Integer> parentPrimaryKeyIndices = FastList.newList();
for (String pkCol : parentSQLKeyColumns) {
parentPrimaryKeyIndices.add(childResult.getResultSet().findColumn(pkCol));
}
RelationalGraphFetchUtils.switchSecondKeyHashingStrategy(parentMap, parentKeyGetters, parentPrimaryKeyIndices);
String databaseConnectionString = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(childResult.getSQLExecutionNode().connection);
if (nonPrimitiveNode) {
List<Method> primaryKeyGetters = executor.primaryKeyGetters();
int primaryKeyCount = primaryKeyGetters.size();
DoubleStrategyHashMap<Object, Object, SQLExecutionResult> currentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
String tempTableName = childrenExist ? ((RelationalTempTableGraphFetchExecutionNode) node).tempTableName : null;
RealizedRelationalResult realizedRelationalResult = childrenExist ? RealizedRelationalResult.emptyRealizedRelationalResult(((RelationalTempTableGraphFetchExecutionNode) node).columns) : null;
DatabaseConnection databaseConnection = childResult.getSQLExecutionNode().connection;
String databaseType = childResult.getDatabaseType();
String databaseTimeZone = childResult.getDatabaseTimeZone();
ResultSet childResultSet = childResult.getResultSet();
try (Scope ignored1 = GlobalTracer.get().buildSpan("Graph Query Relational: Read Child Batch").startActive(true)) {
while (childResultSet.next()) {
graphExecutionState.incrementRowCount();
Object parent = parentMap.getWithSecondKey(childResult);
if (parent == null) {
throw new RuntimeException("No parent");
}
org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?> childGraphInstance = executor.getObjectFromResultSet(childResultSet, childResult.getDatabaseTimeZone(), databaseConnectionString);
Object child = childGraphInstance.getValue();
Object mapObject = currentMap.putIfAbsent(child, child);
if (mapObject == null) {
mapObject = child;
graphExecutionState.addObjectMemoryUtilization(childGraphInstance.instanceSize());
childObjects.add(mapObject);
if (childrenExist) {
this.addKeyRowToRealizedRelationalResult(child, primaryKeyGetters, realizedRelationalResult);
}
}
executor.addChildToParent(parent, mapObject, DefaultExecutionNodeContext.factory().create(graphExecutionState, null));
}
}
childResult.close();
childResult = null;
if (childrenExist) {
this.executeRelationalChildren(node, tempTableName, realizedRelationalResult, databaseConnection, databaseType, databaseTimeZone, currentMap, primaryKeyGetters);
}
} else {
ResultSet childResultSet = childResult.getResultSet();
while (childResultSet.next()) {
Object parent = parentMap.getWithSecondKey(childResult);
if (parent == null) {
throw new RuntimeException("No parent");
}
org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?> childGraphInstance = executor.getObjectFromResultSet(childResultSet, childResult.getDatabaseTimeZone(), databaseConnectionString);
Object child = childGraphInstance.getValue();
childObjects.add(child);
graphExecutionState.addObjectMemoryUtilization(childGraphInstance.instanceSize());
executor.addChildToParent(parent, child, DefaultExecutionNodeContext.factory().create(graphExecutionState, null));
}
childResult.close();
childResult = null;
}
} else {
throw new RuntimeException("Unknown execute class " + executeClass.getCanonicalName());
}
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (childResult != null) {
childResult.close();
}
}
}
use of org.finos.legend.engine.plan.execution.stores.relational.result.SQLExecutionResult in project legend-engine by finos.
the class RelationalExecutionNodeExecutor method executeRelationalClassQueryTempTableGraphFetchExecutionNode.
private Result executeRelationalClassQueryTempTableGraphFetchExecutionNode(RelationalClassQueryTempTableGraphFetchExecutionNode node) {
boolean isLeaf = node.children == null || node.children.isEmpty();
List<Object> childObjects = new ArrayList<>();
List<Pair<IGraphInstance<? extends IReferencedObject>, ExecutionCache<GraphFetchCacheKey, Object>>> childInstancesToDeepFetchAndCache = new ArrayList<>();
Result childResult = null;
try (Scope ignored = GlobalTracer.get().buildSpan("local store property graph fetch").withTag("storeType", "relational").withTag("property", ((PropertyGraphFetchTree) node.graphFetchTree).property).startActive(true)) {
RelationalGraphObjectsBatch relationalGraphObjectsBatch = (RelationalGraphObjectsBatch) this.executionState.graphObjectsBatch;
childResult = node.executionNodes.get(0).accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
SQLExecutionResult childSqlResult = (SQLExecutionResult) childResult;
DatabaseConnection databaseConnection = childSqlResult.getSQLExecutionNode().connection;
ResultSet childResultSet = childSqlResult.getResultSet();
IRelationalClassQueryTempTableGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.profiles);
/* Check if caching is enabled and fetch caches if required */
List<Pair<String, String>> allInstanceSetImplementations = nodeSpecifics.allInstanceSetImplementations();
int setIdCount = allInstanceSetImplementations.size();
boolean isUnion = setIdCount > 1;
RelationalMultiSetExecutionCacheWrapper multiSetCache = new RelationalMultiSetExecutionCacheWrapper(setIdCount);
boolean cachingEnabledForNode = this.checkForCachingAndPopulateCachingHelpers(allInstanceSetImplementations, nodeSpecifics.supportsCaching(), node.graphFetchTree, childSqlResult, nodeSpecifics::primaryKeyColumns, multiSetCache);
DoubleStrategyHashMap<Object, Object, SQLExecutionResult> parentMap = this.switchedParentHashMapPerChildResult(relationalGraphObjectsBatch, node.parentIndex, childResultSet, () -> nodeSpecifics.parentPrimaryKeyColumns(childSqlResult.getResultColumns().stream().map(ResultColumn::getNonQuotedLabel).collect(Collectors.toList())));
List<Method> primaryKeyGetters = nodeSpecifics.primaryKeyGetters();
DoubleStrategyHashMap<Object, Object, SQLExecutionResult> currentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
RealizedRelationalResult realizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.columns);
/* Prepare for reading */
nodeSpecifics.prepare(childResultSet, childSqlResult.getDatabaseTimeZone(), ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(databaseConnection));
while (childResultSet.next()) {
relationalGraphObjectsBatch.incrementRowCount();
Object child;
int setIndex = isUnion ? childResultSet.getInt(1) : 0;
Object cachedObject = RelationalExecutionNodeExecutor.this.checkAndReturnCachedObject(cachingEnabledForNode, setIndex, multiSetCache);
boolean shouldDeepFetchOnThisInstance = cachedObject == null;
if (shouldDeepFetchOnThisInstance) {
IGraphInstance<? extends IReferencedObject> wrappedObject = nodeSpecifics.nextGraphInstance();
Object wrappedValue = wrappedObject.getValue();
Object mapObject = currentMap.putIfAbsent(wrappedValue, wrappedValue);
if (mapObject == null) {
child = wrappedValue;
childInstancesToDeepFetchAndCache.add(Tuples.pair(wrappedObject, multiSetCache.setCaches.get(setIndex)));
relationalGraphObjectsBatch.addObjectMemoryUtilization(wrappedObject.instanceSize());
childObjects.add(child);
if (!isLeaf) {
this.addKeyRowToRealizedRelationalResult(child, primaryKeyGetters, realizedRelationalResult);
}
} else {
child = mapObject;
}
} else {
childObjects.add(cachedObject);
child = cachedObject;
}
Object parent = parentMap.getWithSecondKey(childSqlResult);
if (parent == null) {
throw new RuntimeException("Cannot find the parent for child");
}
nodeSpecifics.addChildToParent(parent, child, DefaultExecutionNodeContext.factory().create(this.executionState, null));
}
relationalGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, childObjects);
childResult.close();
childResult = null;
/* Execute store local children */
if (!isLeaf) {
this.executeTempTableNodeChildren(node, realizedRelationalResult, databaseConnection, childSqlResult.getDatabaseType(), childSqlResult.getDatabaseTimeZone(), currentMap, primaryKeyGetters, this.executionState);
}
childInstancesToDeepFetchAndCache.stream().filter(x -> x.getTwo() != null).forEach(x -> {
Object object = x.getOne().getValue();
x.getTwo().put(new RelationalGraphFetchUtils.RelationalObjectGraphFetchCacheKey(object, primaryKeyGetters), object);
});
return new ConstantResult(childObjects);
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (childResult != null) {
childResult.close();
}
}
}
use of org.finos.legend.engine.plan.execution.stores.relational.result.SQLExecutionResult in project legend-engine by finos.
the class RelationalExecutor method execute.
public Result execute(SQLExecutionNode node, MutableList<CommonProfile> profiles, ExecutionState executionState) {
Connection connectionManagerConnection;
String databaseTimeZone = node.getDatabaseTimeZone() == null ? DEFAULT_DB_TIME_ZONE : node.getDatabaseTimeZone();
String databaseType = node.getDatabaseTypeName();
List<String> tempTableList = FastList.newList();
Span span = GlobalTracer.get().activeSpan();
connectionManagerConnection = getConnection(node, profiles, (RelationalStoreExecutionState) executionState.getStoreExecutionState(StoreType.Relational));
if (span != null) {
span.log("Connection acquired");
}
this.prepareForSQLExecution(node, connectionManagerConnection, databaseTimeZone, databaseType, tempTableList, profiles, executionState);
if (node.isResultVoid()) {
return new VoidRelationalResult(executionState.activities, connectionManagerConnection, profiles);
}
return new SQLExecutionResult(executionState.activities, node, databaseType, databaseTimeZone, connectionManagerConnection, profiles, tempTableList, executionState.topSpan);
}
Aggregations