use of org.finos.legend.engine.plan.dependencies.store.shared.IReferencedObject in project legend-engine by finos.
the class ResultNormalizer method normalizeToSql.
public static Object normalizeToSql(Object o, String databaseTimeZone) {
if (o == null) {
return "null";
}
if (o instanceof Map) {
return ((Map<?, ?>) o).entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> normalizeToSql(e.getValue(), databaseTimeZone)));
}
if (o instanceof Iterable) {
return StreamSupport.stream(((Iterable<?>) o).spliterator(), false).map(v -> normalizeToSql(v, databaseTimeZone)).collect(Collectors.toList());
}
if (o instanceof PureDate) {
PureDate pureDate = (PureDate) o;
if (pureDate.hasSubsecond()) {
return pureDate.format("[" + databaseTimeZone + "]yyyy-MM-dd HH:mm:ss.SSSSSS");
}
if (pureDate.hasSecond()) {
return pureDate.format("[" + databaseTimeZone + "]yyyy-MM-dd HH:mm:ss");
}
return pureDate.format("[" + databaseTimeZone + "]yyyy-MM-dd");
}
if (o instanceof EngineDate) {
return ((EngineDate) o).formatToSql();
}
if (o instanceof String) {
return ((String) o).replace("'", "\'").replace("\\", "\\\\");
}
if (o instanceof IReferencedObject) {
Field[] fields = o.getClass().getDeclaredFields();
Map<String, Object> fieldToNormalizedValueMap = new HashMap<>();
for (Field f : fields) {
try {
f.setAccessible(true);
fieldToNormalizedValueMap.put(f.getName(), normalizeToSql(f.get(o), databaseTimeZone));
} catch (IllegalAccessException e) {
throw new RuntimeException(e.getMessage());
}
}
return fieldToNormalizedValueMap;
}
return o.toString();
}
use of org.finos.legend.engine.plan.dependencies.store.shared.IReferencedObject in project legend-engine by finos.
the class RelationalExecutionNodeExecutor method executeRelationalCrossRootQueryTempTableGraphFetchExecutionNode.
private Result executeRelationalCrossRootQueryTempTableGraphFetchExecutionNode(RelationalCrossRootQueryTempTableGraphFetchExecutionNode node) {
boolean isLeaf = node.children == null || node.children.isEmpty();
List<Object> childObjects = new ArrayList<>();
Result childResult = null;
RelationalStoreExecutionState relationalStoreExecutionState = (RelationalStoreExecutionState) this.executionState.getStoreExecutionState(StoreType.Relational);
BlockConnectionContext oldBlockConnectionContext = relationalStoreExecutionState.getBlockConnectionContext();
boolean oldRetainConnectionFlag = relationalStoreExecutionState.retainConnection();
relationalStoreExecutionState.setBlockConnectionContext(new BlockConnectionContext());
relationalStoreExecutionState.setRetainConnection(true);
try (Scope ignored = GlobalTracer.get().buildSpan("cross store property graph fetch").withTag("storeType", "relational").withTag("property", ((PropertyGraphFetchTree) node.graphFetchTree).property).startActive(true)) {
IRelationalCrossRootQueryTempTableGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.profiles);
RelationalGraphObjectsBatch relationalGraphObjectsBatch = new RelationalGraphObjectsBatch(this.executionState.graphObjectsBatch);
List<?> parentObjects = relationalGraphObjectsBatch.getObjectsForNodeIndex(node.parentIndex);
if ((parentObjects != null) && !parentObjects.isEmpty()) {
GraphFetchTree nodeSubTree = node.graphFetchTree;
boolean cachingEnabled = false;
ExecutionCache<GraphFetchCacheKey, List<Object>> crossCache = relationalGraphObjectsBatch.getXStorePropertyCacheForNodeIndex(node.nodeIndex);
List<Method> parentCrossKeyGettersOrderedPerTargetProperties = null;
if (crossCache != null) {
cachingEnabled = true;
parentCrossKeyGettersOrderedPerTargetProperties = nodeSpecifics.parentCrossKeyGettersOrderedByTargetProperties();
}
List<Object> parentsToDeepFetch = new ArrayList<>();
for (Object parent : parentObjects) {
if (cachingEnabled) {
List<Object> children = crossCache.getIfPresent(new RelationalGraphFetchUtils.RelationalCrossObjectGraphFetchCacheKey(parent, parentCrossKeyGettersOrderedPerTargetProperties));
if (children == null) {
parentsToDeepFetch.add(parent);
} else {
for (Object child : children) {
childObjects.add(child);
nodeSpecifics.addChildToParent(parent, child, DefaultExecutionNodeContext.factory().create(this.executionState, null));
}
}
} else {
parentsToDeepFetch.add(parent);
}
}
if (!parentsToDeepFetch.isEmpty()) {
Map<Object, List<Object>> parentToChildMap = new HashMap<>();
RealizedRelationalResult parentRealizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.parentTempTableColumns);
List<Method> crossKeyGetters = nodeSpecifics.parentCrossKeyGetters();
for (Object parentObject : parentsToDeepFetch) {
this.addKeyRowToRealizedRelationalResult(parentObject, crossKeyGetters, parentRealizedRelationalResult);
parentToChildMap.put(parentObject, new ArrayList<>());
}
this.executionState.addResult(node.parentTempTableName, parentRealizedRelationalResult);
childResult = node.executionNodes.get(0).accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
SQLExecutionResult childSqlResult = (SQLExecutionResult) childResult;
DatabaseConnection databaseConnection = childSqlResult.getSQLExecutionNode().connection;
ResultSet childResultSet = childSqlResult.getResultSet();
List<String> parentSQLKeyColumns = nodeSpecifics.parentCrossKeyColumns(childSqlResult.getResultColumns().stream().map(ResultColumn::getNonQuotedLabel).collect(Collectors.toList()));
List<Integer> parentCrossKeyIndices = parentSQLKeyColumns.stream().map(FunctionHelper.unchecked(childResultSet::findColumn)).collect(Collectors.toList());
@SuppressWarnings("MismatchedQueryAndUpdateOfCollection") DoubleStrategyHashMap<Object, List<Object>, SQLExecutionResult> parentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategy(crossKeyGetters, parentCrossKeyIndices));
parentsToDeepFetch.forEach((o) -> parentMap.getIfAbsentPut(o, ArrayList::new).add(o));
RealizedRelationalResult realizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.columns);
List<Method> primaryKeyGetters = nodeSpecifics.primaryKeyGetters();
DoubleStrategyHashMap<Object, Object, SQLExecutionResult> currentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
/* Prepare for reading */
nodeSpecifics.prepare(childResultSet, childSqlResult.getDatabaseTimeZone(), ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(databaseConnection));
while (childResultSet.next()) {
relationalGraphObjectsBatch.incrementRowCount();
List<Object> parents = parentMap.getWithSecondKey(childSqlResult);
if (parents == null) {
throw new RuntimeException("Cannot find the parent for child");
}
IGraphInstance<? extends IReferencedObject> childGraphInstance = nodeSpecifics.nextGraphInstance();
Object child = childGraphInstance.getValue();
Object mapObject = currentMap.putIfAbsent(child, child);
if (mapObject == null) {
mapObject = child;
childObjects.add(mapObject);
relationalGraphObjectsBatch.addObjectMemoryUtilization(childGraphInstance.instanceSize());
if (!isLeaf) {
this.addKeyRowToRealizedRelationalResult(child, primaryKeyGetters, realizedRelationalResult);
}
}
for (Object parent : parents) {
if (parentToChildMap.containsKey(parent)) {
parentToChildMap.get(parent).add(mapObject);
} else {
parentToChildMap.put(parent, new ArrayList<>());
parentToChildMap.get(parent).add(mapObject);
}
nodeSpecifics.addChildToParent(parent, mapObject, DefaultExecutionNodeContext.factory().create(this.executionState, null));
}
}
relationalGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, childObjects);
childResult.close();
childResult = null;
/* Execute store local children */
if (!isLeaf) {
ExecutionState newState = new ExecutionState(executionState);
newState.graphObjectsBatch = relationalGraphObjectsBatch;
this.executeTempTableNodeChildren(node, realizedRelationalResult, databaseConnection, childSqlResult.getDatabaseType(), childSqlResult.getDatabaseTimeZone(), currentMap, primaryKeyGetters, newState);
}
if (cachingEnabled) {
List<Method> getters = parentCrossKeyGettersOrderedPerTargetProperties;
parentToChildMap.forEach((p, cs) -> {
crossCache.put(new RelationalGraphFetchUtils.RelationalCrossObjectGraphFetchCacheKey(p, getters), cs);
});
}
}
}
return new ConstantResult(childObjects);
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (childResult != null) {
childResult.close();
}
relationalStoreExecutionState.getBlockConnectionContext().unlockAllBlockConnections();
relationalStoreExecutionState.getBlockConnectionContext().closeAllBlockConnectionsAsync();
relationalStoreExecutionState.setBlockConnectionContext(oldBlockConnectionContext);
relationalStoreExecutionState.setRetainConnection(oldRetainConnectionFlag);
}
}
use of org.finos.legend.engine.plan.dependencies.store.shared.IReferencedObject in project legend-engine by finos.
the class RelationalExecutionNodeExecutor method executeRelationalClassQueryTempTableGraphFetchExecutionNode.
private Result executeRelationalClassQueryTempTableGraphFetchExecutionNode(RelationalClassQueryTempTableGraphFetchExecutionNode node) {
boolean isLeaf = node.children == null || node.children.isEmpty();
List<Object> childObjects = new ArrayList<>();
List<Pair<IGraphInstance<? extends IReferencedObject>, ExecutionCache<GraphFetchCacheKey, Object>>> childInstancesToDeepFetchAndCache = new ArrayList<>();
Result childResult = null;
try (Scope ignored = GlobalTracer.get().buildSpan("local store property graph fetch").withTag("storeType", "relational").withTag("property", ((PropertyGraphFetchTree) node.graphFetchTree).property).startActive(true)) {
RelationalGraphObjectsBatch relationalGraphObjectsBatch = (RelationalGraphObjectsBatch) this.executionState.graphObjectsBatch;
childResult = node.executionNodes.get(0).accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
SQLExecutionResult childSqlResult = (SQLExecutionResult) childResult;
DatabaseConnection databaseConnection = childSqlResult.getSQLExecutionNode().connection;
ResultSet childResultSet = childSqlResult.getResultSet();
IRelationalClassQueryTempTableGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.profiles);
/* Check if caching is enabled and fetch caches if required */
List<Pair<String, String>> allInstanceSetImplementations = nodeSpecifics.allInstanceSetImplementations();
int setIdCount = allInstanceSetImplementations.size();
boolean isUnion = setIdCount > 1;
RelationalMultiSetExecutionCacheWrapper multiSetCache = new RelationalMultiSetExecutionCacheWrapper(setIdCount);
boolean cachingEnabledForNode = this.checkForCachingAndPopulateCachingHelpers(allInstanceSetImplementations, nodeSpecifics.supportsCaching(), node.graphFetchTree, childSqlResult, nodeSpecifics::primaryKeyColumns, multiSetCache);
DoubleStrategyHashMap<Object, Object, SQLExecutionResult> parentMap = this.switchedParentHashMapPerChildResult(relationalGraphObjectsBatch, node.parentIndex, childResultSet, () -> nodeSpecifics.parentPrimaryKeyColumns(childSqlResult.getResultColumns().stream().map(ResultColumn::getNonQuotedLabel).collect(Collectors.toList())));
List<Method> primaryKeyGetters = nodeSpecifics.primaryKeyGetters();
DoubleStrategyHashMap<Object, Object, SQLExecutionResult> currentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
RealizedRelationalResult realizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.columns);
/* Prepare for reading */
nodeSpecifics.prepare(childResultSet, childSqlResult.getDatabaseTimeZone(), ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(databaseConnection));
while (childResultSet.next()) {
relationalGraphObjectsBatch.incrementRowCount();
Object child;
int setIndex = isUnion ? childResultSet.getInt(1) : 0;
Object cachedObject = RelationalExecutionNodeExecutor.this.checkAndReturnCachedObject(cachingEnabledForNode, setIndex, multiSetCache);
boolean shouldDeepFetchOnThisInstance = cachedObject == null;
if (shouldDeepFetchOnThisInstance) {
IGraphInstance<? extends IReferencedObject> wrappedObject = nodeSpecifics.nextGraphInstance();
Object wrappedValue = wrappedObject.getValue();
Object mapObject = currentMap.putIfAbsent(wrappedValue, wrappedValue);
if (mapObject == null) {
child = wrappedValue;
childInstancesToDeepFetchAndCache.add(Tuples.pair(wrappedObject, multiSetCache.setCaches.get(setIndex)));
relationalGraphObjectsBatch.addObjectMemoryUtilization(wrappedObject.instanceSize());
childObjects.add(child);
if (!isLeaf) {
this.addKeyRowToRealizedRelationalResult(child, primaryKeyGetters, realizedRelationalResult);
}
} else {
child = mapObject;
}
} else {
childObjects.add(cachedObject);
child = cachedObject;
}
Object parent = parentMap.getWithSecondKey(childSqlResult);
if (parent == null) {
throw new RuntimeException("Cannot find the parent for child");
}
nodeSpecifics.addChildToParent(parent, child, DefaultExecutionNodeContext.factory().create(this.executionState, null));
}
relationalGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, childObjects);
childResult.close();
childResult = null;
/* Execute store local children */
if (!isLeaf) {
this.executeTempTableNodeChildren(node, realizedRelationalResult, databaseConnection, childSqlResult.getDatabaseType(), childSqlResult.getDatabaseTimeZone(), currentMap, primaryKeyGetters, this.executionState);
}
childInstancesToDeepFetchAndCache.stream().filter(x -> x.getTwo() != null).forEach(x -> {
Object object = x.getOne().getValue();
x.getTwo().put(new RelationalGraphFetchUtils.RelationalObjectGraphFetchCacheKey(object, primaryKeyGetters), object);
});
return new ConstantResult(childObjects);
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (childResult != null) {
childResult.close();
}
}
}
use of org.finos.legend.engine.plan.dependencies.store.shared.IReferencedObject in project legend-engine by finos.
the class RelationalExecutionNodeExecutor method executeRelationalRootQueryTempTableGraphFetchExecutionNode.
private Result executeRelationalRootQueryTempTableGraphFetchExecutionNode(RelationalRootQueryTempTableGraphFetchExecutionNode node) {
int batchSize = node.batchSize == null ? 1000 : node.batchSize;
boolean isLeaf = node.children == null || node.children.isEmpty();
Result rootResult = null;
Span graphFetchSpan = GlobalTracer.get().buildSpan("graph fetch").withTag("rootStoreType", "relational").withTag("batchSizeConfig", batchSize).start();
GlobalTracer.get().activateSpan(graphFetchSpan);
try {
rootResult = node.executionNodes.get(0).accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
SQLExecutionResult sqlExecutionResult = (SQLExecutionResult) rootResult;
DatabaseConnection databaseConnection = sqlExecutionResult.getSQLExecutionNode().connection;
ResultSet rootResultSet = ((SQLExecutionResult) rootResult).getResultSet();
IRelationalRootQueryTempTableGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.profiles);
List<Method> primaryKeyGetters = nodeSpecifics.primaryKeyGetters();
/* Check if caching is enabled and fetch caches if required */
List<Pair<String, String>> allInstanceSetImplementations = nodeSpecifics.allInstanceSetImplementations();
int setIdCount = allInstanceSetImplementations.size();
RelationalMultiSetExecutionCacheWrapper multiSetCache = new RelationalMultiSetExecutionCacheWrapper(setIdCount);
boolean cachingEnabledForNode = this.checkForCachingAndPopulateCachingHelpers(allInstanceSetImplementations, nodeSpecifics.supportsCaching(), node.graphFetchTree, sqlExecutionResult, nodeSpecifics::primaryKeyColumns, multiSetCache);
/* Prepare for reading */
nodeSpecifics.prepare(rootResultSet, sqlExecutionResult.getDatabaseTimeZone(), ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(databaseConnection));
boolean isUnion = setIdCount > 1;
AtomicLong batchIndex = new AtomicLong(0L);
Spliterator<GraphObjectsBatch> graphObjectsBatchSpliterator = new Spliterators.AbstractSpliterator<GraphObjectsBatch>(Long.MAX_VALUE, Spliterator.ORDERED) {
@Override
public boolean tryAdvance(Consumer<? super GraphObjectsBatch> action) {
/* Ensure all children run in the same connection */
RelationalStoreExecutionState relationalStoreExecutionState = (RelationalStoreExecutionState) executionState.getStoreExecutionState(StoreType.Relational);
BlockConnectionContext oldBlockConnectionContext = relationalStoreExecutionState.getBlockConnectionContext();
boolean oldRetainConnectionFlag = relationalStoreExecutionState.retainConnection();
relationalStoreExecutionState.setBlockConnectionContext(new BlockConnectionContext());
relationalStoreExecutionState.setRetainConnection(true);
long currentBatch = batchIndex.incrementAndGet();
try (Scope ignored = GlobalTracer.get().buildSpan("graph fetch batch").withTag("storeType", "relational").withTag("batchIndex", currentBatch).withTag("class", ((RootGraphFetchTree) node.graphFetchTree)._class).asChildOf(graphFetchSpan).startActive(true)) {
RelationalGraphObjectsBatch relationalGraphObjectsBatch = new RelationalGraphObjectsBatch(currentBatch);
List<Object> resultObjects = new ArrayList<>();
List<Pair<IGraphInstance<? extends IReferencedObject>, ExecutionCache<GraphFetchCacheKey, Object>>> instancesToDeepFetchAndCache = new ArrayList<>();
int objectCount = 0;
while ((!rootResultSet.isClosed()) && rootResultSet.next()) {
relationalGraphObjectsBatch.incrementRowCount();
int setIndex = isUnion ? rootResultSet.getInt(1) : 0;
Object cachedObject = RelationalExecutionNodeExecutor.this.checkAndReturnCachedObject(cachingEnabledForNode, setIndex, multiSetCache);
boolean shouldDeepFetchOnThisInstance = cachedObject == null;
Object object;
if (shouldDeepFetchOnThisInstance) {
IGraphInstance<? extends IReferencedObject> wrappedObject = nodeSpecifics.nextGraphInstance();
instancesToDeepFetchAndCache.add(Tuples.pair(wrappedObject, multiSetCache.setCaches.get(setIndex)));
object = wrappedObject.getValue();
} else {
object = cachedObject;
}
if (node.checked != null && node.checked) {
resultObjects.add(BasicChecked.newChecked(object, null));
} else {
resultObjects.add(object);
}
objectCount += 1;
if (objectCount >= batchSize) {
break;
}
}
relationalGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, resultObjects);
if (!instancesToDeepFetchAndCache.isEmpty()) {
RealizedRelationalResult realizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.columns);
DoubleStrategyHashMap<Object, Object, SQLExecutionResult> rootMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
for (Pair<IGraphInstance<? extends IReferencedObject>, ExecutionCache<GraphFetchCacheKey, Object>> instanceAndCache : instancesToDeepFetchAndCache) {
IGraphInstance<? extends IReferencedObject> rootGraphInstance = instanceAndCache.getOne();
Object rootObject = rootGraphInstance.getValue();
rootMap.put(rootObject, rootObject);
relationalGraphObjectsBatch.addObjectMemoryUtilization(rootGraphInstance.instanceSize());
if (!isLeaf) {
RelationalExecutionNodeExecutor.this.addKeyRowToRealizedRelationalResult(rootObject, primaryKeyGetters, realizedRelationalResult);
}
}
/* Execute store local children */
if (!isLeaf) {
ExecutionState newState = new ExecutionState(executionState);
newState.graphObjectsBatch = relationalGraphObjectsBatch;
RelationalExecutionNodeExecutor.this.executeTempTableNodeChildren(node, realizedRelationalResult, databaseConnection, sqlExecutionResult.getDatabaseType(), sqlExecutionResult.getDatabaseTimeZone(), rootMap, primaryKeyGetters, newState);
}
}
instancesToDeepFetchAndCache.stream().filter(x -> x.getTwo() != null).forEach(x -> {
Object object = x.getOne().getValue();
x.getTwo().put(new RelationalGraphFetchUtils.RelationalObjectGraphFetchCacheKey(object, primaryKeyGetters), object);
});
action.accept(relationalGraphObjectsBatch);
return !resultObjects.isEmpty();
} catch (SQLException | InvocationTargetException | IllegalAccessException e) {
throw new RuntimeException(e);
} finally {
relationalStoreExecutionState.getBlockConnectionContext().unlockAllBlockConnections();
relationalStoreExecutionState.getBlockConnectionContext().closeAllBlockConnectionsAsync();
relationalStoreExecutionState.setBlockConnectionContext(oldBlockConnectionContext);
relationalStoreExecutionState.setRetainConnection(oldRetainConnectionFlag);
}
}
};
Stream<GraphObjectsBatch> graphObjectsBatchStream = StreamSupport.stream(graphObjectsBatchSpliterator, false);
return new GraphFetchResult(graphObjectsBatchStream, rootResult).withGraphFetchSpan(graphFetchSpan);
} catch (RuntimeException e) {
if (rootResult != null) {
rootResult.close();
}
if (graphFetchSpan != null) {
graphFetchSpan.finish();
}
throw e;
} catch (Exception e) {
if (rootResult != null) {
rootResult.close();
}
if (graphFetchSpan != null) {
graphFetchSpan.finish();
}
throw new RuntimeException(e);
}
}
Aggregations