Examples with IGraphInstance org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance used on opensource projects

Example 1 with IGraphInstance

use of org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance in project legend-engine by finos.

the class RelationalExecutionNodeExecutor method executeRelationalClassQueryTempTableGraphFetchExecutionNode.

private Result executeRelationalClassQueryTempTableGraphFetchExecutionNode(RelationalClassQueryTempTableGraphFetchExecutionNode node) {
    boolean isLeaf = node.children == null || node.children.isEmpty();
    List<Object> childObjects = new ArrayList<>();
    List<Pair<IGraphInstance<? extends IReferencedObject>, ExecutionCache<GraphFetchCacheKey, Object>>> childInstancesToDeepFetchAndCache = new ArrayList<>();
    Result childResult = null;
    try (Scope ignored = GlobalTracer.get().buildSpan("local store property graph fetch").withTag("storeType", "relational").withTag("property", ((PropertyGraphFetchTree) node.graphFetchTree).property).startActive(true)) {
        RelationalGraphObjectsBatch relationalGraphObjectsBatch = (RelationalGraphObjectsBatch) this.executionState.graphObjectsBatch;
        childResult = node.executionNodes.get(0).accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
        SQLExecutionResult childSqlResult = (SQLExecutionResult) childResult;
        DatabaseConnection databaseConnection = childSqlResult.getSQLExecutionNode().connection;
        ResultSet childResultSet = childSqlResult.getResultSet();
        IRelationalClassQueryTempTableGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.profiles);
        /* Check if caching is enabled and fetch caches if required */
        List<Pair<String, String>> allInstanceSetImplementations = nodeSpecifics.allInstanceSetImplementations();
        int setIdCount = allInstanceSetImplementations.size();
        boolean isUnion = setIdCount > 1;
        RelationalMultiSetExecutionCacheWrapper multiSetCache = new RelationalMultiSetExecutionCacheWrapper(setIdCount);
        boolean cachingEnabledForNode = this.checkForCachingAndPopulateCachingHelpers(allInstanceSetImplementations, nodeSpecifics.supportsCaching(), node.graphFetchTree, childSqlResult, nodeSpecifics::primaryKeyColumns, multiSetCache);
        DoubleStrategyHashMap<Object, Object, SQLExecutionResult> parentMap = this.switchedParentHashMapPerChildResult(relationalGraphObjectsBatch, node.parentIndex, childResultSet, () -> nodeSpecifics.parentPrimaryKeyColumns(childSqlResult.getResultColumns().stream().map(ResultColumn::getNonQuotedLabel).collect(Collectors.toList())));
        List<Method> primaryKeyGetters = nodeSpecifics.primaryKeyGetters();
        DoubleStrategyHashMap<Object, Object, SQLExecutionResult> currentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
        RealizedRelationalResult realizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.columns);
        /* Prepare for reading */
        nodeSpecifics.prepare(childResultSet, childSqlResult.getDatabaseTimeZone(), ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(databaseConnection));
        while (childResultSet.next()) {
            relationalGraphObjectsBatch.incrementRowCount();
            Object child;
            int setIndex = isUnion ? childResultSet.getInt(1) : 0;
            Object cachedObject = RelationalExecutionNodeExecutor.this.checkAndReturnCachedObject(cachingEnabledForNode, setIndex, multiSetCache);
            boolean shouldDeepFetchOnThisInstance = cachedObject == null;
            if (shouldDeepFetchOnThisInstance) {
                IGraphInstance<? extends IReferencedObject> wrappedObject = nodeSpecifics.nextGraphInstance();
                Object wrappedValue = wrappedObject.getValue();
                Object mapObject = currentMap.putIfAbsent(wrappedValue, wrappedValue);
                if (mapObject == null) {
                    child = wrappedValue;
                    childInstancesToDeepFetchAndCache.add(Tuples.pair(wrappedObject, multiSetCache.setCaches.get(setIndex)));
                    relationalGraphObjectsBatch.addObjectMemoryUtilization(wrappedObject.instanceSize());
                    childObjects.add(child);
                    if (!isLeaf) {
                        this.addKeyRowToRealizedRelationalResult(child, primaryKeyGetters, realizedRelationalResult);
                    }
                } else {
                    child = mapObject;
                }
            } else {
                childObjects.add(cachedObject);
                child = cachedObject;
            }
            Object parent = parentMap.getWithSecondKey(childSqlResult);
            if (parent == null) {
                throw new RuntimeException("Cannot find the parent for child");
            }
            nodeSpecifics.addChildToParent(parent, child, DefaultExecutionNodeContext.factory().create(this.executionState, null));
        }
        relationalGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, childObjects);
        childResult.close();
        childResult = null;
        /* Execute store local children */
        if (!isLeaf) {
            this.executeTempTableNodeChildren(node, realizedRelationalResult, databaseConnection, childSqlResult.getDatabaseType(), childSqlResult.getDatabaseTimeZone(), currentMap, primaryKeyGetters, this.executionState);
        }
        childInstancesToDeepFetchAndCache.stream().filter(x -> x.getTwo() != null).forEach(x -> {
            Object object = x.getOne().getValue();
            x.getTwo().put(new RelationalGraphFetchUtils.RelationalObjectGraphFetchCacheKey(object, primaryKeyGetters), object);
        });
        return new ConstantResult(childObjects);
    } catch (RuntimeException e) {
        throw e;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        if (childResult != null) {
            childResult.close();
        }
    }
}

Example 2 with IGraphInstance

use of org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance in project legend-engine by finos.

the class InMemoryExecutionNodeExecutor method visit.

@Override
public Result visit(InMemoryPropertyGraphFetchExecutionNode node) {
    boolean isLeaf = node.children == null || node.children.isEmpty();
    IInMemoryPropertyGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.pm);
    GraphObjectsBatch graphObjectsBatch = this.executionState.graphObjectsBatch;
    List<?> parentObjects = graphObjectsBatch.getObjectsForNodeIndex(node.parentIndex).stream().map(x -> x instanceof IChecked ? ((IChecked<?>) x).getValue() : x).collect(Collectors.toList());
    Stream<IGraphInstance> childGraphInstancesStream = nodeSpecifics.transformProperty(parentObjects);
    List<Object> childObjects = childGraphInstancesStream.filter(Objects::nonNull).map(graphInstance -> {
        graphObjectsBatch.addObjectMemoryUtilization(graphInstance.instanceSize());
        return graphInstance.getValue();
    }).collect(Collectors.toList());
    graphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, childObjects);
    if (!childObjects.isEmpty() && (!isLeaf)) {
        node.children.forEach(x -> x.accept(new ExecutionNodeExecutor(this.pm, executionState)));
    }
    return new ConstantResult(childObjects);
}

Example 3 with IGraphInstance

use of org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance in project legend-engine by finos.

the class InMemoryExecutionNodeExecutor method mergeInMemoryNode.

private Result mergeInMemoryNode(InMemoryRootGraphFetchExecutionNode node) {
    IInMemoryRootGraphFetchMergeExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.pm);
    List<GraphFetchResult> results = node.executionNodes.stream().map(n -> (GraphFetchResult) n.accept(new ExecutionNodeExecutor(this.pm, this.executionState))).collect(Collectors.toList());
    List<Object> subObjects = results.stream().map(g -> g.getGraphObjectsBatchStream().findFirst().get().getObjectsForNodeIndex(0).get(0)).collect(Collectors.toList());
    // merged object
    Object targetObject = nodeSpecifics.transform(subObjects);
    Spliterator<GraphObjectsBatch> graphObjectsBatchSpliterator = new Spliterators.AbstractSpliterator<GraphObjectsBatch>(Long.MAX_VALUE, Spliterator.ORDERED) {

        AtomicLong batchIndex = new AtomicLong(0L);

        @Override
        public boolean tryAdvance(Consumer<? super GraphObjectsBatch> action) {
            long currentBatch = batchIndex.incrementAndGet();
            if (// run only once
            currentBatch > 1) {
                return false;
            }
            List<Object> resultObjects = new ArrayList<>();
            GraphObjectsBatch inMemoryGraphObjectsBatch = new GraphObjectsBatch(currentBatch, executionState.getGraphFetchBatchMemoryLimit());
            IGraphInstance<?> target = (IGraphInstance<?>) targetObject;
            inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
            resultObjects.add(target.getValue());
            inMemoryGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, resultObjects);
            action.accept(inMemoryGraphObjectsBatch);
            return false;
        }
    };
    Stream<GraphObjectsBatch> graphObjectsBatchStream = StreamSupport.stream(graphObjectsBatchSpliterator, false);
    return new GraphFetchResult(graphObjectsBatchStream, new ConstantResult(targetObject));
}

Example 4 with IGraphInstance

use of org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance in project legend-engine by finos.

the class InMemoryExecutionNodeExecutor method visit.

@Override
public Result visit(InMemoryRootGraphFetchExecutionNode node) {
    int batchSize = node.batchSize == null ? 1 : node.batchSize;
    boolean isLeaf = node.children == null || node.children.isEmpty();
    boolean checked = node.checked;
    ClassResultType classResultType = (ClassResultType) node.resultType;
    String _class = classResultType._class;
    Result childResult = null;
    JavaPlatformImplementation javaPlatformImpl = (JavaPlatformImplementation) node.implementation;
    String executionClassName = JavaHelper.getExecutionClassFullName(javaPlatformImpl);
    Class<?> clazz = ExecutionNodeJavaPlatformHelper.getClassToExecute(node, executionClassName, this.executionState, this.pm);
    Span graphFetchSpan = GlobalTracer.get().buildSpan("graph fetch").withTag("rootStoreType", "inMemory").withTag("batchSizeConfig", batchSize).start();
    GlobalTracer.get().activateSpan(graphFetchSpan);
    try {
        if ((Arrays.asList(clazz.getInterfaces()).contains(IInMemoryRootGraphFetchMergeExecutionNodeSpecifics.class))) {
            return mergeInMemoryNode(node);
        } else {
            IInMemoryRootGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.pm);
            childResult = node.executionNodes.get(0).accept(new ExecutionNodeExecutor(this.pm, this.executionState));
            Iterator<?> sourceObjectsIterator;
            if (childResult instanceof StoreStreamReadingResult) {
                StoreStreamReadingResult<?> storeStreamReadingResult = (StoreStreamReadingResult) childResult;
                sourceObjectsIterator = storeStreamReadingResult.getObjectsIterator();
            } else if (childResult instanceof StreamingObjectResult) {
                StreamingObjectResult<?> streamingObjectResult = (StreamingObjectResult) childResult;
                sourceObjectsIterator = streamingObjectResult.getObjectStream().iterator();
            } else {
                throw new IllegalStateException("Unsupported result type: " + childResult.getClass().getSimpleName());
            }
            AtomicLong batchIndex = new AtomicLong(0L);
            Spliterator<GraphObjectsBatch> graphObjectsBatchSpliterator = new Spliterators.AbstractSpliterator<GraphObjectsBatch>(Long.MAX_VALUE, Spliterator.ORDERED) {

                @Override
                public boolean tryAdvance(Consumer<? super GraphObjectsBatch> action) {
                    long currentBatch = batchIndex.incrementAndGet();
                    GraphObjectsBatch inMemoryGraphObjectsBatch = new GraphObjectsBatch(currentBatch, executionState.getGraphFetchBatchMemoryLimit());
                    List<Object> resultObjects = new ArrayList<>();
                    int objectCount = 0;
                    if (checked) {
                        while (sourceObjectsIterator.hasNext()) {
                            IChecked<?> checkedSource = (IChecked<?>) sourceObjectsIterator.next();
                            Object value = checkedSource.getValue();
                            if (value == null) {
                                resultObjects.add(newDynamicChecked(Collections.singletonList(BasicDefect.newNoInputDefect(_class)), checkedSource, null));
                            } else {
                                Object targetObject = nodeSpecifics.transform(value);
                                if (targetObject != null) {
                                    if (targetObject instanceof List) {
                                        ((List<?>) targetObject).forEach(x -> {
                                            IGraphInstance<?> target = (IGraphInstance<?>) x;
                                            inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
                                            resultObjects.add(newDynamicChecked(Collections.emptyList(), checkedSource, target.getValue()));
                                        });
                                    } else {
                                        IGraphInstance<?> target = (IGraphInstance<?>) targetObject;
                                        inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
                                        resultObjects.add(newDynamicChecked(Collections.emptyList(), checkedSource, target.getValue()));
                                    }
                                }
                            }
                            objectCount += 1;
                            if (objectCount >= batchSize)
                                break;
                        }
                    } else {
                        while (sourceObjectsIterator.hasNext()) {
                            Object targetObject = nodeSpecifics.transform(sourceObjectsIterator.next());
                            if (targetObject != null) {
                                if (targetObject instanceof List) {
                                    ((List<?>) targetObject).forEach(x -> {
                                        IGraphInstance<?> target = (IGraphInstance<?>) x;
                                        inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
                                        resultObjects.add(target.getValue());
                                    });
                                } else {
                                    IGraphInstance<?> target = (IGraphInstance<?>) targetObject;
                                    inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
                                    resultObjects.add(target.getValue());
                                }
                            }
                            objectCount += 1;
                            if (objectCount >= batchSize)
                                break;
                        }
                    }
                    inMemoryGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, resultObjects);
                    if (!resultObjects.isEmpty() && (!isLeaf)) {
                        ExecutionState newState = new ExecutionState(executionState);
                        newState.graphObjectsBatch = inMemoryGraphObjectsBatch;
                        node.children.forEach(x -> x.accept(new ExecutionNodeExecutor(InMemoryExecutionNodeExecutor.this.pm, newState)));
                    }
                    action.accept(inMemoryGraphObjectsBatch);
                    return objectCount != 0;
                }
            };
            Stream<GraphObjectsBatch> graphObjectsBatchStream = StreamSupport.stream(graphObjectsBatchSpliterator, false);
            return new GraphFetchResult(graphObjectsBatchStream, childResult).withGraphFetchSpan(graphFetchSpan);
        }
    } catch (Exception e) {
        if (childResult != null) {
            childResult.close();
        }
        if (graphFetchSpan != null) {
            graphFetchSpan.finish();
        }
        if (e instanceof RuntimeException) {
            throw e;
        }
        Throwable cause = e.getCause();
        if (cause instanceof RuntimeException)
            throw (RuntimeException) cause;
        if (cause instanceof Error)
            throw (Error) cause;
        throw new RuntimeException(cause);
    }
}

Example 5 with IGraphInstance

use of org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance in project legend-engine by finos.

the class InMemoryExecutionNodeExecutor method visit.

@Override
public Result visit(InMemoryCrossStoreGraphFetchExecutionNode node) {
    List<Object> childObjects = new ArrayList<>();
    Result childResult = null;
    try {
        IInMemoryCrossStoreGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.pm);
        GraphObjectsBatch graphObjectsBatch = new GraphObjectsBatch(this.executionState.graphObjectsBatch);
        List<?> parentObjects = graphObjectsBatch.getObjectsForNodeIndex(node.parentIndex);
        if ((parentObjects != null) && !parentObjects.isEmpty()) {
            DoubleStrategyHashMap<Object, List<Object>, Object> parentMap = new DoubleStrategyHashMap<>(InMemoryGraphFetchUtils.parentChildDoubleHashStrategy(nodeSpecifics));
            parentObjects.forEach(parentObject -> parentMap.getIfAbsentPut(parentObject, ArrayList::new).add(parentObject));
            if (node.supportsBatching) {
                Map<String, List<Object>> keyValuePairs = Maps.mutable.empty();
                nodeSpecifics.getCrossStoreKeysValueForChildren(parentObjects.get(0)).keySet().forEach(key -> keyValuePairs.put(key, Lists.mutable.empty()));
                parentMap.keySet().forEach(parentObject -> nodeSpecifics.getCrossStoreKeysValueForChildren(parentObject).forEach((key, value) -> keyValuePairs.get(key).add(value)));
                keyValuePairs.forEach((key, value) -> this.executionState.addResult(key, new ConstantResult(value)));
                childResult = this.visit((InMemoryRootGraphFetchExecutionNode) node);
                GraphFetchResult childGraphFetchResult = (GraphFetchResult) childResult;
                Stream<GraphObjectsBatch> graphObjectsBatchStream = childGraphFetchResult.getGraphObjectsBatchStream();
                graphObjectsBatchStream.forEach(batch -> {
                    batch.getObjectsForNodeIndex(node.nodeIndex).forEach(child -> {
                        IGraphInstance<?> childGraphInstance = nodeSpecifics.wrapChildInGraphInstance(child);
                        Object childObject = childGraphInstance.getValue();
                        List<Object> parentsInScope = parentMap.getWithSecondKey(childObject);
                        if (parentsInScope != null) {
                            for (Object parentObject : parentsInScope) {
                                boolean isChildAdded = nodeSpecifics.attemptAddingChildToParent(parentObject, childObject);
                                if (isChildAdded) {
                                    graphObjectsBatch.addObjectMemoryUtilization(childGraphInstance.instanceSize());
                                    childObjects.add(childObject);
                                }
                            }
                        }
                    });
                });
            } else {
                for (Map.Entry<Object, List<Object>> entry : parentMap.entrySet()) {
                    Map<String, Object> keyValuePairs = nodeSpecifics.getCrossStoreKeysValueForChildren(entry.getKey());
                    keyValuePairs.forEach((key, value) -> this.executionState.addResult(key, new ConstantResult(value)));
                    childResult = this.visit((InMemoryRootGraphFetchExecutionNode) node);
                    GraphFetchResult childGraphFetchResult = (GraphFetchResult) childResult;
                    Stream<GraphObjectsBatch> graphObjectsBatchStream = childGraphFetchResult.getGraphObjectsBatchStream();
                    graphObjectsBatchStream.forEach(batch -> {
                        batch.getObjectsForNodeIndex(node.nodeIndex).forEach(child -> {
                            IGraphInstance<?> childGraphInstance = nodeSpecifics.wrapChildInGraphInstance(child);
                            Object childObject = childGraphInstance.getValue();
                            List<Object> parentsInScope = entry.getValue();
                            if (parentsInScope != null) {
                                for (Object parentObject : parentsInScope) {
                                    boolean isChildAdded = nodeSpecifics.attemptAddingChildToParent(parentObject, childObject);
                                    if (isChildAdded) {
                                        graphObjectsBatch.addObjectMemoryUtilization(childGraphInstance.instanceSize());
                                        childObjects.add(childObject);
                                    }
                                }
                            }
                        });
                    });
                }
            }
            graphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, childObjects);
        }
        return new ConstantResult(childObjects);
    } catch (RuntimeException e) {
        throw e;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        if (childResult != null) {
            childResult.close();
        }
    }
}