Examples with GraphObjectsBatch org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch used on opensource projects

Example 1 with GraphObjectsBatch

use of org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch in project legend-engine by finos.

the class ExecutionNodeExecutor method visit.

@Override
public Result visit(GlobalGraphFetchExecutionNode globalGraphFetchExecutionNode) {
    final Span topSpan = GlobalTracer.get().activeSpan();
    final boolean isGraphRoot = globalGraphFetchExecutionNode.parentIndex == null;
    if (isGraphRoot) {
        final boolean enableConstraints = globalGraphFetchExecutionNode.enableConstraints == null ? false : globalGraphFetchExecutionNode.enableConstraints;
        final boolean checked = globalGraphFetchExecutionNode.checked == null ? false : globalGraphFetchExecutionNode.checked;
        // Handle batching at root level
        final AtomicLong rowCount = new AtomicLong(0L);
        final AtomicLong objectCount = new AtomicLong(0L);
        final DoubleSummaryStatistics memoryStatistics = new DoubleSummaryStatistics();
        GraphFetchResult graphFetchResult = (GraphFetchResult) globalGraphFetchExecutionNode.localGraphFetchExecutionNode.accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
        Stream<?> objectStream = graphFetchResult.getGraphObjectsBatchStream().map(batch -> {
            List<?> parentObjects = batch.getObjectsForNodeIndex(0);
            boolean nonEmptyObjectList = !parentObjects.isEmpty();
            ExecutionState newState = new ExecutionState(this.executionState).setGraphObjectsBatch(batch);
            if (globalGraphFetchExecutionNode.children != null && !globalGraphFetchExecutionNode.children.isEmpty() && nonEmptyObjectList) {
                globalGraphFetchExecutionNode.children.forEach(c -> c.accept(new ExecutionNodeExecutor(this.profiles, newState)));
            }
            rowCount.addAndGet(batch.getRowCount());
            if (nonEmptyObjectList) {
                long currentObjectCount = objectCount.addAndGet(parentObjects.size());
                memoryStatistics.accept(batch.getTotalObjectMemoryUtilization() / (parentObjects.size() * 1.0));
                if (graphFetchResult.getGraphFetchSpan() != null) {
                    Span graphFetchSpan = graphFetchResult.getGraphFetchSpan();
                    graphFetchSpan.setTag("batchCount", memoryStatistics.getCount());
                    graphFetchSpan.setTag("objectCount", currentObjectCount);
                    graphFetchSpan.setTag("avgMemoryUtilizationInBytesPerObject", memoryStatistics.getAverage());
                }
            }
            if (!nonEmptyObjectList) {
                if (topSpan != null && rowCount.get() > 0) {
                    topSpan.setTag("lastQueryRowCount", rowCount);
                }
            }
            if (checked) {
                return parentObjects.stream().map(x -> (IChecked<?>) x).map(x -> x.getValue() instanceof Constrained ? ((Constrained<?>) x.getValue()).toChecked(x.getSource(), enableConstraints) : x).collect(Collectors.toList());
            }
            if (enableConstraints) {
                return parentObjects.stream().map(x -> x instanceof Constrained ? ((Constrained<?>) x).withConstraintsApplied() : x).collect(Collectors.toList());
            }
            return parentObjects;
        }).flatMap(Collection::stream);
        boolean realizeAsConstant = this.executionState.inAllocation && ExecutionNodeResultHelper.isResultSizeRangeSet(globalGraphFetchExecutionNode) && ExecutionNodeResultHelper.isSingleRecordResult(globalGraphFetchExecutionNode);
        if (realizeAsConstant) {
            return new ConstantResult(objectStream.findFirst().orElseThrow(() -> new RuntimeException("Constant value not found")));
        }
        return new StreamingObjectResult<>(objectStream, new PartialClassBuilder(globalGraphFetchExecutionNode), graphFetchResult);
    } else {
        GraphObjectsBatch graphObjectsBatch = this.executionState.graphObjectsBatch;
        List<?> parentObjects = graphObjectsBatch.getObjectsForNodeIndex(globalGraphFetchExecutionNode.parentIndex);
        if ((parentObjects != null) && !parentObjects.isEmpty()) {
            if (globalGraphFetchExecutionNode.xStorePropertyFetchDetails != null && globalGraphFetchExecutionNode.xStorePropertyFetchDetails.supportsCaching && this.executionState.graphFetchCaches != null) {
                graphObjectsBatch.setXStorePropertyCachesForNodeIndex(globalGraphFetchExecutionNode.localGraphFetchExecutionNode.nodeIndex, findGraphFetchCacheByTargetCrossKeys(globalGraphFetchExecutionNode));
            }
            globalGraphFetchExecutionNode.localGraphFetchExecutionNode.accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
            if (globalGraphFetchExecutionNode.children != null && !globalGraphFetchExecutionNode.children.isEmpty()) {
                globalGraphFetchExecutionNode.children.forEach(c -> c.accept(new ExecutionNodeExecutor(this.profiles, this.executionState)));
            }
        }
        return new ConstantResult(parentObjects);
    }
}

Example 2 with GraphObjectsBatch

use of org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch in project legend-engine by finos.

the class InMemoryExecutionNodeExecutor method visit.

@Override
public Result visit(InMemoryPropertyGraphFetchExecutionNode node) {
    boolean isLeaf = node.children == null || node.children.isEmpty();
    IInMemoryPropertyGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.pm);
    GraphObjectsBatch graphObjectsBatch = this.executionState.graphObjectsBatch;
    List<?> parentObjects = graphObjectsBatch.getObjectsForNodeIndex(node.parentIndex).stream().map(x -> x instanceof IChecked ? ((IChecked<?>) x).getValue() : x).collect(Collectors.toList());
    Stream<IGraphInstance> childGraphInstancesStream = nodeSpecifics.transformProperty(parentObjects);
    List<Object> childObjects = childGraphInstancesStream.filter(Objects::nonNull).map(graphInstance -> {
        graphObjectsBatch.addObjectMemoryUtilization(graphInstance.instanceSize());
        return graphInstance.getValue();
    }).collect(Collectors.toList());
    graphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, childObjects);
    if (!childObjects.isEmpty() && (!isLeaf)) {
        node.children.forEach(x -> x.accept(new ExecutionNodeExecutor(this.pm, executionState)));
    }
    return new ConstantResult(childObjects);
}

Example 3 with GraphObjectsBatch

use of org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch in project legend-engine by finos.

the class InMemoryExecutionNodeExecutor method mergeInMemoryNode.

private Result mergeInMemoryNode(InMemoryRootGraphFetchExecutionNode node) {
    IInMemoryRootGraphFetchMergeExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.pm);
    List<GraphFetchResult> results = node.executionNodes.stream().map(n -> (GraphFetchResult) n.accept(new ExecutionNodeExecutor(this.pm, this.executionState))).collect(Collectors.toList());
    List<Object> subObjects = results.stream().map(g -> g.getGraphObjectsBatchStream().findFirst().get().getObjectsForNodeIndex(0).get(0)).collect(Collectors.toList());
    // merged object
    Object targetObject = nodeSpecifics.transform(subObjects);
    Spliterator<GraphObjectsBatch> graphObjectsBatchSpliterator = new Spliterators.AbstractSpliterator<GraphObjectsBatch>(Long.MAX_VALUE, Spliterator.ORDERED) {

        AtomicLong batchIndex = new AtomicLong(0L);

        @Override
        public boolean tryAdvance(Consumer<? super GraphObjectsBatch> action) {
            long currentBatch = batchIndex.incrementAndGet();
            if (// run only once
            currentBatch > 1) {
                return false;
            }
            List<Object> resultObjects = new ArrayList<>();
            GraphObjectsBatch inMemoryGraphObjectsBatch = new GraphObjectsBatch(currentBatch, executionState.getGraphFetchBatchMemoryLimit());
            IGraphInstance<?> target = (IGraphInstance<?>) targetObject;
            inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
            resultObjects.add(target.getValue());
            inMemoryGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, resultObjects);
            action.accept(inMemoryGraphObjectsBatch);
            return false;
        }
    };
    Stream<GraphObjectsBatch> graphObjectsBatchStream = StreamSupport.stream(graphObjectsBatchSpliterator, false);
    return new GraphFetchResult(graphObjectsBatchStream, new ConstantResult(targetObject));
}

Example 4 with GraphObjectsBatch

use of org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch in project legend-engine by finos.

the class InMemoryExecutionNodeExecutor method visit.

@Override
public Result visit(InMemoryRootGraphFetchExecutionNode node) {
    int batchSize = node.batchSize == null ? 1 : node.batchSize;
    boolean isLeaf = node.children == null || node.children.isEmpty();
    boolean checked = node.checked;
    ClassResultType classResultType = (ClassResultType) node.resultType;
    String _class = classResultType._class;
    Result childResult = null;
    JavaPlatformImplementation javaPlatformImpl = (JavaPlatformImplementation) node.implementation;
    String executionClassName = JavaHelper.getExecutionClassFullName(javaPlatformImpl);
    Class<?> clazz = ExecutionNodeJavaPlatformHelper.getClassToExecute(node, executionClassName, this.executionState, this.pm);
    Span graphFetchSpan = GlobalTracer.get().buildSpan("graph fetch").withTag("rootStoreType", "inMemory").withTag("batchSizeConfig", batchSize).start();
    GlobalTracer.get().activateSpan(graphFetchSpan);
    try {
        if ((Arrays.asList(clazz.getInterfaces()).contains(IInMemoryRootGraphFetchMergeExecutionNodeSpecifics.class))) {
            return mergeInMemoryNode(node);
        } else {
            IInMemoryRootGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.pm);
            childResult = node.executionNodes.get(0).accept(new ExecutionNodeExecutor(this.pm, this.executionState));
            Iterator<?> sourceObjectsIterator;
            if (childResult instanceof StoreStreamReadingResult) {
                StoreStreamReadingResult<?> storeStreamReadingResult = (StoreStreamReadingResult) childResult;
                sourceObjectsIterator = storeStreamReadingResult.getObjectsIterator();
            } else if (childResult instanceof StreamingObjectResult) {
                StreamingObjectResult<?> streamingObjectResult = (StreamingObjectResult) childResult;
                sourceObjectsIterator = streamingObjectResult.getObjectStream().iterator();
            } else {
                throw new IllegalStateException("Unsupported result type: " + childResult.getClass().getSimpleName());
            }
            AtomicLong batchIndex = new AtomicLong(0L);
            Spliterator<GraphObjectsBatch> graphObjectsBatchSpliterator = new Spliterators.AbstractSpliterator<GraphObjectsBatch>(Long.MAX_VALUE, Spliterator.ORDERED) {

                @Override
                public boolean tryAdvance(Consumer<? super GraphObjectsBatch> action) {
                    long currentBatch = batchIndex.incrementAndGet();
                    GraphObjectsBatch inMemoryGraphObjectsBatch = new GraphObjectsBatch(currentBatch, executionState.getGraphFetchBatchMemoryLimit());
                    List<Object> resultObjects = new ArrayList<>();
                    int objectCount = 0;
                    if (checked) {
                        while (sourceObjectsIterator.hasNext()) {
                            IChecked<?> checkedSource = (IChecked<?>) sourceObjectsIterator.next();
                            Object value = checkedSource.getValue();
                            if (value == null) {
                                resultObjects.add(newDynamicChecked(Collections.singletonList(BasicDefect.newNoInputDefect(_class)), checkedSource, null));
                            } else {
                                Object targetObject = nodeSpecifics.transform(value);
                                if (targetObject != null) {
                                    if (targetObject instanceof List) {
                                        ((List<?>) targetObject).forEach(x -> {
                                            IGraphInstance<?> target = (IGraphInstance<?>) x;
                                            inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
                                            resultObjects.add(newDynamicChecked(Collections.emptyList(), checkedSource, target.getValue()));
                                        });
                                    } else {
                                        IGraphInstance<?> target = (IGraphInstance<?>) targetObject;
                                        inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
                                        resultObjects.add(newDynamicChecked(Collections.emptyList(), checkedSource, target.getValue()));
                                    }
                                }
                            }
                            objectCount += 1;
                            if (objectCount >= batchSize)
                                break;
                        }
                    } else {
                        while (sourceObjectsIterator.hasNext()) {
                            Object targetObject = nodeSpecifics.transform(sourceObjectsIterator.next());
                            if (targetObject != null) {
                                if (targetObject instanceof List) {
                                    ((List<?>) targetObject).forEach(x -> {
                                        IGraphInstance<?> target = (IGraphInstance<?>) x;
                                        inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
                                        resultObjects.add(target.getValue());
                                    });
                                } else {
                                    IGraphInstance<?> target = (IGraphInstance<?>) targetObject;
                                    inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
                                    resultObjects.add(target.getValue());
                                }
                            }
                            objectCount += 1;
                            if (objectCount >= batchSize)
                                break;
                        }
                    }
                    inMemoryGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, resultObjects);
                    if (!resultObjects.isEmpty() && (!isLeaf)) {
                        ExecutionState newState = new ExecutionState(executionState);
                        newState.graphObjectsBatch = inMemoryGraphObjectsBatch;
                        node.children.forEach(x -> x.accept(new ExecutionNodeExecutor(InMemoryExecutionNodeExecutor.this.pm, newState)));
                    }
                    action.accept(inMemoryGraphObjectsBatch);
                    return objectCount != 0;
                }
            };
            Stream<GraphObjectsBatch> graphObjectsBatchStream = StreamSupport.stream(graphObjectsBatchSpliterator, false);
            return new GraphFetchResult(graphObjectsBatchStream, childResult).withGraphFetchSpan(graphFetchSpan);
        }
    } catch (Exception e) {
        if (childResult != null) {
            childResult.close();
        }
        if (graphFetchSpan != null) {
            graphFetchSpan.finish();
        }
        if (e instanceof RuntimeException) {
            throw e;
        }
        Throwable cause = e.getCause();
        if (cause instanceof RuntimeException)
            throw (RuntimeException) cause;
        if (cause instanceof Error)
            throw (Error) cause;
        throw new RuntimeException(cause);
    }
}

Example 5 with GraphObjectsBatch

use of org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch in project legend-engine by finos.

the class InMemoryExecutionNodeExecutor method visit.

@Override
public Result visit(InMemoryCrossStoreGraphFetchExecutionNode node) {
    List<Object> childObjects = new ArrayList<>();
    Result childResult = null;
    try {
        IInMemoryCrossStoreGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.pm);
        GraphObjectsBatch graphObjectsBatch = new GraphObjectsBatch(this.executionState.graphObjectsBatch);
        List<?> parentObjects = graphObjectsBatch.getObjectsForNodeIndex(node.parentIndex);
        if ((parentObjects != null) && !parentObjects.isEmpty()) {
            DoubleStrategyHashMap<Object, List<Object>, Object> parentMap = new DoubleStrategyHashMap<>(InMemoryGraphFetchUtils.parentChildDoubleHashStrategy(nodeSpecifics));
            parentObjects.forEach(parentObject -> parentMap.getIfAbsentPut(parentObject, ArrayList::new).add(parentObject));
            if (node.supportsBatching) {
                Map<String, List<Object>> keyValuePairs = Maps.mutable.empty();
                nodeSpecifics.getCrossStoreKeysValueForChildren(parentObjects.get(0)).keySet().forEach(key -> keyValuePairs.put(key, Lists.mutable.empty()));
                parentMap.keySet().forEach(parentObject -> nodeSpecifics.getCrossStoreKeysValueForChildren(parentObject).forEach((key, value) -> keyValuePairs.get(key).add(value)));
                keyValuePairs.forEach((key, value) -> this.executionState.addResult(key, new ConstantResult(value)));
                childResult = this.visit((InMemoryRootGraphFetchExecutionNode) node);
                GraphFetchResult childGraphFetchResult = (GraphFetchResult) childResult;
                Stream<GraphObjectsBatch> graphObjectsBatchStream = childGraphFetchResult.getGraphObjectsBatchStream();
                graphObjectsBatchStream.forEach(batch -> {
                    batch.getObjectsForNodeIndex(node.nodeIndex).forEach(child -> {
                        IGraphInstance<?> childGraphInstance = nodeSpecifics.wrapChildInGraphInstance(child);
                        Object childObject = childGraphInstance.getValue();
                        List<Object> parentsInScope = parentMap.getWithSecondKey(childObject);
                        if (parentsInScope != null) {
                            for (Object parentObject : parentsInScope) {
                                boolean isChildAdded = nodeSpecifics.attemptAddingChildToParent(parentObject, childObject);
                                if (isChildAdded) {
                                    graphObjectsBatch.addObjectMemoryUtilization(childGraphInstance.instanceSize());
                                    childObjects.add(childObject);
                                }
                            }
                        }
                    });
                });
            } else {
                for (Map.Entry<Object, List<Object>> entry : parentMap.entrySet()) {
                    Map<String, Object> keyValuePairs = nodeSpecifics.getCrossStoreKeysValueForChildren(entry.getKey());
                    keyValuePairs.forEach((key, value) -> this.executionState.addResult(key, new ConstantResult(value)));
                    childResult = this.visit((InMemoryRootGraphFetchExecutionNode) node);
                    GraphFetchResult childGraphFetchResult = (GraphFetchResult) childResult;
                    Stream<GraphObjectsBatch> graphObjectsBatchStream = childGraphFetchResult.getGraphObjectsBatchStream();
                    graphObjectsBatchStream.forEach(batch -> {
                        batch.getObjectsForNodeIndex(node.nodeIndex).forEach(child -> {
                            IGraphInstance<?> childGraphInstance = nodeSpecifics.wrapChildInGraphInstance(child);
                            Object childObject = childGraphInstance.getValue();
                            List<Object> parentsInScope = entry.getValue();
                            if (parentsInScope != null) {
                                for (Object parentObject : parentsInScope) {
                                    boolean isChildAdded = nodeSpecifics.attemptAddingChildToParent(parentObject, childObject);
                                    if (isChildAdded) {
                                        graphObjectsBatch.addObjectMemoryUtilization(childGraphInstance.instanceSize());
                                        childObjects.add(childObject);
                                    }
                                }
                            }
                        });
                    });
                }
            }
            graphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, childObjects);
        }
        return new ConstantResult(childObjects);
    } catch (RuntimeException e) {
        throw e;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        if (childResult != null) {
            childResult.close();
        }
    }
}