Examples with StoreStreamReadingResult org.finos.legend.engine.plan.execution.stores.inMemory.result.graphFetch.StoreStreamReadingResult used on opensource projects

Example 1 with StoreStreamReadingResult

use of org.finos.legend.engine.plan.execution.stores.inMemory.result.graphFetch.StoreStreamReadingResult in project legend-engine by finos.

the class InMemoryExecutionNodeExecutor method visit.

@Override
public Result visit(InMemoryRootGraphFetchExecutionNode node) {
    int batchSize = node.batchSize == null ? 1 : node.batchSize;
    boolean isLeaf = node.children == null || node.children.isEmpty();
    boolean checked = node.checked;
    ClassResultType classResultType = (ClassResultType) node.resultType;
    String _class = classResultType._class;
    Result childResult = null;
    JavaPlatformImplementation javaPlatformImpl = (JavaPlatformImplementation) node.implementation;
    String executionClassName = JavaHelper.getExecutionClassFullName(javaPlatformImpl);
    Class<?> clazz = ExecutionNodeJavaPlatformHelper.getClassToExecute(node, executionClassName, this.executionState, this.pm);
    Span graphFetchSpan = GlobalTracer.get().buildSpan("graph fetch").withTag("rootStoreType", "inMemory").withTag("batchSizeConfig", batchSize).start();
    GlobalTracer.get().activateSpan(graphFetchSpan);
    try {
        if ((Arrays.asList(clazz.getInterfaces()).contains(IInMemoryRootGraphFetchMergeExecutionNodeSpecifics.class))) {
            return mergeInMemoryNode(node);
        } else {
            IInMemoryRootGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.pm);
            childResult = node.executionNodes.get(0).accept(new ExecutionNodeExecutor(this.pm, this.executionState));
            Iterator<?> sourceObjectsIterator;
            if (childResult instanceof StoreStreamReadingResult) {
                StoreStreamReadingResult<?> storeStreamReadingResult = (StoreStreamReadingResult) childResult;
                sourceObjectsIterator = storeStreamReadingResult.getObjectsIterator();
            } else if (childResult instanceof StreamingObjectResult) {
                StreamingObjectResult<?> streamingObjectResult = (StreamingObjectResult) childResult;
                sourceObjectsIterator = streamingObjectResult.getObjectStream().iterator();
            } else {
                throw new IllegalStateException("Unsupported result type: " + childResult.getClass().getSimpleName());
            }
            AtomicLong batchIndex = new AtomicLong(0L);
            Spliterator<GraphObjectsBatch> graphObjectsBatchSpliterator = new Spliterators.AbstractSpliterator<GraphObjectsBatch>(Long.MAX_VALUE, Spliterator.ORDERED) {

                @Override
                public boolean tryAdvance(Consumer<? super GraphObjectsBatch> action) {
                    long currentBatch = batchIndex.incrementAndGet();
                    GraphObjectsBatch inMemoryGraphObjectsBatch = new GraphObjectsBatch(currentBatch, executionState.getGraphFetchBatchMemoryLimit());
                    List<Object> resultObjects = new ArrayList<>();
                    int objectCount = 0;
                    if (checked) {
                        while (sourceObjectsIterator.hasNext()) {
                            IChecked<?> checkedSource = (IChecked<?>) sourceObjectsIterator.next();
                            Object value = checkedSource.getValue();
                            if (value == null) {
                                resultObjects.add(newDynamicChecked(Collections.singletonList(BasicDefect.newNoInputDefect(_class)), checkedSource, null));
                            } else {
                                Object targetObject = nodeSpecifics.transform(value);
                                if (targetObject != null) {
                                    if (targetObject instanceof List) {
                                        ((List<?>) targetObject).forEach(x -> {
                                            IGraphInstance<?> target = (IGraphInstance<?>) x;
                                            inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
                                            resultObjects.add(newDynamicChecked(Collections.emptyList(), checkedSource, target.getValue()));
                                        });
                                    } else {
                                        IGraphInstance<?> target = (IGraphInstance<?>) targetObject;
                                        inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
                                        resultObjects.add(newDynamicChecked(Collections.emptyList(), checkedSource, target.getValue()));
                                    }
                                }
                            }
                            objectCount += 1;
                            if (objectCount >= batchSize)
                                break;
                        }
                    } else {
                        while (sourceObjectsIterator.hasNext()) {
                            Object targetObject = nodeSpecifics.transform(sourceObjectsIterator.next());
                            if (targetObject != null) {
                                if (targetObject instanceof List) {
                                    ((List<?>) targetObject).forEach(x -> {
                                        IGraphInstance<?> target = (IGraphInstance<?>) x;
                                        inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
                                        resultObjects.add(target.getValue());
                                    });
                                } else {
                                    IGraphInstance<?> target = (IGraphInstance<?>) targetObject;
                                    inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
                                    resultObjects.add(target.getValue());
                                }
                            }
                            objectCount += 1;
                            if (objectCount >= batchSize)
                                break;
                        }
                    }
                    inMemoryGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, resultObjects);
                    if (!resultObjects.isEmpty() && (!isLeaf)) {
                        ExecutionState newState = new ExecutionState(executionState);
                        newState.graphObjectsBatch = inMemoryGraphObjectsBatch;
                        node.children.forEach(x -> x.accept(new ExecutionNodeExecutor(InMemoryExecutionNodeExecutor.this.pm, newState)));
                    }
                    action.accept(inMemoryGraphObjectsBatch);
                    return objectCount != 0;
                }
            };
            Stream<GraphObjectsBatch> graphObjectsBatchStream = StreamSupport.stream(graphObjectsBatchSpliterator, false);
            return new GraphFetchResult(graphObjectsBatchStream, childResult).withGraphFetchSpan(graphFetchSpan);
        }
    } catch (Exception e) {
        if (childResult != null) {
            childResult.close();
        }
        if (graphFetchSpan != null) {
            graphFetchSpan.finish();
        }
        if (e instanceof RuntimeException) {
            throw e;
        }
        Throwable cause = e.getCause();
        if (cause instanceof RuntimeException)
            throw (RuntimeException) cause;
        if (cause instanceof Error)
            throw (Error) cause;
        throw new RuntimeException(cause);
    }
}