Search in sources :

Example 1 with GraphFetchResult

use of org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult in project legend-engine by finos.

the class ExecutionNodeExecutor method visit.

@Override
public Result visit(GlobalGraphFetchExecutionNode globalGraphFetchExecutionNode) {
    final Span topSpan = GlobalTracer.get().activeSpan();
    final boolean isGraphRoot = globalGraphFetchExecutionNode.parentIndex == null;
    if (isGraphRoot) {
        final boolean enableConstraints = globalGraphFetchExecutionNode.enableConstraints == null ? false : globalGraphFetchExecutionNode.enableConstraints;
        final boolean checked = globalGraphFetchExecutionNode.checked == null ? false : globalGraphFetchExecutionNode.checked;
        // Handle batching at root level
        final AtomicLong rowCount = new AtomicLong(0L);
        final AtomicLong objectCount = new AtomicLong(0L);
        final DoubleSummaryStatistics memoryStatistics = new DoubleSummaryStatistics();
        GraphFetchResult graphFetchResult = (GraphFetchResult) globalGraphFetchExecutionNode.localGraphFetchExecutionNode.accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
        Stream<?> objectStream = graphFetchResult.getGraphObjectsBatchStream().map(batch -> {
            List<?> parentObjects = batch.getObjectsForNodeIndex(0);
            boolean nonEmptyObjectList = !parentObjects.isEmpty();
            ExecutionState newState = new ExecutionState(this.executionState).setGraphObjectsBatch(batch);
            if (globalGraphFetchExecutionNode.children != null && !globalGraphFetchExecutionNode.children.isEmpty() && nonEmptyObjectList) {
                globalGraphFetchExecutionNode.children.forEach(c -> c.accept(new ExecutionNodeExecutor(this.profiles, newState)));
            }
            rowCount.addAndGet(batch.getRowCount());
            if (nonEmptyObjectList) {
                long currentObjectCount = objectCount.addAndGet(parentObjects.size());
                memoryStatistics.accept(batch.getTotalObjectMemoryUtilization() / (parentObjects.size() * 1.0));
                if (graphFetchResult.getGraphFetchSpan() != null) {
                    Span graphFetchSpan = graphFetchResult.getGraphFetchSpan();
                    graphFetchSpan.setTag("batchCount", memoryStatistics.getCount());
                    graphFetchSpan.setTag("objectCount", currentObjectCount);
                    graphFetchSpan.setTag("avgMemoryUtilizationInBytesPerObject", memoryStatistics.getAverage());
                }
            }
            if (!nonEmptyObjectList) {
                if (topSpan != null && rowCount.get() > 0) {
                    topSpan.setTag("lastQueryRowCount", rowCount);
                }
            }
            if (checked) {
                return parentObjects.stream().map(x -> (IChecked<?>) x).map(x -> x.getValue() instanceof Constrained ? ((Constrained<?>) x.getValue()).toChecked(x.getSource(), enableConstraints) : x).collect(Collectors.toList());
            }
            if (enableConstraints) {
                return parentObjects.stream().map(x -> x instanceof Constrained ? ((Constrained<?>) x).withConstraintsApplied() : x).collect(Collectors.toList());
            }
            return parentObjects;
        }).flatMap(Collection::stream);
        boolean realizeAsConstant = this.executionState.inAllocation && ExecutionNodeResultHelper.isResultSizeRangeSet(globalGraphFetchExecutionNode) && ExecutionNodeResultHelper.isSingleRecordResult(globalGraphFetchExecutionNode);
        if (realizeAsConstant) {
            return new ConstantResult(objectStream.findFirst().orElseThrow(() -> new RuntimeException("Constant value not found")));
        }
        return new StreamingObjectResult<>(objectStream, new PartialClassBuilder(globalGraphFetchExecutionNode), graphFetchResult);
    } else {
        GraphObjectsBatch graphObjectsBatch = this.executionState.graphObjectsBatch;
        List<?> parentObjects = graphObjectsBatch.getObjectsForNodeIndex(globalGraphFetchExecutionNode.parentIndex);
        if ((parentObjects != null) && !parentObjects.isEmpty()) {
            if (globalGraphFetchExecutionNode.xStorePropertyFetchDetails != null && globalGraphFetchExecutionNode.xStorePropertyFetchDetails.supportsCaching && this.executionState.graphFetchCaches != null) {
                graphObjectsBatch.setXStorePropertyCachesForNodeIndex(globalGraphFetchExecutionNode.localGraphFetchExecutionNode.nodeIndex, findGraphFetchCacheByTargetCrossKeys(globalGraphFetchExecutionNode));
            }
            globalGraphFetchExecutionNode.localGraphFetchExecutionNode.accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
            if (globalGraphFetchExecutionNode.children != null && !globalGraphFetchExecutionNode.children.isEmpty()) {
                globalGraphFetchExecutionNode.children.forEach(c -> c.accept(new ExecutionNodeExecutor(this.profiles, this.executionState)));
            }
        }
        return new ConstantResult(parentObjects);
    }
}
Also used : Maps(org.eclipse.collections.impl.factory.Maps) ExecutionNodeVisitor(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ExecutionNodeVisitor) Arrays(java.util.Arrays) DefaultExecutionNodeContext(org.finos.legend.engine.plan.execution.nodes.helpers.platform.DefaultExecutionNodeContext) Spliterators(java.util.Spliterators) GraphFetchCacheKey(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheKey) ExecutionNodeSerializerHelper(org.finos.legend.engine.plan.execution.nodes.helpers.ExecutionNodeSerializerHelper) DoubleSummaryStatistics(java.util.DoubleSummaryStatistics) MutableList(org.eclipse.collections.api.list.MutableList) InMemoryCrossStoreGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryCrossStoreGraphFetchExecutionNode) ErrorExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ErrorExecutionNode) IPlatformPureExpressionExecutionNodeSerializeSpecifics(org.finos.legend.engine.plan.dependencies.store.platform.IPlatformPureExpressionExecutionNodeSerializeSpecifics) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) StoreStreamReadingExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.StoreStreamReadingExecutionNode) Map(java.util.Map) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) ExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ExecutionNode) FunctionParametersValidationNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.FunctionParametersValidationNode) AppliedFunction(org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.application.AppliedFunction) AllocationExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.AllocationExecutionNode) ConstantExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ConstantExecutionNode) Collection(java.util.Collection) GraphExecutionState(org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState) FunctionParametersParametersValidation(org.finos.legend.engine.plan.execution.validation.FunctionParametersParametersValidation) SerializationConfig(org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.SerializationConfig) Collectors(java.util.stream.Collectors) Constrained(org.finos.legend.engine.plan.dependencies.domain.dataQuality.Constrained) IExecutionNodeContext(org.finos.legend.engine.plan.dependencies.store.shared.IExecutionNodeContext) Objects(java.util.Objects) InMemoryRootGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryRootGraphFetchExecutionNode) List(java.util.List) Stream(java.util.stream.Stream) Lists(org.eclipse.collections.impl.factory.Lists) Span(io.opentracing.Span) Scope(io.opentracing.Scope) FreeMarkerConditionalExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.FreeMarkerConditionalExecutionNode) Spliterator(java.util.Spliterator) FreeMarkerExecutor(org.finos.legend.engine.plan.execution.nodes.helpers.freemarker.FreeMarkerExecutor) IPlatformPureExpressionExecutionNodeGraphFetchUnionSpecifics(org.finos.legend.engine.plan.dependencies.store.platform.IPlatformPureExpressionExecutionNodeGraphFetchUnionSpecifics) StoreType(org.finos.legend.engine.plan.execution.stores.StoreType) MultiResultSequenceExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.MultiResultSequenceExecutionNode) CommonProfile(org.pac4j.core.profile.CommonProfile) ExecutionState(org.finos.legend.engine.plan.execution.nodes.state.ExecutionState) GraphFetchCache(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCache) AggregationAwareExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.AggregationAwareExecutionNode) IPlatformPureExpressionExecutionNodeGraphFetchMergeSpecifics(org.finos.legend.engine.plan.dependencies.store.platform.IPlatformPureExpressionExecutionNodeGraphFetchMergeSpecifics) PartialClassBuilder(org.finos.legend.engine.plan.execution.result.builder._class.PartialClassBuilder) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) GraphFetchM2MExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.GraphFetchM2MExecutionNode) ExecutionNodeResultHelper(org.finos.legend.engine.plan.execution.nodes.helpers.ExecutionNodeResultHelper) GraphFetchCrossAssociationKeys(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCrossAssociationKeys) ExecutionCache(org.finos.legend.engine.plan.execution.cache.ExecutionCache) PureExpressionPlatformExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.PureExpressionPlatformExecutionNode) GraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.GraphFetchExecutionNode) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) JavaHelper(org.finos.legend.engine.plan.execution.nodes.helpers.platform.JavaHelper) StreamSupport(java.util.stream.StreamSupport) IChecked(org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked) JavaPlatformImplementation(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.JavaPlatformImplementation) ErrorResult(org.finos.legend.engine.plan.execution.result.ErrorResult) SequenceExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.SequenceExecutionNode) Result(org.finos.legend.engine.plan.execution.result.Result) ResultVisitor(org.finos.legend.engine.plan.execution.result.ResultVisitor) XStorePropertyFetchDetails(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.XStorePropertyFetchDetails) GraphFetchCacheByTargetCrossKeys(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheByTargetCrossKeys) ExecutionNodeJavaPlatformHelper(org.finos.legend.engine.plan.execution.nodes.helpers.platform.ExecutionNodeJavaPlatformHelper) GlobalTracer(io.opentracing.util.GlobalTracer) GraphObjectsBatch(org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch) InMemoryPropertyGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryPropertyGraphFetchExecutionNode) ProfileManager(org.pac4j.core.profile.ProfileManager) Consumer(java.util.function.Consumer) AtomicLong(java.util.concurrent.atomic.AtomicLong) MultiResult(org.finos.legend.engine.plan.execution.result.MultiResult) GlobalGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.GlobalGraphFetchExecutionNode) LocalGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.LocalGraphFetchExecutionNode) GraphExecutionState(org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState) ExecutionState(org.finos.legend.engine.plan.execution.nodes.state.ExecutionState) PartialClassBuilder(org.finos.legend.engine.plan.execution.result.builder._class.PartialClassBuilder) GraphObjectsBatch(org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) DoubleSummaryStatistics(java.util.DoubleSummaryStatistics) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) Span(io.opentracing.Span) AtomicLong(java.util.concurrent.atomic.AtomicLong) IChecked(org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked) Constrained(org.finos.legend.engine.plan.dependencies.domain.dataQuality.Constrained) Collection(java.util.Collection) MutableList(org.eclipse.collections.api.list.MutableList) List(java.util.List)

Example 2 with GraphFetchResult

use of org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult in project legend-engine by finos.

the class InMemoryExecutionNodeExecutor method mergeInMemoryNode.

private Result mergeInMemoryNode(InMemoryRootGraphFetchExecutionNode node) {
    IInMemoryRootGraphFetchMergeExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.pm);
    List<GraphFetchResult> results = node.executionNodes.stream().map(n -> (GraphFetchResult) n.accept(new ExecutionNodeExecutor(this.pm, this.executionState))).collect(Collectors.toList());
    List<Object> subObjects = results.stream().map(g -> g.getGraphObjectsBatchStream().findFirst().get().getObjectsForNodeIndex(0).get(0)).collect(Collectors.toList());
    // merged object
    Object targetObject = nodeSpecifics.transform(subObjects);
    Spliterator<GraphObjectsBatch> graphObjectsBatchSpliterator = new Spliterators.AbstractSpliterator<GraphObjectsBatch>(Long.MAX_VALUE, Spliterator.ORDERED) {

        AtomicLong batchIndex = new AtomicLong(0L);

        @Override
        public boolean tryAdvance(Consumer<? super GraphObjectsBatch> action) {
            long currentBatch = batchIndex.incrementAndGet();
            if (// run only once
            currentBatch > 1) {
                return false;
            }
            List<Object> resultObjects = new ArrayList<>();
            GraphObjectsBatch inMemoryGraphObjectsBatch = new GraphObjectsBatch(currentBatch, executionState.getGraphFetchBatchMemoryLimit());
            IGraphInstance<?> target = (IGraphInstance<?>) targetObject;
            inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
            resultObjects.add(target.getValue());
            inMemoryGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, resultObjects);
            action.accept(inMemoryGraphObjectsBatch);
            return false;
        }
    };
    Stream<GraphObjectsBatch> graphObjectsBatchStream = StreamSupport.stream(graphObjectsBatchSpliterator, false);
    return new GraphFetchResult(graphObjectsBatchStream, new ConstantResult(targetObject));
}
Also used : java.util(java.util) org.finos.legend.engine.plan.dependencies.store.inMemory.graphFetch(org.finos.legend.engine.plan.dependencies.store.inMemory.graphFetch) CommonProfile(org.pac4j.core.profile.CommonProfile) IGraphInstance(org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance) DoubleStrategyHashMap(org.finos.legend.engine.shared.core.collectionsExtensions.DoubleStrategyHashMap) ExecutionState(org.finos.legend.engine.plan.execution.nodes.state.ExecutionState) org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes) Lists(org.eclipse.collections.api.factory.Lists) BasicDefect(org.finos.legend.engine.plan.dependencies.domain.dataQuality.BasicDefect) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) ClassResultType(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.result.ClassResultType) MutableList(org.eclipse.collections.api.list.MutableList) InMemoryCrossStoreGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryCrossStoreGraphFetchExecutionNode) StoreStreamReadingResult(org.finos.legend.engine.plan.execution.stores.inMemory.result.graphFetch.StoreStreamReadingResult) Maps(org.eclipse.collections.api.factory.Maps) GraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.GraphFetchExecutionNode) ExecutionNodeExecutor(org.finos.legend.engine.plan.execution.nodes.ExecutionNodeExecutor) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) JavaHelper(org.finos.legend.engine.plan.execution.nodes.helpers.platform.JavaHelper) StoreStreamReadingExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.StoreStreamReadingExecutionNode) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) StreamSupport(java.util.stream.StreamSupport) IChecked(org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked) Result(org.finos.legend.engine.plan.execution.result.Result) ExecutionNodeJavaPlatformHelper(org.finos.legend.engine.plan.execution.nodes.helpers.platform.ExecutionNodeJavaPlatformHelper) GlobalTracer(io.opentracing.util.GlobalTracer) GraphObjectsBatch(org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch) InMemoryPropertyGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryPropertyGraphFetchExecutionNode) Collectors(java.util.stream.Collectors) Consumer(java.util.function.Consumer) AtomicLong(java.util.concurrent.atomic.AtomicLong) IDefect(org.finos.legend.engine.plan.dependencies.domain.dataQuality.IDefect) InMemoryRootGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryRootGraphFetchExecutionNode) Stream(java.util.stream.Stream) Span(io.opentracing.Span) GlobalGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.GlobalGraphFetchExecutionNode) InMemoryGraphFetchUtils(org.finos.legend.engine.plan.execution.stores.inMemory.utils.InMemoryGraphFetchUtils) LocalGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.LocalGraphFetchExecutionNode) IGraphInstance(org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance) GraphObjectsBatch(org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) ExecutionNodeExecutor(org.finos.legend.engine.plan.execution.nodes.ExecutionNodeExecutor) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) AtomicLong(java.util.concurrent.atomic.AtomicLong) Consumer(java.util.function.Consumer)

Example 3 with GraphFetchResult

use of org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult in project legend-engine by finos.

the class InMemoryExecutionNodeExecutor method visit.

@Override
public Result visit(InMemoryRootGraphFetchExecutionNode node) {
    int batchSize = node.batchSize == null ? 1 : node.batchSize;
    boolean isLeaf = node.children == null || node.children.isEmpty();
    boolean checked = node.checked;
    ClassResultType classResultType = (ClassResultType) node.resultType;
    String _class = classResultType._class;
    Result childResult = null;
    JavaPlatformImplementation javaPlatformImpl = (JavaPlatformImplementation) node.implementation;
    String executionClassName = JavaHelper.getExecutionClassFullName(javaPlatformImpl);
    Class<?> clazz = ExecutionNodeJavaPlatformHelper.getClassToExecute(node, executionClassName, this.executionState, this.pm);
    Span graphFetchSpan = GlobalTracer.get().buildSpan("graph fetch").withTag("rootStoreType", "inMemory").withTag("batchSizeConfig", batchSize).start();
    GlobalTracer.get().activateSpan(graphFetchSpan);
    try {
        if ((Arrays.asList(clazz.getInterfaces()).contains(IInMemoryRootGraphFetchMergeExecutionNodeSpecifics.class))) {
            return mergeInMemoryNode(node);
        } else {
            IInMemoryRootGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.pm);
            childResult = node.executionNodes.get(0).accept(new ExecutionNodeExecutor(this.pm, this.executionState));
            Iterator<?> sourceObjectsIterator;
            if (childResult instanceof StoreStreamReadingResult) {
                StoreStreamReadingResult<?> storeStreamReadingResult = (StoreStreamReadingResult) childResult;
                sourceObjectsIterator = storeStreamReadingResult.getObjectsIterator();
            } else if (childResult instanceof StreamingObjectResult) {
                StreamingObjectResult<?> streamingObjectResult = (StreamingObjectResult) childResult;
                sourceObjectsIterator = streamingObjectResult.getObjectStream().iterator();
            } else {
                throw new IllegalStateException("Unsupported result type: " + childResult.getClass().getSimpleName());
            }
            AtomicLong batchIndex = new AtomicLong(0L);
            Spliterator<GraphObjectsBatch> graphObjectsBatchSpliterator = new Spliterators.AbstractSpliterator<GraphObjectsBatch>(Long.MAX_VALUE, Spliterator.ORDERED) {

                @Override
                public boolean tryAdvance(Consumer<? super GraphObjectsBatch> action) {
                    long currentBatch = batchIndex.incrementAndGet();
                    GraphObjectsBatch inMemoryGraphObjectsBatch = new GraphObjectsBatch(currentBatch, executionState.getGraphFetchBatchMemoryLimit());
                    List<Object> resultObjects = new ArrayList<>();
                    int objectCount = 0;
                    if (checked) {
                        while (sourceObjectsIterator.hasNext()) {
                            IChecked<?> checkedSource = (IChecked<?>) sourceObjectsIterator.next();
                            Object value = checkedSource.getValue();
                            if (value == null) {
                                resultObjects.add(newDynamicChecked(Collections.singletonList(BasicDefect.newNoInputDefect(_class)), checkedSource, null));
                            } else {
                                Object targetObject = nodeSpecifics.transform(value);
                                if (targetObject != null) {
                                    if (targetObject instanceof List) {
                                        ((List<?>) targetObject).forEach(x -> {
                                            IGraphInstance<?> target = (IGraphInstance<?>) x;
                                            inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
                                            resultObjects.add(newDynamicChecked(Collections.emptyList(), checkedSource, target.getValue()));
                                        });
                                    } else {
                                        IGraphInstance<?> target = (IGraphInstance<?>) targetObject;
                                        inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
                                        resultObjects.add(newDynamicChecked(Collections.emptyList(), checkedSource, target.getValue()));
                                    }
                                }
                            }
                            objectCount += 1;
                            if (objectCount >= batchSize)
                                break;
                        }
                    } else {
                        while (sourceObjectsIterator.hasNext()) {
                            Object targetObject = nodeSpecifics.transform(sourceObjectsIterator.next());
                            if (targetObject != null) {
                                if (targetObject instanceof List) {
                                    ((List<?>) targetObject).forEach(x -> {
                                        IGraphInstance<?> target = (IGraphInstance<?>) x;
                                        inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
                                        resultObjects.add(target.getValue());
                                    });
                                } else {
                                    IGraphInstance<?> target = (IGraphInstance<?>) targetObject;
                                    inMemoryGraphObjectsBatch.addObjectMemoryUtilization(target.instanceSize());
                                    resultObjects.add(target.getValue());
                                }
                            }
                            objectCount += 1;
                            if (objectCount >= batchSize)
                                break;
                        }
                    }
                    inMemoryGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, resultObjects);
                    if (!resultObjects.isEmpty() && (!isLeaf)) {
                        ExecutionState newState = new ExecutionState(executionState);
                        newState.graphObjectsBatch = inMemoryGraphObjectsBatch;
                        node.children.forEach(x -> x.accept(new ExecutionNodeExecutor(InMemoryExecutionNodeExecutor.this.pm, newState)));
                    }
                    action.accept(inMemoryGraphObjectsBatch);
                    return objectCount != 0;
                }
            };
            Stream<GraphObjectsBatch> graphObjectsBatchStream = StreamSupport.stream(graphObjectsBatchSpliterator, false);
            return new GraphFetchResult(graphObjectsBatchStream, childResult).withGraphFetchSpan(graphFetchSpan);
        }
    } catch (Exception e) {
        if (childResult != null) {
            childResult.close();
        }
        if (graphFetchSpan != null) {
            graphFetchSpan.finish();
        }
        if (e instanceof RuntimeException) {
            throw e;
        }
        Throwable cause = e.getCause();
        if (cause instanceof RuntimeException)
            throw (RuntimeException) cause;
        if (cause instanceof Error)
            throw (Error) cause;
        throw new RuntimeException(cause);
    }
}
Also used : ExecutionState(org.finos.legend.engine.plan.execution.nodes.state.ExecutionState) IGraphInstance(org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance) ClassResultType(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.result.ClassResultType) ExecutionNodeExecutor(org.finos.legend.engine.plan.execution.nodes.ExecutionNodeExecutor) Span(io.opentracing.Span) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) StoreStreamReadingResult(org.finos.legend.engine.plan.execution.stores.inMemory.result.graphFetch.StoreStreamReadingResult) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) Result(org.finos.legend.engine.plan.execution.result.Result) Consumer(java.util.function.Consumer) MutableList(org.eclipse.collections.api.list.MutableList) GraphObjectsBatch(org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) AtomicLong(java.util.concurrent.atomic.AtomicLong) IChecked(org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked) StoreStreamReadingResult(org.finos.legend.engine.plan.execution.stores.inMemory.result.graphFetch.StoreStreamReadingResult)

Example 4 with GraphFetchResult

use of org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult in project legend-engine by finos.

the class InMemoryExecutionNodeExecutor method visit.

@Override
public Result visit(InMemoryCrossStoreGraphFetchExecutionNode node) {
    List<Object> childObjects = new ArrayList<>();
    Result childResult = null;
    try {
        IInMemoryCrossStoreGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.pm);
        GraphObjectsBatch graphObjectsBatch = new GraphObjectsBatch(this.executionState.graphObjectsBatch);
        List<?> parentObjects = graphObjectsBatch.getObjectsForNodeIndex(node.parentIndex);
        if ((parentObjects != null) && !parentObjects.isEmpty()) {
            DoubleStrategyHashMap<Object, List<Object>, Object> parentMap = new DoubleStrategyHashMap<>(InMemoryGraphFetchUtils.parentChildDoubleHashStrategy(nodeSpecifics));
            parentObjects.forEach(parentObject -> parentMap.getIfAbsentPut(parentObject, ArrayList::new).add(parentObject));
            if (node.supportsBatching) {
                Map<String, List<Object>> keyValuePairs = Maps.mutable.empty();
                nodeSpecifics.getCrossStoreKeysValueForChildren(parentObjects.get(0)).keySet().forEach(key -> keyValuePairs.put(key, Lists.mutable.empty()));
                parentMap.keySet().forEach(parentObject -> nodeSpecifics.getCrossStoreKeysValueForChildren(parentObject).forEach((key, value) -> keyValuePairs.get(key).add(value)));
                keyValuePairs.forEach((key, value) -> this.executionState.addResult(key, new ConstantResult(value)));
                childResult = this.visit((InMemoryRootGraphFetchExecutionNode) node);
                GraphFetchResult childGraphFetchResult = (GraphFetchResult) childResult;
                Stream<GraphObjectsBatch> graphObjectsBatchStream = childGraphFetchResult.getGraphObjectsBatchStream();
                graphObjectsBatchStream.forEach(batch -> {
                    batch.getObjectsForNodeIndex(node.nodeIndex).forEach(child -> {
                        IGraphInstance<?> childGraphInstance = nodeSpecifics.wrapChildInGraphInstance(child);
                        Object childObject = childGraphInstance.getValue();
                        List<Object> parentsInScope = parentMap.getWithSecondKey(childObject);
                        if (parentsInScope != null) {
                            for (Object parentObject : parentsInScope) {
                                boolean isChildAdded = nodeSpecifics.attemptAddingChildToParent(parentObject, childObject);
                                if (isChildAdded) {
                                    graphObjectsBatch.addObjectMemoryUtilization(childGraphInstance.instanceSize());
                                    childObjects.add(childObject);
                                }
                            }
                        }
                    });
                });
            } else {
                for (Map.Entry<Object, List<Object>> entry : parentMap.entrySet()) {
                    Map<String, Object> keyValuePairs = nodeSpecifics.getCrossStoreKeysValueForChildren(entry.getKey());
                    keyValuePairs.forEach((key, value) -> this.executionState.addResult(key, new ConstantResult(value)));
                    childResult = this.visit((InMemoryRootGraphFetchExecutionNode) node);
                    GraphFetchResult childGraphFetchResult = (GraphFetchResult) childResult;
                    Stream<GraphObjectsBatch> graphObjectsBatchStream = childGraphFetchResult.getGraphObjectsBatchStream();
                    graphObjectsBatchStream.forEach(batch -> {
                        batch.getObjectsForNodeIndex(node.nodeIndex).forEach(child -> {
                            IGraphInstance<?> childGraphInstance = nodeSpecifics.wrapChildInGraphInstance(child);
                            Object childObject = childGraphInstance.getValue();
                            List<Object> parentsInScope = entry.getValue();
                            if (parentsInScope != null) {
                                for (Object parentObject : parentsInScope) {
                                    boolean isChildAdded = nodeSpecifics.attemptAddingChildToParent(parentObject, childObject);
                                    if (isChildAdded) {
                                        graphObjectsBatch.addObjectMemoryUtilization(childGraphInstance.instanceSize());
                                        childObjects.add(childObject);
                                    }
                                }
                            }
                        });
                    });
                }
            }
            graphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, childObjects);
        }
        return new ConstantResult(childObjects);
    } catch (RuntimeException e) {
        throw e;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        if (childResult != null) {
            childResult.close();
        }
    }
}
Also used : java.util(java.util) org.finos.legend.engine.plan.dependencies.store.inMemory.graphFetch(org.finos.legend.engine.plan.dependencies.store.inMemory.graphFetch) CommonProfile(org.pac4j.core.profile.CommonProfile) IGraphInstance(org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance) DoubleStrategyHashMap(org.finos.legend.engine.shared.core.collectionsExtensions.DoubleStrategyHashMap) ExecutionState(org.finos.legend.engine.plan.execution.nodes.state.ExecutionState) org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes) Lists(org.eclipse.collections.api.factory.Lists) BasicDefect(org.finos.legend.engine.plan.dependencies.domain.dataQuality.BasicDefect) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) ClassResultType(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.result.ClassResultType) MutableList(org.eclipse.collections.api.list.MutableList) InMemoryCrossStoreGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryCrossStoreGraphFetchExecutionNode) StoreStreamReadingResult(org.finos.legend.engine.plan.execution.stores.inMemory.result.graphFetch.StoreStreamReadingResult) Maps(org.eclipse.collections.api.factory.Maps) GraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.GraphFetchExecutionNode) ExecutionNodeExecutor(org.finos.legend.engine.plan.execution.nodes.ExecutionNodeExecutor) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) JavaHelper(org.finos.legend.engine.plan.execution.nodes.helpers.platform.JavaHelper) StoreStreamReadingExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.StoreStreamReadingExecutionNode) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) StreamSupport(java.util.stream.StreamSupport) IChecked(org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked) Result(org.finos.legend.engine.plan.execution.result.Result) ExecutionNodeJavaPlatformHelper(org.finos.legend.engine.plan.execution.nodes.helpers.platform.ExecutionNodeJavaPlatformHelper) GlobalTracer(io.opentracing.util.GlobalTracer) GraphObjectsBatch(org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch) InMemoryPropertyGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryPropertyGraphFetchExecutionNode) Collectors(java.util.stream.Collectors) Consumer(java.util.function.Consumer) AtomicLong(java.util.concurrent.atomic.AtomicLong) IDefect(org.finos.legend.engine.plan.dependencies.domain.dataQuality.IDefect) InMemoryRootGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryRootGraphFetchExecutionNode) Stream(java.util.stream.Stream) Span(io.opentracing.Span) GlobalGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.GlobalGraphFetchExecutionNode) InMemoryGraphFetchUtils(org.finos.legend.engine.plan.execution.stores.inMemory.utils.InMemoryGraphFetchUtils) LocalGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.LocalGraphFetchExecutionNode) InMemoryRootGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryRootGraphFetchExecutionNode) GraphObjectsBatch(org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) DoubleStrategyHashMap(org.finos.legend.engine.shared.core.collectionsExtensions.DoubleStrategyHashMap) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) StoreStreamReadingResult(org.finos.legend.engine.plan.execution.stores.inMemory.result.graphFetch.StoreStreamReadingResult) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) Result(org.finos.legend.engine.plan.execution.result.Result) MutableList(org.eclipse.collections.api.list.MutableList) DoubleStrategyHashMap(org.finos.legend.engine.shared.core.collectionsExtensions.DoubleStrategyHashMap)

Example 5 with GraphFetchResult

use of org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult in project legend-engine by finos.

the class RelationalExecutionNodeExecutor method executeRelationalRootQueryTempTableGraphFetchExecutionNode.

private Result executeRelationalRootQueryTempTableGraphFetchExecutionNode(RelationalRootQueryTempTableGraphFetchExecutionNode node) {
    int batchSize = node.batchSize == null ? 1000 : node.batchSize;
    boolean isLeaf = node.children == null || node.children.isEmpty();
    Result rootResult = null;
    Span graphFetchSpan = GlobalTracer.get().buildSpan("graph fetch").withTag("rootStoreType", "relational").withTag("batchSizeConfig", batchSize).start();
    GlobalTracer.get().activateSpan(graphFetchSpan);
    try {
        rootResult = node.executionNodes.get(0).accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
        SQLExecutionResult sqlExecutionResult = (SQLExecutionResult) rootResult;
        DatabaseConnection databaseConnection = sqlExecutionResult.getSQLExecutionNode().connection;
        ResultSet rootResultSet = ((SQLExecutionResult) rootResult).getResultSet();
        IRelationalRootQueryTempTableGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.profiles);
        List<Method> primaryKeyGetters = nodeSpecifics.primaryKeyGetters();
        /* Check if caching is enabled and fetch caches if required */
        List<Pair<String, String>> allInstanceSetImplementations = nodeSpecifics.allInstanceSetImplementations();
        int setIdCount = allInstanceSetImplementations.size();
        RelationalMultiSetExecutionCacheWrapper multiSetCache = new RelationalMultiSetExecutionCacheWrapper(setIdCount);
        boolean cachingEnabledForNode = this.checkForCachingAndPopulateCachingHelpers(allInstanceSetImplementations, nodeSpecifics.supportsCaching(), node.graphFetchTree, sqlExecutionResult, nodeSpecifics::primaryKeyColumns, multiSetCache);
        /* Prepare for reading */
        nodeSpecifics.prepare(rootResultSet, sqlExecutionResult.getDatabaseTimeZone(), ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(databaseConnection));
        boolean isUnion = setIdCount > 1;
        AtomicLong batchIndex = new AtomicLong(0L);
        Spliterator<GraphObjectsBatch> graphObjectsBatchSpliterator = new Spliterators.AbstractSpliterator<GraphObjectsBatch>(Long.MAX_VALUE, Spliterator.ORDERED) {

            @Override
            public boolean tryAdvance(Consumer<? super GraphObjectsBatch> action) {
                /* Ensure all children run in the same connection */
                RelationalStoreExecutionState relationalStoreExecutionState = (RelationalStoreExecutionState) executionState.getStoreExecutionState(StoreType.Relational);
                BlockConnectionContext oldBlockConnectionContext = relationalStoreExecutionState.getBlockConnectionContext();
                boolean oldRetainConnectionFlag = relationalStoreExecutionState.retainConnection();
                relationalStoreExecutionState.setBlockConnectionContext(new BlockConnectionContext());
                relationalStoreExecutionState.setRetainConnection(true);
                long currentBatch = batchIndex.incrementAndGet();
                try (Scope ignored = GlobalTracer.get().buildSpan("graph fetch batch").withTag("storeType", "relational").withTag("batchIndex", currentBatch).withTag("class", ((RootGraphFetchTree) node.graphFetchTree)._class).asChildOf(graphFetchSpan).startActive(true)) {
                    RelationalGraphObjectsBatch relationalGraphObjectsBatch = new RelationalGraphObjectsBatch(currentBatch);
                    List<Object> resultObjects = new ArrayList<>();
                    List<Pair<IGraphInstance<? extends IReferencedObject>, ExecutionCache<GraphFetchCacheKey, Object>>> instancesToDeepFetchAndCache = new ArrayList<>();
                    int objectCount = 0;
                    while ((!rootResultSet.isClosed()) && rootResultSet.next()) {
                        relationalGraphObjectsBatch.incrementRowCount();
                        int setIndex = isUnion ? rootResultSet.getInt(1) : 0;
                        Object cachedObject = RelationalExecutionNodeExecutor.this.checkAndReturnCachedObject(cachingEnabledForNode, setIndex, multiSetCache);
                        boolean shouldDeepFetchOnThisInstance = cachedObject == null;
                        Object object;
                        if (shouldDeepFetchOnThisInstance) {
                            IGraphInstance<? extends IReferencedObject> wrappedObject = nodeSpecifics.nextGraphInstance();
                            instancesToDeepFetchAndCache.add(Tuples.pair(wrappedObject, multiSetCache.setCaches.get(setIndex)));
                            object = wrappedObject.getValue();
                        } else {
                            object = cachedObject;
                        }
                        if (node.checked != null && node.checked) {
                            resultObjects.add(BasicChecked.newChecked(object, null));
                        } else {
                            resultObjects.add(object);
                        }
                        objectCount += 1;
                        if (objectCount >= batchSize) {
                            break;
                        }
                    }
                    relationalGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, resultObjects);
                    if (!instancesToDeepFetchAndCache.isEmpty()) {
                        RealizedRelationalResult realizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.columns);
                        DoubleStrategyHashMap<Object, Object, SQLExecutionResult> rootMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
                        for (Pair<IGraphInstance<? extends IReferencedObject>, ExecutionCache<GraphFetchCacheKey, Object>> instanceAndCache : instancesToDeepFetchAndCache) {
                            IGraphInstance<? extends IReferencedObject> rootGraphInstance = instanceAndCache.getOne();
                            Object rootObject = rootGraphInstance.getValue();
                            rootMap.put(rootObject, rootObject);
                            relationalGraphObjectsBatch.addObjectMemoryUtilization(rootGraphInstance.instanceSize());
                            if (!isLeaf) {
                                RelationalExecutionNodeExecutor.this.addKeyRowToRealizedRelationalResult(rootObject, primaryKeyGetters, realizedRelationalResult);
                            }
                        }
                        /* Execute store local children */
                        if (!isLeaf) {
                            ExecutionState newState = new ExecutionState(executionState);
                            newState.graphObjectsBatch = relationalGraphObjectsBatch;
                            RelationalExecutionNodeExecutor.this.executeTempTableNodeChildren(node, realizedRelationalResult, databaseConnection, sqlExecutionResult.getDatabaseType(), sqlExecutionResult.getDatabaseTimeZone(), rootMap, primaryKeyGetters, newState);
                        }
                    }
                    instancesToDeepFetchAndCache.stream().filter(x -> x.getTwo() != null).forEach(x -> {
                        Object object = x.getOne().getValue();
                        x.getTwo().put(new RelationalGraphFetchUtils.RelationalObjectGraphFetchCacheKey(object, primaryKeyGetters), object);
                    });
                    action.accept(relationalGraphObjectsBatch);
                    return !resultObjects.isEmpty();
                } catch (SQLException | InvocationTargetException | IllegalAccessException e) {
                    throw new RuntimeException(e);
                } finally {
                    relationalStoreExecutionState.getBlockConnectionContext().unlockAllBlockConnections();
                    relationalStoreExecutionState.getBlockConnectionContext().closeAllBlockConnectionsAsync();
                    relationalStoreExecutionState.setBlockConnectionContext(oldBlockConnectionContext);
                    relationalStoreExecutionState.setRetainConnection(oldRetainConnectionFlag);
                }
            }
        };
        Stream<GraphObjectsBatch> graphObjectsBatchStream = StreamSupport.stream(graphObjectsBatchSpliterator, false);
        return new GraphFetchResult(graphObjectsBatchStream, rootResult).withGraphFetchSpan(graphFetchSpan);
    } catch (RuntimeException e) {
        if (rootResult != null) {
            rootResult.close();
        }
        if (graphFetchSpan != null) {
            graphFetchSpan.finish();
        }
        throw e;
    } catch (Exception e) {
        if (rootResult != null) {
            rootResult.close();
        }
        if (graphFetchSpan != null) {
            graphFetchSpan.finish();
        }
        throw new RuntimeException(e);
    }
}
Also used : org.finos.legend.engine.plan.dependencies.store.relational.graphFetch(org.finos.legend.engine.plan.dependencies.store.relational.graphFetch) DatabaseManager(org.finos.legend.engine.plan.execution.stores.relational.connection.driver.DatabaseManager) DefaultExecutionNodeContext(org.finos.legend.engine.plan.execution.nodes.helpers.platform.DefaultExecutionNodeContext) Connection(java.sql.Connection) RelationalDatabaseCommandsVisitorBuilder(org.finos.legend.engine.plan.execution.stores.relational.RelationalDatabaseCommandsVisitorBuilder) BlockConnection(org.finos.legend.engine.plan.execution.stores.relational.blockConnection.BlockConnection) GraphFetchCacheKey(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheKey) ClassResultType(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.result.ClassResultType) MutableList(org.eclipse.collections.api.list.MutableList) InMemoryCrossStoreGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryCrossStoreGraphFetchExecutionNode) ClassUtils(org.apache.commons.lang3.ClassUtils) FunctionHelper(org.finos.legend.engine.plan.execution.stores.relational.result.FunctionHelper) GraphFetchTree(org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.graph.GraphFetchTree) ExecutionNodeExecutor(org.finos.legend.engine.plan.execution.nodes.ExecutionNodeExecutor) ResultSet(java.sql.ResultSet) StoreStreamReadingExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.StoreStreamReadingExecutionNode) Tuples(org.eclipse.collections.impl.tuple.Tuples) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) AggregationAwareActivity(org.finos.legend.engine.plan.execution.stores.relational.activity.AggregationAwareActivity) ResultColumn(org.finos.legend.engine.plan.execution.stores.relational.result.ResultColumn) Method(java.lang.reflect.Method) RelationalExecutor(org.finos.legend.engine.plan.execution.stores.relational.RelationalExecutor) IRelationalResult(org.finos.legend.engine.plan.dependencies.store.relational.IRelationalResult) RelationalGraphObjectsBatch(org.finos.legend.engine.plan.execution.stores.relational.result.graphFetch.RelationalGraphObjectsBatch) GraphExecutionState(org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState) Iterate(org.eclipse.collections.impl.utility.Iterate) TempTableStreamingResult(org.finos.legend.engine.plan.execution.stores.relational.result.TempTableStreamingResult) Collectors(java.util.stream.Collectors) InvocationTargetException(java.lang.reflect.InvocationTargetException) org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch) InMemoryRootGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryRootGraphFetchExecutionNode) DatabaseConnection(org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection) Stream(java.util.stream.Stream) RelationalDatabaseCommands(org.finos.legend.engine.plan.execution.stores.relational.connection.driver.commands.RelationalDatabaseCommands) ObjectMapperFactory(org.finos.legend.engine.shared.core.ObjectMapperFactory) PreparedTempTableResult(org.finos.legend.engine.plan.execution.stores.relational.result.PreparedTempTableResult) Span(io.opentracing.Span) Scope(io.opentracing.Scope) SQLExecutionResult(org.finos.legend.engine.plan.execution.stores.relational.result.SQLExecutionResult) StoreType(org.finos.legend.engine.plan.execution.stores.StoreType) RootGraphFetchTree(org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.graph.RootGraphFetchTree) java.util(java.util) CommonProfile(org.pac4j.core.profile.CommonProfile) IGraphInstance(org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance) DoubleStrategyHashMap(org.finos.legend.engine.shared.core.collectionsExtensions.DoubleStrategyHashMap) ExecutionState(org.finos.legend.engine.plan.execution.nodes.state.ExecutionState) org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes) Lists(org.eclipse.collections.api.factory.Lists) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) ResultInterpreterExtension(org.finos.legend.engine.plan.execution.stores.relational.result.ResultInterpreterExtension) ExecutionNodeResultHelper(org.finos.legend.engine.plan.execution.nodes.helpers.ExecutionNodeResultHelper) PropertyGraphFetchTree(org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.graph.PropertyGraphFetchTree) Function(java.util.function.Function) Supplier(java.util.function.Supplier) FastList(org.eclipse.collections.impl.list.mutable.FastList) ExecutionCache(org.finos.legend.engine.plan.execution.cache.ExecutionCache) SQLException(java.sql.SQLException) RelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RelationalResult) Function2(org.eclipse.collections.api.block.function.Function2) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) JavaHelper(org.finos.legend.engine.plan.execution.nodes.helpers.platform.JavaHelper) JsonIgnore(com.fasterxml.jackson.annotation.JsonIgnore) GraphFetchCacheByEqualityKeys(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheByEqualityKeys) StreamSupport(java.util.stream.StreamSupport) Pair(org.eclipse.collections.api.tuple.Pair) BasicChecked(org.finos.legend.engine.plan.dependencies.domain.dataQuality.BasicChecked) IReferencedObject(org.finos.legend.engine.plan.dependencies.store.shared.IReferencedObject) RealizedRelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RealizedRelationalResult) Result(org.finos.legend.engine.plan.execution.result.Result) ExecutionNodeJavaPlatformHelper(org.finos.legend.engine.plan.execution.nodes.helpers.platform.ExecutionNodeJavaPlatformHelper) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) BlockConnectionContext(org.finos.legend.engine.plan.execution.stores.relational.blockConnection.BlockConnectionContext) GlobalTracer(io.opentracing.util.GlobalTracer) ClassBuilder(org.finos.legend.engine.plan.execution.result.builder._class.ClassBuilder) GraphObjectsBatch(org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch) InMemoryPropertyGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryPropertyGraphFetchExecutionNode) Consumer(java.util.function.Consumer) AtomicLong(java.util.concurrent.atomic.AtomicLong) IRelationalClassInstantiationNodeExecutor(org.finos.legend.engine.plan.dependencies.store.relational.classResult.IRelationalClassInstantiationNodeExecutor) IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics(org.finos.legend.engine.plan.dependencies.store.relational.IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics) GraphExecutionState(org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState) ExecutionState(org.finos.legend.engine.plan.execution.nodes.state.ExecutionState) IGraphInstance(org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance) ExecutionNodeExecutor(org.finos.legend.engine.plan.execution.nodes.ExecutionNodeExecutor) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) IRelationalResult(org.finos.legend.engine.plan.dependencies.store.relational.IRelationalResult) TempTableStreamingResult(org.finos.legend.engine.plan.execution.stores.relational.result.TempTableStreamingResult) PreparedTempTableResult(org.finos.legend.engine.plan.execution.stores.relational.result.PreparedTempTableResult) SQLExecutionResult(org.finos.legend.engine.plan.execution.stores.relational.result.SQLExecutionResult) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) RelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RelationalResult) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) RealizedRelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RealizedRelationalResult) Result(org.finos.legend.engine.plan.execution.result.Result) Consumer(java.util.function.Consumer) ResultSet(java.sql.ResultSet) DoubleStrategyHashMap(org.finos.legend.engine.shared.core.collectionsExtensions.DoubleStrategyHashMap) Method(java.lang.reflect.Method) InvocationTargetException(java.lang.reflect.InvocationTargetException) IReferencedObject(org.finos.legend.engine.plan.dependencies.store.shared.IReferencedObject) SQLExecutionResult(org.finos.legend.engine.plan.execution.stores.relational.result.SQLExecutionResult) RelationalGraphObjectsBatch(org.finos.legend.engine.plan.execution.stores.relational.result.graphFetch.RelationalGraphObjectsBatch) SQLException(java.sql.SQLException) Span(io.opentracing.Span) IReferencedObject(org.finos.legend.engine.plan.dependencies.store.shared.IReferencedObject) BlockConnectionContext(org.finos.legend.engine.plan.execution.stores.relational.blockConnection.BlockConnectionContext) DatabaseConnection(org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection) Pair(org.eclipse.collections.api.tuple.Pair) RealizedRelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RealizedRelationalResult) ExecutionCache(org.finos.legend.engine.plan.execution.cache.ExecutionCache) RelationalGraphObjectsBatch(org.finos.legend.engine.plan.execution.stores.relational.result.graphFetch.RelationalGraphObjectsBatch) GraphObjectsBatch(org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) InvocationTargetException(java.lang.reflect.InvocationTargetException) SQLException(java.sql.SQLException) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) AtomicLong(java.util.concurrent.atomic.AtomicLong) GraphFetchCacheKey(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheKey) Scope(io.opentracing.Scope)

Aggregations

Span (io.opentracing.Span)5 AtomicLong (java.util.concurrent.atomic.AtomicLong)5 Consumer (java.util.function.Consumer)5 MutableList (org.eclipse.collections.api.list.MutableList)5 ExecutionState (org.finos.legend.engine.plan.execution.nodes.state.ExecutionState)5 ConstantResult (org.finos.legend.engine.plan.execution.result.ConstantResult)5 GlobalTracer (io.opentracing.util.GlobalTracer)4 Collectors (java.util.stream.Collectors)4 Stream (java.util.stream.Stream)4 StreamSupport (java.util.stream.StreamSupport)4 IChecked (org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked)4 IGraphInstance (org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance)4 ExecutionNodeExecutor (org.finos.legend.engine.plan.execution.nodes.ExecutionNodeExecutor)4 ExecutionNodeJavaPlatformHelper (org.finos.legend.engine.plan.execution.nodes.helpers.platform.ExecutionNodeJavaPlatformHelper)4 JavaHelper (org.finos.legend.engine.plan.execution.nodes.helpers.platform.JavaHelper)4 Result (org.finos.legend.engine.plan.execution.result.Result)4 GraphFetchResult (org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult)4 GraphObjectsBatch (org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch)4 StreamingObjectResult (org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult)4 ClassResultType (org.finos.legend.engine.protocol.pure.v1.model.executionPlan.result.ClassResultType)4