Search in sources :

Example 1 with GraphExecutionState

use of org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState in project legend-engine by finos.

the class RelationalExecutionNodeExecutor method visit.

@Override
public Result visit(ExecutionNode executionNode) {
    if (executionNode instanceof RelationalBlockExecutionNode) {
        RelationalBlockExecutionNode relationalBlockExecutionNode = (RelationalBlockExecutionNode) executionNode;
        ExecutionState connectionAwareState = new ExecutionState(this.executionState);
        ((RelationalStoreExecutionState) connectionAwareState.getStoreExecutionState(StoreType.Relational)).setRetainConnection(true);
        try {
            Result res = new ExecutionNodeExecutor(this.profiles, connectionAwareState).visit((SequenceExecutionNode) relationalBlockExecutionNode);
            ((RelationalStoreExecutionState) connectionAwareState.getStoreExecutionState(StoreType.Relational)).getBlockConnectionContext().unlockAllBlockConnections();
            return res;
        } catch (Exception e) {
            ((RelationalStoreExecutionState) connectionAwareState.getStoreExecutionState(StoreType.Relational)).getBlockConnectionContext().unlockAllBlockConnections();
            ((RelationalStoreExecutionState) connectionAwareState.getStoreExecutionState(StoreType.Relational)).getBlockConnectionContext().closeAllBlockConnections();
            throw e;
        }
    } else if (executionNode instanceof CreateAndPopulateTempTableExecutionNode) {
        CreateAndPopulateTempTableExecutionNode createAndPopulateTempTableExecutionNode = (CreateAndPopulateTempTableExecutionNode) executionNode;
        Stream<Result> results = createAndPopulateTempTableExecutionNode.inputVarNames.stream().map(this.executionState::getResult);
        Stream<?> inputStream = results.flatMap(result -> {
            if (result instanceof ConstantResult) {
                Object value = ((ConstantResult) result).getValue();
                if (value instanceof Map && ((Map<?, ?>) value).get("values") instanceof List) {
                    return ((List<?>) ((Map<?, ?>) value).get("values")).stream().map(val -> ((List<?>) ((Map<?, ?>) val).get("values")).get(0));
                }
                if (value instanceof List) {
                    return ((List<?>) value).stream();
                }
                if (ClassUtils.isPrimitiveOrWrapper(value.getClass()) || (value instanceof String)) {
                    return Stream.of(value);
                }
                if (value instanceof Stream) {
                    return (Stream<?>) value;
                }
                throw new IllegalArgumentException("Result passed into CreateAndPopulateTempTableExecutionNode should be a stream");
            }
            if (result instanceof StreamingObjectResult) {
                return ((StreamingObjectResult<?>) result).getObjectStream();
            }
            throw new IllegalArgumentException("Unexpected Result Type : " + result.getClass().getName());
        });
        if (!(createAndPopulateTempTableExecutionNode.implementation instanceof JavaPlatformImplementation)) {
            throw new RuntimeException("Only Java implementations are currently supported, found: " + createAndPopulateTempTableExecutionNode.implementation);
        }
        JavaPlatformImplementation javaPlatformImpl = (JavaPlatformImplementation) createAndPopulateTempTableExecutionNode.implementation;
        String executionClassName = JavaHelper.getExecutionClassFullName(javaPlatformImpl);
        Class<?> clazz = ExecutionNodeJavaPlatformHelper.getClassToExecute(createAndPopulateTempTableExecutionNode, executionClassName, this.executionState, this.profiles);
        if (Arrays.asList(clazz.getInterfaces()).contains(IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics.class)) {
            try {
                IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics nodeSpecifics = (IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics) clazz.newInstance();
                createAndPopulateTempTableExecutionNode.tempTableColumnMetaData.forEach(t -> t.identifierForGetter = nodeSpecifics.getGetterNameForProperty(t.identifierForGetter));
            } catch (InstantiationException | IllegalAccessException e) {
                throw new RuntimeException(e);
            }
        } else {
            // TODO Remove once platform version supports above and existing plans mitigated
            String executionMethodName = JavaHelper.getExecutionMethodName(javaPlatformImpl);
            createAndPopulateTempTableExecutionNode.tempTableColumnMetaData.forEach(t -> t.identifierForGetter = ExecutionNodeJavaPlatformHelper.executeStaticJavaMethod(createAndPopulateTempTableExecutionNode, executionClassName, executionMethodName, Collections.singletonList(Result.class), Collections.singletonList(new ConstantResult(t.identifierForGetter)), this.executionState, this.profiles));
        }
        RelationalDatabaseCommands databaseCommands = DatabaseManager.fromString(createAndPopulateTempTableExecutionNode.connection.type.name()).relationalDatabaseSupport();
        try (Connection connectionManagerConnection = this.getConnection(createAndPopulateTempTableExecutionNode, databaseCommands, this.profiles, this.executionState)) {
            TempTableStreamingResult tempTableStreamingResult = new TempTableStreamingResult(inputStream, createAndPopulateTempTableExecutionNode);
            String databaseTimeZone = createAndPopulateTempTableExecutionNode.connection.timeZone == null ? RelationalExecutor.DEFAULT_DB_TIME_ZONE : createAndPopulateTempTableExecutionNode.connection.timeZone;
            databaseCommands.accept(RelationalDatabaseCommandsVisitorBuilder.getStreamResultToTempTableVisitor(((RelationalStoreExecutionState) this.executionState.getStoreExecutionState(StoreType.Relational)).getRelationalExecutor().getRelationalExecutionConfiguration(), connectionManagerConnection, tempTableStreamingResult, createAndPopulateTempTableExecutionNode.tempTableName, databaseTimeZone));
        } catch (SQLException e) {
            throw new RuntimeException(e);
        }
        return new ConstantResult("success");
    } else if (executionNode instanceof RelationalExecutionNode) {
        RelationalExecutionNode relationalExecutionNode = (RelationalExecutionNode) executionNode;
        Span topSpan = GlobalTracer.get().activeSpan();
        this.executionState.topSpan = topSpan;
        try (Scope scope = GlobalTracer.get().buildSpan("Relational DB Execution").startActive(true)) {
            scope.span().setTag("databaseType", relationalExecutionNode.getDatabaseTypeName());
            scope.span().setTag("sql", relationalExecutionNode.sqlQuery());
            Result result = ((RelationalStoreExecutionState) executionState.getStoreExecutionState(StoreType.Relational)).getRelationalExecutor().execute(relationalExecutionNode, this.profiles, this.executionState);
            if (result instanceof RelationalResult) {
                scope.span().setTag("executedSql", ((RelationalResult) result).executedSQl);
            }
            if (relationalExecutionNode.implementation != null && !(ExecutionNodeResultHelper.isResultSizeRangeSet(relationalExecutionNode) && ExecutionNodeResultHelper.isSingleRecordResult(relationalExecutionNode))) {
                return executeImplementation(relationalExecutionNode, result, this.executionState, this.profiles);
            }
            return result;
        }
    } else if (executionNode instanceof SQLExecutionNode) {
        SQLExecutionNode SQLExecutionNode = (SQLExecutionNode) executionNode;
        this.executionState.topSpan = GlobalTracer.get().activeSpan();
        try (Scope scope = GlobalTracer.get().buildSpan("Relational DB Execution").startActive(true)) {
            scope.span().setTag("databaseType", SQLExecutionNode.getDatabaseTypeName());
            scope.span().setTag("sql", SQLExecutionNode.sqlQuery());
            Result result = ((RelationalStoreExecutionState) executionState.getStoreExecutionState(StoreType.Relational)).getRelationalExecutor().execute(SQLExecutionNode, profiles, executionState);
            if (result instanceof SQLExecutionResult) {
                scope.span().setTag("executedSql", ((SQLExecutionResult) result).getExecutedSql());
            }
            return result;
        }
    } else if (executionNode instanceof RelationalTdsInstantiationExecutionNode) {
        RelationalTdsInstantiationExecutionNode relationalTdsInstantiationExecutionNode = (RelationalTdsInstantiationExecutionNode) executionNode;
        SQLExecutionResult sqlExecutionResult = null;
        try {
            sqlExecutionResult = (SQLExecutionResult) this.visit((SQLExecutionNode) relationalTdsInstantiationExecutionNode.executionNodes.get(0));
            RelationalResult relationalTdsResult = new RelationalResult(sqlExecutionResult, relationalTdsInstantiationExecutionNode);
            if (this.executionState.inAllocation) {
                if (!this.executionState.transformAllocation) {
                    return relationalTdsResult;
                }
                RealizedRelationalResult realizedRelationalResult = (RealizedRelationalResult) relationalTdsResult.realizeInMemory();
                List<Map<String, Object>> rowValueMaps = realizedRelationalResult.getRowValueMaps(false);
                Result res = RelationalExecutor.evaluateAdditionalExtractors(this.resultInterpreterExtensions, this.executionState, rowValueMaps);
                if (res != null) {
                    return res;
                } else {
                    return new ConstantResult(rowValueMaps);
                }
            }
            return relationalTdsResult;
        } catch (Exception e) {
            if (sqlExecutionResult != null) {
                sqlExecutionResult.close();
            }
            throw e;
        }
    } else if (executionNode instanceof RelationalClassInstantiationExecutionNode) {
        RelationalClassInstantiationExecutionNode node = (RelationalClassInstantiationExecutionNode) executionNode;
        SQLExecutionResult sqlExecutionResult = null;
        try {
            SQLExecutionNode innerNode = (SQLExecutionNode) node.executionNodes.get(0);
            sqlExecutionResult = (SQLExecutionResult) this.visit(innerNode);
            RelationalResult relationalResult = new RelationalResult(sqlExecutionResult, node);
            boolean realizeAsConstant = this.executionState.inAllocation && ExecutionNodeResultHelper.isResultSizeRangeSet(node) && ExecutionNodeResultHelper.isSingleRecordResult(node);
            if (realizeAsConstant) {
                RealizedRelationalResult realizedRelationalResult = (RealizedRelationalResult) relationalResult.realizeInMemory();
                List<Map<String, Object>> rowValueMaps = realizedRelationalResult.getRowValueMaps(false);
                if (rowValueMaps.size() == 1) {
                    return new ConstantResult(rowValueMaps.get(0));
                } else {
                    return new ConstantResult(rowValueMaps);
                }
            }
            return this.getStreamingObjectResultFromRelationalResult(node, relationalResult, innerNode.connection);
        } catch (Exception e) {
            if (sqlExecutionResult != null) {
                sqlExecutionResult.close();
            }
            throw (e instanceof RuntimeException) ? (RuntimeException) e : new RuntimeException(e);
        }
    } else if (executionNode instanceof RelationalRelationDataInstantiationExecutionNode) {
        RelationalRelationDataInstantiationExecutionNode node = (RelationalRelationDataInstantiationExecutionNode) executionNode;
        SQLExecutionResult sqlExecutionResult = null;
        try {
            sqlExecutionResult = (SQLExecutionResult) this.visit((SQLExecutionNode) node.executionNodes.get(0));
            return new RelationalResult(sqlExecutionResult, node);
        } catch (Exception e) {
            if (sqlExecutionResult != null) {
                sqlExecutionResult.close();
            }
            throw e;
        }
    } else if (executionNode instanceof RelationalDataTypeInstantiationExecutionNode) {
        RelationalDataTypeInstantiationExecutionNode node = (RelationalDataTypeInstantiationExecutionNode) executionNode;
        SQLExecutionResult sqlExecutionResult = null;
        try {
            sqlExecutionResult = (SQLExecutionResult) this.visit((SQLExecutionNode) node.executionNodes.get(0));
            RelationalResult relationalPrimitiveResult = new RelationalResult(sqlExecutionResult, node);
            if (this.executionState.inAllocation) {
                if ((ExecutionNodeResultHelper.isResultSizeRangeSet(node) && !ExecutionNodeResultHelper.isSingleRecordResult(node)) && !this.executionState.transformAllocation) {
                    return relationalPrimitiveResult;
                }
                if (relationalPrimitiveResult.getResultSet().next()) {
                    List<org.eclipse.collections.api.block.function.Function<Object, Object>> transformers = relationalPrimitiveResult.getTransformers();
                    Object convertedValue = transformers.get(0).valueOf(relationalPrimitiveResult.getResultSet().getObject(1));
                    return new ConstantResult(convertedValue);
                } else {
                    throw new RuntimeException("Result set is empty for allocation node");
                }
            }
            return relationalPrimitiveResult;
        } catch (Exception e) {
            if (sqlExecutionResult != null) {
                sqlExecutionResult.close();
            }
            throw (e instanceof RuntimeException) ? (RuntimeException) e : new RuntimeException(e);
        }
    } else if (executionNode instanceof RelationalRootQueryTempTableGraphFetchExecutionNode) {
        return this.executeRelationalRootQueryTempTableGraphFetchExecutionNode((RelationalRootQueryTempTableGraphFetchExecutionNode) executionNode);
    } else if (executionNode instanceof RelationalCrossRootQueryTempTableGraphFetchExecutionNode) {
        return this.executeRelationalCrossRootQueryTempTableGraphFetchExecutionNode((RelationalCrossRootQueryTempTableGraphFetchExecutionNode) executionNode);
    } else if (executionNode instanceof RelationalPrimitiveQueryGraphFetchExecutionNode) {
        return this.executeRelationalPrimitiveQueryGraphFetchExecutionNode((RelationalPrimitiveQueryGraphFetchExecutionNode) executionNode);
    } else if (executionNode instanceof RelationalClassQueryTempTableGraphFetchExecutionNode) {
        return this.executeRelationalClassQueryTempTableGraphFetchExecutionNode((RelationalClassQueryTempTableGraphFetchExecutionNode) executionNode);
    } else if (executionNode instanceof RelationalRootGraphFetchExecutionNode) {
        RelationalRootGraphFetchExecutionNode node = (RelationalRootGraphFetchExecutionNode) executionNode;
        /* Fetch info from execution state */
        GraphExecutionState graphExecutionState = (GraphExecutionState) executionState;
        int batchSize = graphExecutionState.getBatchSize();
        SQLExecutionResult rootResult = (SQLExecutionResult) graphExecutionState.getRootResult();
        ResultSet rootResultSet = rootResult.getResultSet();
        /* Ensure all children run in the same connection */
        RelationalStoreExecutionState relationalStoreExecutionState = (RelationalStoreExecutionState) graphExecutionState.getStoreExecutionState(StoreType.Relational);
        BlockConnectionContext oldBlockConnectionContext = relationalStoreExecutionState.getBlockConnectionContext();
        boolean oldRetainConnectionFlag = relationalStoreExecutionState.retainConnection();
        relationalStoreExecutionState.setBlockConnectionContext(new BlockConnectionContext());
        relationalStoreExecutionState.setRetainConnection(true);
        try (Scope ignored1 = GlobalTracer.get().buildSpan("Graph Query Relational: Execute Relational Root").startActive(true)) {
            String databaseTimeZone = rootResult.getDatabaseTimeZone();
            String databaseConnectionString = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(rootResult.getSQLExecutionNode().connection);
            /* Get Java executor */
            Class<?> executeClass = this.getExecuteClass(node);
            if (Arrays.asList(executeClass.getInterfaces()).contains(IRelationalRootGraphNodeExecutor.class)) {
                IRelationalRootGraphNodeExecutor executor = (IRelationalRootGraphNodeExecutor) executeClass.getConstructor().newInstance();
                List<Method> primaryKeyGetters = executor.primaryKeyGetters();
                int primaryKeyCount = primaryKeyGetters.size();
                /* Check if caching is enabled and fetch the cache if required */
                boolean cachingEnabledForNode = false;
                ExecutionCache<GraphFetchCacheKey, Object> graphCache = null;
                RelationalGraphFetchUtils.RelationalSQLResultGraphFetchCacheKey rootResultCacheKey = null;
                if ((this.executionState.graphFetchCaches != null) && executor.supportsCaching()) {
                    GraphFetchCacheByEqualityKeys graphFetchCacheByEqualityKeys = RelationalGraphFetchUtils.findCacheByEqualityKeys(node.graphFetchTree, executor.getMappingId(rootResultSet, databaseTimeZone, databaseConnectionString), executor.getInstanceSetId(rootResultSet, databaseTimeZone, databaseConnectionString), this.executionState.graphFetchCaches);
                    if (graphFetchCacheByEqualityKeys != null) {
                        List<String> parentSQLKeyColumns = executor.primaryKeyColumns();
                        List<Integer> parentPrimaryKeyIndices = FastList.newList();
                        for (String pkCol : parentSQLKeyColumns) {
                            parentPrimaryKeyIndices.add(rootResultSet.findColumn(pkCol));
                        }
                        cachingEnabledForNode = true;
                        graphCache = graphFetchCacheByEqualityKeys.getExecutionCache();
                        rootResultCacheKey = new RelationalGraphFetchUtils.RelationalSQLResultGraphFetchCacheKey(rootResult, parentPrimaryKeyIndices);
                    }
                }
                /* Get the next batch of root records */
                List<Object> resultObjectsBatch = new ArrayList<>();
                List<org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?>> instancesToDeepFetch = new ArrayList<>();
                int objectCount = 0;
                try (Scope ignored2 = GlobalTracer.get().buildSpan("Graph Query Relational: Read Next Batch").startActive(true)) {
                    while (rootResultSet.next()) {
                        graphExecutionState.incrementRowCount();
                        boolean shouldDeepFetchOnThisInstance = true;
                        if (cachingEnabledForNode) {
                            Object cachedObject = graphCache.getIfPresent(rootResultCacheKey);
                            if (cachedObject != null) {
                                resultObjectsBatch.add(executor.deepCopy(cachedObject));
                                shouldDeepFetchOnThisInstance = false;
                            }
                        }
                        if (shouldDeepFetchOnThisInstance) {
                            org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?> wrappedObject = executor.getObjectFromResultSet(rootResultSet, databaseTimeZone, databaseConnectionString);
                            instancesToDeepFetch.add(wrappedObject);
                            resultObjectsBatch.add(wrappedObject.getValue());
                        }
                        objectCount += 1;
                        if (objectCount >= batchSize) {
                            break;
                        }
                    }
                }
                if (!instancesToDeepFetch.isEmpty()) {
                    boolean childrenExist = node.children != null && !node.children.isEmpty();
                    String tempTableName = node.tempTableName;
                    RealizedRelationalResult realizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.columns);
                    /* Create and populate double strategy map with key being object with its PK getters */
                    DoubleStrategyHashMap<Object, Object, SQLExecutionResult> rootMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
                    for (org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?> rootGraphInstance : instancesToDeepFetch) {
                        Object rootObject = rootGraphInstance.getValue();
                        rootMap.put(rootObject, rootObject);
                        graphExecutionState.addObjectMemoryUtilization(rootGraphInstance.instanceSize());
                        if (childrenExist) {
                            this.addKeyRowToRealizedRelationalResult(rootObject, primaryKeyGetters, realizedRelationalResult);
                        }
                    }
                    /* Execute store local children */
                    if (childrenExist) {
                        this.executeRelationalChildren(node, tempTableName, realizedRelationalResult, rootResult.getSQLExecutionNode().connection, rootResult.getDatabaseType(), databaseTimeZone, rootMap, primaryKeyGetters);
                    }
                }
                graphExecutionState.setObjectsForNodeIndex(node.nodeIndex, resultObjectsBatch);
                if (cachingEnabledForNode) {
                    for (org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?> deepFetchedInstance : instancesToDeepFetch) {
                        Object objectClone = executor.deepCopy(deepFetchedInstance.getValue());
                        graphCache.put(new RelationalGraphFetchUtils.RelationalObjectGraphFetchCacheKey(objectClone, primaryKeyGetters), objectClone);
                    }
                }
                return new ConstantResult(resultObjectsBatch);
            } else {
                throw new RuntimeException("Unknown execute class " + executeClass.getCanonicalName());
            }
        } catch (RuntimeException e) {
            throw e;
        } catch (Exception e) {
            throw new RuntimeException(e);
        } finally {
            relationalStoreExecutionState.getBlockConnectionContext().unlockAllBlockConnections();
            relationalStoreExecutionState.getBlockConnectionContext().closeAllBlockConnectionsAsync();
            relationalStoreExecutionState.setBlockConnectionContext(oldBlockConnectionContext);
            relationalStoreExecutionState.setRetainConnection(oldRetainConnectionFlag);
        }
    } else if (executionNode instanceof RelationalCrossRootGraphFetchExecutionNode) {
        RelationalCrossRootGraphFetchExecutionNode node = (RelationalCrossRootGraphFetchExecutionNode) executionNode;
        GraphExecutionState graphExecutionState = (GraphExecutionState) executionState;
        List<?> parentObjects = graphExecutionState.getObjectsToGraphFetch();
        List<Object> childObjects = FastList.newList();
        graphExecutionState.setObjectsForNodeIndex(node.nodeIndex, childObjects);
        RelationalStoreExecutionState relationalStoreExecutionState = (RelationalStoreExecutionState) graphExecutionState.getStoreExecutionState(StoreType.Relational);
        BlockConnectionContext oldBlockConnectionContext = relationalStoreExecutionState.getBlockConnectionContext();
        boolean oldRetainConnectionFlag = relationalStoreExecutionState.retainConnection();
        relationalStoreExecutionState.setBlockConnectionContext(new BlockConnectionContext());
        relationalStoreExecutionState.setRetainConnection(true);
        SQLExecutionResult childResult = null;
        try (Scope ignored1 = GlobalTracer.get().buildSpan("Graph Query Relational: Execute Relational Cross Root").startActive(true)) {
            /* Get Java executor */
            Class<?> executeClass = this.getExecuteClass(node);
            if (Arrays.asList(executeClass.getInterfaces()).contains(IRelationalCrossRootGraphNodeExecutor.class)) {
                IRelationalCrossRootGraphNodeExecutor executor = (IRelationalCrossRootGraphNodeExecutor) executeClass.getConstructor().newInstance();
                if (!parentObjects.isEmpty()) {
                    String parentTempTableName = node.parentTempTableName;
                    RealizedRelationalResult parentRealizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.parentTempTableColumns);
                    List<Method> crossKeyGetters = executor.parentCrossKeyGetters();
                    int parentKeyCount = crossKeyGetters.size();
                    for (Object parentObject : parentObjects) {
                        this.addKeyRowToRealizedRelationalResult(parentObject, crossKeyGetters, parentRealizedRelationalResult);
                    }
                    graphExecutionState.addResult(parentTempTableName, parentRealizedRelationalResult);
                    /* Execute relational node corresponding to the cross root */
                    childResult = (SQLExecutionResult) node.relationalNode.accept(new ExecutionNodeExecutor(this.profiles, graphExecutionState));
                    ResultSet childResultSet = childResult.getResultSet();
                    boolean childrenExist = node.children != null && !node.children.isEmpty();
                    String tempTableName = childrenExist ? node.tempTableName : null;
                    RealizedRelationalResult realizedRelationalResult = childrenExist ? RealizedRelationalResult.emptyRealizedRelationalResult(node.columns) : null;
                    DatabaseConnection databaseConnection = childResult.getSQLExecutionNode().connection;
                    String databaseType = childResult.getDatabaseType();
                    String databaseTimeZone = childResult.getDatabaseTimeZone();
                    List<String> parentSQLKeyColumns = executor.parentSQLColumnsInResultSet(childResult.getResultColumns().stream().map(ResultColumn::getNonQuotedLabel).collect(Collectors.toList()));
                    List<Integer> parentCrossKeyIndices = FastList.newList();
                    for (String pkCol : parentSQLKeyColumns) {
                        parentCrossKeyIndices.add(childResultSet.findColumn(pkCol));
                    }
                    DoubleStrategyHashMap<Object, List<Object>, SQLExecutionResult> parentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategy(crossKeyGetters, parentCrossKeyIndices));
                    for (Object parentObject : parentObjects) {
                        List<Object> mapped = parentMap.get(parentObject);
                        if (mapped == null) {
                            parentMap.put(parentObject, FastList.newListWith(parentObject));
                        } else {
                            mapped.add(parentObject);
                        }
                    }
                    List<Method> primaryKeyGetters = executor.primaryKeyGetters();
                    final int primaryKeyCount = primaryKeyGetters.size();
                    DoubleStrategyHashMap<Object, Object, SQLExecutionResult> currentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
                    String databaseConnectionString = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(childResult.getSQLExecutionNode().connection);
                    while (childResultSet.next()) {
                        graphExecutionState.incrementRowCount();
                        List<Object> parents = parentMap.getWithSecondKey(childResult);
                        if (parents == null) {
                            throw new RuntimeException("No parent");
                        }
                        org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?> childGraphInstance = executor.getObjectFromResultSet(childResultSet, childResult.getDatabaseTimeZone(), databaseConnectionString);
                        Object child = childGraphInstance.getValue();
                        Object mapObject = currentMap.putIfAbsent(child, child);
                        if (mapObject == null) {
                            mapObject = child;
                            childObjects.add(mapObject);
                            graphExecutionState.addObjectMemoryUtilization(childGraphInstance.instanceSize());
                            if (childrenExist) {
                                this.addKeyRowToRealizedRelationalResult(child, primaryKeyGetters, realizedRelationalResult);
                            }
                        }
                        for (Object parent : parents) {
                            executor.addChildToParent(parent, mapObject, DefaultExecutionNodeContext.factory().create(graphExecutionState, null));
                        }
                    }
                    childResult.close();
                    childResult = null;
                    if (childrenExist) {
                        this.executeRelationalChildren(node, tempTableName, realizedRelationalResult, databaseConnection, databaseType, databaseTimeZone, currentMap, primaryKeyGetters);
                    }
                }
            } else {
                throw new RuntimeException("Unknown execute class " + executeClass.getCanonicalName());
            }
        } catch (RuntimeException e) {
            throw e;
        } catch (Exception e) {
            throw new RuntimeException(e);
        } finally {
            if (childResult != null) {
                childResult.close();
            }
            relationalStoreExecutionState.getBlockConnectionContext().unlockAllBlockConnections();
            relationalStoreExecutionState.getBlockConnectionContext().closeAllBlockConnectionsAsync();
            relationalStoreExecutionState.setBlockConnectionContext(oldBlockConnectionContext);
            relationalStoreExecutionState.setRetainConnection(oldRetainConnectionFlag);
        }
        return new ConstantResult(childObjects);
    }
    throw new RuntimeException("Not implemented!");
}
Also used : org.finos.legend.engine.plan.dependencies.store.relational.graphFetch(org.finos.legend.engine.plan.dependencies.store.relational.graphFetch) DatabaseManager(org.finos.legend.engine.plan.execution.stores.relational.connection.driver.DatabaseManager) DefaultExecutionNodeContext(org.finos.legend.engine.plan.execution.nodes.helpers.platform.DefaultExecutionNodeContext) Connection(java.sql.Connection) RelationalDatabaseCommandsVisitorBuilder(org.finos.legend.engine.plan.execution.stores.relational.RelationalDatabaseCommandsVisitorBuilder) BlockConnection(org.finos.legend.engine.plan.execution.stores.relational.blockConnection.BlockConnection) GraphFetchCacheKey(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheKey) ClassResultType(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.result.ClassResultType) MutableList(org.eclipse.collections.api.list.MutableList) InMemoryCrossStoreGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryCrossStoreGraphFetchExecutionNode) ClassUtils(org.apache.commons.lang3.ClassUtils) FunctionHelper(org.finos.legend.engine.plan.execution.stores.relational.result.FunctionHelper) GraphFetchTree(org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.graph.GraphFetchTree) ExecutionNodeExecutor(org.finos.legend.engine.plan.execution.nodes.ExecutionNodeExecutor) ResultSet(java.sql.ResultSet) StoreStreamReadingExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.StoreStreamReadingExecutionNode) Tuples(org.eclipse.collections.impl.tuple.Tuples) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) AggregationAwareActivity(org.finos.legend.engine.plan.execution.stores.relational.activity.AggregationAwareActivity) ResultColumn(org.finos.legend.engine.plan.execution.stores.relational.result.ResultColumn) Method(java.lang.reflect.Method) RelationalExecutor(org.finos.legend.engine.plan.execution.stores.relational.RelationalExecutor) IRelationalResult(org.finos.legend.engine.plan.dependencies.store.relational.IRelationalResult) RelationalGraphObjectsBatch(org.finos.legend.engine.plan.execution.stores.relational.result.graphFetch.RelationalGraphObjectsBatch) GraphExecutionState(org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState) Iterate(org.eclipse.collections.impl.utility.Iterate) TempTableStreamingResult(org.finos.legend.engine.plan.execution.stores.relational.result.TempTableStreamingResult) Collectors(java.util.stream.Collectors) InvocationTargetException(java.lang.reflect.InvocationTargetException) org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch) InMemoryRootGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryRootGraphFetchExecutionNode) DatabaseConnection(org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection) Stream(java.util.stream.Stream) RelationalDatabaseCommands(org.finos.legend.engine.plan.execution.stores.relational.connection.driver.commands.RelationalDatabaseCommands) ObjectMapperFactory(org.finos.legend.engine.shared.core.ObjectMapperFactory) PreparedTempTableResult(org.finos.legend.engine.plan.execution.stores.relational.result.PreparedTempTableResult) Span(io.opentracing.Span) Scope(io.opentracing.Scope) SQLExecutionResult(org.finos.legend.engine.plan.execution.stores.relational.result.SQLExecutionResult) StoreType(org.finos.legend.engine.plan.execution.stores.StoreType) RootGraphFetchTree(org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.graph.RootGraphFetchTree) java.util(java.util) CommonProfile(org.pac4j.core.profile.CommonProfile) IGraphInstance(org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance) DoubleStrategyHashMap(org.finos.legend.engine.shared.core.collectionsExtensions.DoubleStrategyHashMap) ExecutionState(org.finos.legend.engine.plan.execution.nodes.state.ExecutionState) org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes) Lists(org.eclipse.collections.api.factory.Lists) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) ResultInterpreterExtension(org.finos.legend.engine.plan.execution.stores.relational.result.ResultInterpreterExtension) ExecutionNodeResultHelper(org.finos.legend.engine.plan.execution.nodes.helpers.ExecutionNodeResultHelper) PropertyGraphFetchTree(org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.graph.PropertyGraphFetchTree) Function(java.util.function.Function) Supplier(java.util.function.Supplier) FastList(org.eclipse.collections.impl.list.mutable.FastList) ExecutionCache(org.finos.legend.engine.plan.execution.cache.ExecutionCache) SQLException(java.sql.SQLException) RelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RelationalResult) Function2(org.eclipse.collections.api.block.function.Function2) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) JavaHelper(org.finos.legend.engine.plan.execution.nodes.helpers.platform.JavaHelper) JsonIgnore(com.fasterxml.jackson.annotation.JsonIgnore) GraphFetchCacheByEqualityKeys(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheByEqualityKeys) StreamSupport(java.util.stream.StreamSupport) Pair(org.eclipse.collections.api.tuple.Pair) BasicChecked(org.finos.legend.engine.plan.dependencies.domain.dataQuality.BasicChecked) IReferencedObject(org.finos.legend.engine.plan.dependencies.store.shared.IReferencedObject) RealizedRelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RealizedRelationalResult) Result(org.finos.legend.engine.plan.execution.result.Result) ExecutionNodeJavaPlatformHelper(org.finos.legend.engine.plan.execution.nodes.helpers.platform.ExecutionNodeJavaPlatformHelper) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) BlockConnectionContext(org.finos.legend.engine.plan.execution.stores.relational.blockConnection.BlockConnectionContext) GlobalTracer(io.opentracing.util.GlobalTracer) ClassBuilder(org.finos.legend.engine.plan.execution.result.builder._class.ClassBuilder) GraphObjectsBatch(org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch) InMemoryPropertyGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryPropertyGraphFetchExecutionNode) Consumer(java.util.function.Consumer) AtomicLong(java.util.concurrent.atomic.AtomicLong) IRelationalClassInstantiationNodeExecutor(org.finos.legend.engine.plan.dependencies.store.relational.classResult.IRelationalClassInstantiationNodeExecutor) IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics(org.finos.legend.engine.plan.dependencies.store.relational.IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics) GraphExecutionState(org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState) ExecutionState(org.finos.legend.engine.plan.execution.nodes.state.ExecutionState) IGraphInstance(org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance) ExecutionNodeExecutor(org.finos.legend.engine.plan.execution.nodes.ExecutionNodeExecutor) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) IRelationalResult(org.finos.legend.engine.plan.dependencies.store.relational.IRelationalResult) TempTableStreamingResult(org.finos.legend.engine.plan.execution.stores.relational.result.TempTableStreamingResult) PreparedTempTableResult(org.finos.legend.engine.plan.execution.stores.relational.result.PreparedTempTableResult) SQLExecutionResult(org.finos.legend.engine.plan.execution.stores.relational.result.SQLExecutionResult) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) RelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RelationalResult) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) RealizedRelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RealizedRelationalResult) Result(org.finos.legend.engine.plan.execution.result.Result) ResultSet(java.sql.ResultSet) Stream(java.util.stream.Stream) MutableList(org.eclipse.collections.api.list.MutableList) FastList(org.eclipse.collections.impl.list.mutable.FastList) org.finos.legend.engine.plan.dependencies.store.relational.graphFetch(org.finos.legend.engine.plan.dependencies.store.relational.graphFetch) org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch) GraphExecutionState(org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) DoubleStrategyHashMap(org.finos.legend.engine.shared.core.collectionsExtensions.DoubleStrategyHashMap) Method(java.lang.reflect.Method) IReferencedObject(org.finos.legend.engine.plan.dependencies.store.shared.IReferencedObject) DoubleStrategyHashMap(org.finos.legend.engine.shared.core.collectionsExtensions.DoubleStrategyHashMap) SQLExecutionResult(org.finos.legend.engine.plan.execution.stores.relational.result.SQLExecutionResult) SQLException(java.sql.SQLException) TempTableStreamingResult(org.finos.legend.engine.plan.execution.stores.relational.result.TempTableStreamingResult) Span(io.opentracing.Span) BlockConnectionContext(org.finos.legend.engine.plan.execution.stores.relational.blockConnection.BlockConnectionContext) DatabaseConnection(org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection) RealizedRelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RealizedRelationalResult) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) Connection(java.sql.Connection) BlockConnection(org.finos.legend.engine.plan.execution.stores.relational.blockConnection.BlockConnection) DatabaseConnection(org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection) RelationalDatabaseCommands(org.finos.legend.engine.plan.execution.stores.relational.connection.driver.commands.RelationalDatabaseCommands) InvocationTargetException(java.lang.reflect.InvocationTargetException) SQLException(java.sql.SQLException) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) GraphFetchCacheKey(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheKey) Scope(io.opentracing.Scope) IRelationalResult(org.finos.legend.engine.plan.dependencies.store.relational.IRelationalResult) RelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RelationalResult) RealizedRelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RealizedRelationalResult) GraphFetchCacheByEqualityKeys(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheByEqualityKeys) IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics(org.finos.legend.engine.plan.dependencies.store.relational.IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics)

Example 2 with GraphExecutionState

use of org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState in project legend-engine by finos.

the class RelationalExecutionNodeExecutor method executeLocalRelationalGraphOperation.

private void executeLocalRelationalGraphOperation(RelationalGraphFetchExecutionNode node, DoubleStrategyHashMap<Object, Object, SQLExecutionResult> parentMap, List<Method> parentKeyGetters) {
    GraphExecutionState graphExecutionState = (GraphExecutionState) executionState;
    List<Object> childObjects = FastList.newList();
    graphExecutionState.setObjectsForNodeIndex(node.nodeIndex, childObjects);
    SQLExecutionResult childResult = null;
    try {
        /* Get Java executor */
        Class<?> executeClass = this.getExecuteClass(node);
        if (Arrays.asList(executeClass.getInterfaces()).contains(IRelationalChildGraphNodeExecutor.class)) {
            IRelationalChildGraphNodeExecutor executor = (IRelationalChildGraphNodeExecutor) executeClass.getConstructor().newInstance();
            /* Execute relational node corresponding to the child */
            childResult = (SQLExecutionResult) node.relationalNode.accept(new ExecutionNodeExecutor(profiles, executionState));
            boolean nonPrimitiveNode = node.resultType instanceof ClassResultType;
            boolean childrenExist = node.children != null && !node.children.isEmpty();
            /* Change the second strategy to suit the primary key indices of parent PKs in the child result set*/
            List<String> parentSQLKeyColumns = executor.parentSQLColumnsInResultSet(childResult.getResultColumns().stream().map(ResultColumn::getNonQuotedLabel).collect(Collectors.toList()));
            List<Integer> parentPrimaryKeyIndices = FastList.newList();
            for (String pkCol : parentSQLKeyColumns) {
                parentPrimaryKeyIndices.add(childResult.getResultSet().findColumn(pkCol));
            }
            RelationalGraphFetchUtils.switchSecondKeyHashingStrategy(parentMap, parentKeyGetters, parentPrimaryKeyIndices);
            String databaseConnectionString = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(childResult.getSQLExecutionNode().connection);
            if (nonPrimitiveNode) {
                List<Method> primaryKeyGetters = executor.primaryKeyGetters();
                int primaryKeyCount = primaryKeyGetters.size();
                DoubleStrategyHashMap<Object, Object, SQLExecutionResult> currentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
                String tempTableName = childrenExist ? ((RelationalTempTableGraphFetchExecutionNode) node).tempTableName : null;
                RealizedRelationalResult realizedRelationalResult = childrenExist ? RealizedRelationalResult.emptyRealizedRelationalResult(((RelationalTempTableGraphFetchExecutionNode) node).columns) : null;
                DatabaseConnection databaseConnection = childResult.getSQLExecutionNode().connection;
                String databaseType = childResult.getDatabaseType();
                String databaseTimeZone = childResult.getDatabaseTimeZone();
                ResultSet childResultSet = childResult.getResultSet();
                try (Scope ignored1 = GlobalTracer.get().buildSpan("Graph Query Relational: Read Child Batch").startActive(true)) {
                    while (childResultSet.next()) {
                        graphExecutionState.incrementRowCount();
                        Object parent = parentMap.getWithSecondKey(childResult);
                        if (parent == null) {
                            throw new RuntimeException("No parent");
                        }
                        org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?> childGraphInstance = executor.getObjectFromResultSet(childResultSet, childResult.getDatabaseTimeZone(), databaseConnectionString);
                        Object child = childGraphInstance.getValue();
                        Object mapObject = currentMap.putIfAbsent(child, child);
                        if (mapObject == null) {
                            mapObject = child;
                            graphExecutionState.addObjectMemoryUtilization(childGraphInstance.instanceSize());
                            childObjects.add(mapObject);
                            if (childrenExist) {
                                this.addKeyRowToRealizedRelationalResult(child, primaryKeyGetters, realizedRelationalResult);
                            }
                        }
                        executor.addChildToParent(parent, mapObject, DefaultExecutionNodeContext.factory().create(graphExecutionState, null));
                    }
                }
                childResult.close();
                childResult = null;
                if (childrenExist) {
                    this.executeRelationalChildren(node, tempTableName, realizedRelationalResult, databaseConnection, databaseType, databaseTimeZone, currentMap, primaryKeyGetters);
                }
            } else {
                ResultSet childResultSet = childResult.getResultSet();
                while (childResultSet.next()) {
                    Object parent = parentMap.getWithSecondKey(childResult);
                    if (parent == null) {
                        throw new RuntimeException("No parent");
                    }
                    org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?> childGraphInstance = executor.getObjectFromResultSet(childResultSet, childResult.getDatabaseTimeZone(), databaseConnectionString);
                    Object child = childGraphInstance.getValue();
                    childObjects.add(child);
                    graphExecutionState.addObjectMemoryUtilization(childGraphInstance.instanceSize());
                    executor.addChildToParent(parent, child, DefaultExecutionNodeContext.factory().create(graphExecutionState, null));
                }
                childResult.close();
                childResult = null;
            }
        } else {
            throw new RuntimeException("Unknown execute class " + executeClass.getCanonicalName());
        }
    } catch (RuntimeException e) {
        throw e;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        if (childResult != null) {
            childResult.close();
        }
    }
}
Also used : ClassResultType(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.result.ClassResultType) ExecutionNodeExecutor(org.finos.legend.engine.plan.execution.nodes.ExecutionNodeExecutor) ResultSet(java.sql.ResultSet) DatabaseConnection(org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection) org.finos.legend.engine.plan.dependencies.store.relational.graphFetch(org.finos.legend.engine.plan.dependencies.store.relational.graphFetch) org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch) RealizedRelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RealizedRelationalResult) GraphExecutionState(org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState) ResultColumn(org.finos.legend.engine.plan.execution.stores.relational.result.ResultColumn) DoubleStrategyHashMap(org.finos.legend.engine.shared.core.collectionsExtensions.DoubleStrategyHashMap) Method(java.lang.reflect.Method) InvocationTargetException(java.lang.reflect.InvocationTargetException) SQLException(java.sql.SQLException) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) Scope(io.opentracing.Scope) IReferencedObject(org.finos.legend.engine.plan.dependencies.store.shared.IReferencedObject) SQLExecutionResult(org.finos.legend.engine.plan.execution.stores.relational.result.SQLExecutionResult)

Example 3 with GraphExecutionState

use of org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState in project legend-engine by finos.

the class ExecutionNodeExecutor method visit.

@Deprecated
@Override
public Result visit(GraphFetchExecutionNode graphFetchExecutionNode) {
    final Span topSpan = GlobalTracer.get().activeSpan();
    try (Scope ignored1 = GlobalTracer.get().buildSpan("Graph Query: Execute").startActive(true)) {
        Result rootResult;
        try (Scope ignored2 = GlobalTracer.get().buildSpan("Graph Query: Execute Root").startActive(true)) {
            rootResult = graphFetchExecutionNode.rootExecutionNode.accept(new ExecutionNodeExecutor(profiles, executionState));
        }
        if (graphFetchExecutionNode.implementation != null) {
            if (!(rootResult instanceof StreamingObjectResult)) {
                throw new RuntimeException("Unexpected result : " + rootResult.getClass().getName());
            }
            StreamingObjectResult<?> objectResult = (StreamingObjectResult<?>) rootResult;
            try {
                if (!(graphFetchExecutionNode.implementation instanceof JavaPlatformImplementation)) {
                    throw new RuntimeException("Only Java implementations are currently supported, found: " + graphFetchExecutionNode.implementation);
                }
                JavaPlatformImplementation javaPlatformImpl = (JavaPlatformImplementation) graphFetchExecutionNode.implementation;
                String executionClassName = JavaHelper.getExecutionClassFullName(javaPlatformImpl);
                String executionMethodName = JavaHelper.getExecutionMethodName(javaPlatformImpl);
                Stream<?> transformedResult = ExecutionNodeJavaPlatformHelper.executeStaticJavaMethod(graphFetchExecutionNode, executionClassName, executionMethodName, Arrays.asList(StreamingObjectResult.class, ExecutionNode.class, ExecutionState.class, ProfileManager.class), Arrays.asList(objectResult, graphFetchExecutionNode, this.executionState, this.profiles), this.executionState, this.profiles);
                return new StreamingObjectResult<>(transformedResult, objectResult.getResultBuilder(), objectResult);
            } catch (Exception e) {
                objectResult.close();
                throw e;
            }
        } else {
            final long maxMemoryBytesForGraph = 52_428_800L;
            /* 50MB - 50 * 1024 * 1024 */
            final int batchSize = graphFetchExecutionNode.batchSize;
            final GlobalGraphFetchExecutionNode rootGlobalNode = graphFetchExecutionNode.globalGraphFetchExecutionNode;
            final LocalGraphFetchExecutionNode rootLocalNode = rootGlobalNode.localGraphFetchExecutionNode;
            final AtomicInteger batchIndex = new AtomicInteger(1);
            final AtomicLong rowCount = new AtomicLong(0);
            try {
                Stream<List<?>> batchedStoreLocalObjectStream = StreamSupport.stream(new Spliterators.AbstractSpliterator<List<?>>(Long.MAX_VALUE, Spliterator.ORDERED) {

                    @Override
                    public boolean tryAdvance(Consumer<? super List<?>> action) {
                        int currentBatch = batchIndex.getAndIncrement();
                        try (Scope scope = GlobalTracer.get().buildSpan("Graph Query: Execute Batch " + currentBatch).startActive(true)) {
                            GraphExecutionState graphExecutionState = new GraphExecutionState(executionState, batchSize, rootResult, maxMemoryBytesForGraph);
                            ConstantResult constantResult = (ConstantResult) rootLocalNode.accept(new ExecutionNodeExecutor(ExecutionNodeExecutor.this.profiles, graphExecutionState));
                            List<?> objects = (List<?>) constantResult.getValue();
                            boolean nonEmptyObjectList = !objects.isEmpty();
                            if (rootGlobalNode.children != null && (rootGlobalNode.children.size() > 0) && nonEmptyObjectList) {
                                try (Scope ignored3 = GlobalTracer.get().buildSpan("Graph Query: Execute Cross Store Children").startActive(true)) {
                                    for (GlobalGraphFetchExecutionNode crossChild : rootGlobalNode.children) {
                                        try (Scope ignored4 = GlobalTracer.get().buildSpan("Graph Query: Execute Cross Store Child").startActive(true)) {
                                            graphExecutionState.setObjectsToGraphFetch(objects);
                                            ExecutionNodeExecutor.this.executeGlobalGraphOperation(crossChild, graphExecutionState);
                                        }
                                    }
                                }
                            }
                            rowCount.addAndGet(graphExecutionState.getRowCount());
                            action.accept(objects);
                            if (nonEmptyObjectList) {
                                scope.span().setTag("batchObjectCount", objects.size());
                                scope.span().setTag("batchMemoryUtilization", graphExecutionState.getTotalObjectMemoryUtilization());
                            } else {
                                if (topSpan != null) {
                                    topSpan.setTag("lastQueryRowCount", rowCount);
                                }
                            }
                            return nonEmptyObjectList && (objects.size() >= batchSize);
                        }
                    }
                }, false);
                Stream<?> globalObjectStream = batchedStoreLocalObjectStream.flatMap(Collection::stream);
                return new StreamingObjectResult<>(globalObjectStream, new PartialClassBuilder(graphFetchExecutionNode), rootResult);
            } catch (Exception e) {
                rootResult.close();
                throw e;
            }
        }
    }
}
Also used : ProfileManager(org.pac4j.core.profile.ProfileManager) GraphExecutionState(org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState) ExecutionState(org.finos.legend.engine.plan.execution.nodes.state.ExecutionState) PartialClassBuilder(org.finos.legend.engine.plan.execution.result.builder._class.PartialClassBuilder) LocalGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.LocalGraphFetchExecutionNode) InMemoryCrossStoreGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryCrossStoreGraphFetchExecutionNode) ErrorExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ErrorExecutionNode) StoreStreamReadingExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.StoreStreamReadingExecutionNode) ExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ExecutionNode) AllocationExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.AllocationExecutionNode) ConstantExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ConstantExecutionNode) InMemoryRootGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryRootGraphFetchExecutionNode) FreeMarkerConditionalExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.FreeMarkerConditionalExecutionNode) MultiResultSequenceExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.MultiResultSequenceExecutionNode) AggregationAwareExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.AggregationAwareExecutionNode) GraphFetchM2MExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.GraphFetchM2MExecutionNode) PureExpressionPlatformExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.PureExpressionPlatformExecutionNode) GraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.GraphFetchExecutionNode) SequenceExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.SequenceExecutionNode) InMemoryPropertyGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryPropertyGraphFetchExecutionNode) GlobalGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.GlobalGraphFetchExecutionNode) LocalGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.LocalGraphFetchExecutionNode) Span(io.opentracing.Span) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) ErrorResult(org.finos.legend.engine.plan.execution.result.ErrorResult) Result(org.finos.legend.engine.plan.execution.result.Result) MultiResult(org.finos.legend.engine.plan.execution.result.MultiResult) JavaPlatformImplementation(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.JavaPlatformImplementation) MutableList(org.eclipse.collections.api.list.MutableList) List(java.util.List) GraphExecutionState(org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) Spliterators(java.util.Spliterators) AtomicLong(java.util.concurrent.atomic.AtomicLong) Scope(io.opentracing.Scope) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Collection(java.util.Collection) GlobalGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.GlobalGraphFetchExecutionNode)

Aggregations

Scope (io.opentracing.Scope)3 GraphExecutionState (org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState)3 JsonProcessingException (com.fasterxml.jackson.core.JsonProcessingException)2 Span (io.opentracing.Span)2 InvocationTargetException (java.lang.reflect.InvocationTargetException)2 Method (java.lang.reflect.Method)2 ResultSet (java.sql.ResultSet)2 SQLException (java.sql.SQLException)2 AtomicLong (java.util.concurrent.atomic.AtomicLong)2 MutableList (org.eclipse.collections.api.list.MutableList)2 org.finos.legend.engine.plan.dependencies.store.relational.graphFetch (org.finos.legend.engine.plan.dependencies.store.relational.graphFetch)2 IReferencedObject (org.finos.legend.engine.plan.dependencies.store.shared.IReferencedObject)2 ExecutionNodeExecutor (org.finos.legend.engine.plan.execution.nodes.ExecutionNodeExecutor)2 RealizedRelationalResult (org.finos.legend.engine.plan.execution.stores.relational.result.RealizedRelationalResult)2 ResultColumn (org.finos.legend.engine.plan.execution.stores.relational.result.ResultColumn)2 SQLExecutionResult (org.finos.legend.engine.plan.execution.stores.relational.result.SQLExecutionResult)2 org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch (org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch)2 ClassResultType (org.finos.legend.engine.protocol.pure.v1.model.executionPlan.result.ClassResultType)2 DatabaseConnection (org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection)2 DoubleStrategyHashMap (org.finos.legend.engine.shared.core.collectionsExtensions.DoubleStrategyHashMap)2