Search in sources :

Example 1 with SequenceExecutionNode

use of org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.SequenceExecutionNode in project legend-engine by finos.

the class RelationalExecutionNodeExecutor method visit.

@Override
public Result visit(ExecutionNode executionNode) {
    if (executionNode instanceof RelationalBlockExecutionNode) {
        RelationalBlockExecutionNode relationalBlockExecutionNode = (RelationalBlockExecutionNode) executionNode;
        ExecutionState connectionAwareState = new ExecutionState(this.executionState);
        ((RelationalStoreExecutionState) connectionAwareState.getStoreExecutionState(StoreType.Relational)).setRetainConnection(true);
        try {
            Result res = new ExecutionNodeExecutor(this.profiles, connectionAwareState).visit((SequenceExecutionNode) relationalBlockExecutionNode);
            ((RelationalStoreExecutionState) connectionAwareState.getStoreExecutionState(StoreType.Relational)).getBlockConnectionContext().unlockAllBlockConnections();
            return res;
        } catch (Exception e) {
            ((RelationalStoreExecutionState) connectionAwareState.getStoreExecutionState(StoreType.Relational)).getBlockConnectionContext().unlockAllBlockConnections();
            ((RelationalStoreExecutionState) connectionAwareState.getStoreExecutionState(StoreType.Relational)).getBlockConnectionContext().closeAllBlockConnections();
            throw e;
        }
    } else if (executionNode instanceof CreateAndPopulateTempTableExecutionNode) {
        CreateAndPopulateTempTableExecutionNode createAndPopulateTempTableExecutionNode = (CreateAndPopulateTempTableExecutionNode) executionNode;
        Stream<Result> results = createAndPopulateTempTableExecutionNode.inputVarNames.stream().map(this.executionState::getResult);
        Stream<?> inputStream = results.flatMap(result -> {
            if (result instanceof ConstantResult) {
                Object value = ((ConstantResult) result).getValue();
                if (value instanceof Map && ((Map<?, ?>) value).get("values") instanceof List) {
                    return ((List<?>) ((Map<?, ?>) value).get("values")).stream().map(val -> ((List<?>) ((Map<?, ?>) val).get("values")).get(0));
                }
                if (value instanceof List) {
                    return ((List<?>) value).stream();
                }
                if (ClassUtils.isPrimitiveOrWrapper(value.getClass()) || (value instanceof String)) {
                    return Stream.of(value);
                }
                if (value instanceof Stream) {
                    return (Stream<?>) value;
                }
                throw new IllegalArgumentException("Result passed into CreateAndPopulateTempTableExecutionNode should be a stream");
            }
            if (result instanceof StreamingObjectResult) {
                return ((StreamingObjectResult<?>) result).getObjectStream();
            }
            throw new IllegalArgumentException("Unexpected Result Type : " + result.getClass().getName());
        });
        if (!(createAndPopulateTempTableExecutionNode.implementation instanceof JavaPlatformImplementation)) {
            throw new RuntimeException("Only Java implementations are currently supported, found: " + createAndPopulateTempTableExecutionNode.implementation);
        }
        JavaPlatformImplementation javaPlatformImpl = (JavaPlatformImplementation) createAndPopulateTempTableExecutionNode.implementation;
        String executionClassName = JavaHelper.getExecutionClassFullName(javaPlatformImpl);
        Class<?> clazz = ExecutionNodeJavaPlatformHelper.getClassToExecute(createAndPopulateTempTableExecutionNode, executionClassName, this.executionState, this.profiles);
        if (Arrays.asList(clazz.getInterfaces()).contains(IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics.class)) {
            try {
                IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics nodeSpecifics = (IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics) clazz.newInstance();
                createAndPopulateTempTableExecutionNode.tempTableColumnMetaData.forEach(t -> t.identifierForGetter = nodeSpecifics.getGetterNameForProperty(t.identifierForGetter));
            } catch (InstantiationException | IllegalAccessException e) {
                throw new RuntimeException(e);
            }
        } else {
            // TODO Remove once platform version supports above and existing plans mitigated
            String executionMethodName = JavaHelper.getExecutionMethodName(javaPlatformImpl);
            createAndPopulateTempTableExecutionNode.tempTableColumnMetaData.forEach(t -> t.identifierForGetter = ExecutionNodeJavaPlatformHelper.executeStaticJavaMethod(createAndPopulateTempTableExecutionNode, executionClassName, executionMethodName, Collections.singletonList(Result.class), Collections.singletonList(new ConstantResult(t.identifierForGetter)), this.executionState, this.profiles));
        }
        RelationalDatabaseCommands databaseCommands = DatabaseManager.fromString(createAndPopulateTempTableExecutionNode.connection.type.name()).relationalDatabaseSupport();
        try (Connection connectionManagerConnection = this.getConnection(createAndPopulateTempTableExecutionNode, databaseCommands, this.profiles, this.executionState)) {
            TempTableStreamingResult tempTableStreamingResult = new TempTableStreamingResult(inputStream, createAndPopulateTempTableExecutionNode);
            String databaseTimeZone = createAndPopulateTempTableExecutionNode.connection.timeZone == null ? RelationalExecutor.DEFAULT_DB_TIME_ZONE : createAndPopulateTempTableExecutionNode.connection.timeZone;
            databaseCommands.accept(RelationalDatabaseCommandsVisitorBuilder.getStreamResultToTempTableVisitor(((RelationalStoreExecutionState) this.executionState.getStoreExecutionState(StoreType.Relational)).getRelationalExecutor().getRelationalExecutionConfiguration(), connectionManagerConnection, tempTableStreamingResult, createAndPopulateTempTableExecutionNode.tempTableName, databaseTimeZone));
        } catch (SQLException e) {
            throw new RuntimeException(e);
        }
        return new ConstantResult("success");
    } else if (executionNode instanceof RelationalExecutionNode) {
        RelationalExecutionNode relationalExecutionNode = (RelationalExecutionNode) executionNode;
        Span topSpan = GlobalTracer.get().activeSpan();
        this.executionState.topSpan = topSpan;
        try (Scope scope = GlobalTracer.get().buildSpan("Relational DB Execution").startActive(true)) {
            scope.span().setTag("databaseType", relationalExecutionNode.getDatabaseTypeName());
            scope.span().setTag("sql", relationalExecutionNode.sqlQuery());
            Result result = ((RelationalStoreExecutionState) executionState.getStoreExecutionState(StoreType.Relational)).getRelationalExecutor().execute(relationalExecutionNode, this.profiles, this.executionState);
            if (result instanceof RelationalResult) {
                scope.span().setTag("executedSql", ((RelationalResult) result).executedSQl);
            }
            if (relationalExecutionNode.implementation != null && !(ExecutionNodeResultHelper.isResultSizeRangeSet(relationalExecutionNode) && ExecutionNodeResultHelper.isSingleRecordResult(relationalExecutionNode))) {
                return executeImplementation(relationalExecutionNode, result, this.executionState, this.profiles);
            }
            return result;
        }
    } else if (executionNode instanceof SQLExecutionNode) {
        SQLExecutionNode SQLExecutionNode = (SQLExecutionNode) executionNode;
        this.executionState.topSpan = GlobalTracer.get().activeSpan();
        try (Scope scope = GlobalTracer.get().buildSpan("Relational DB Execution").startActive(true)) {
            scope.span().setTag("databaseType", SQLExecutionNode.getDatabaseTypeName());
            scope.span().setTag("sql", SQLExecutionNode.sqlQuery());
            Result result = ((RelationalStoreExecutionState) executionState.getStoreExecutionState(StoreType.Relational)).getRelationalExecutor().execute(SQLExecutionNode, profiles, executionState);
            if (result instanceof SQLExecutionResult) {
                scope.span().setTag("executedSql", ((SQLExecutionResult) result).getExecutedSql());
            }
            return result;
        }
    } else if (executionNode instanceof RelationalTdsInstantiationExecutionNode) {
        RelationalTdsInstantiationExecutionNode relationalTdsInstantiationExecutionNode = (RelationalTdsInstantiationExecutionNode) executionNode;
        SQLExecutionResult sqlExecutionResult = null;
        try {
            sqlExecutionResult = (SQLExecutionResult) this.visit((SQLExecutionNode) relationalTdsInstantiationExecutionNode.executionNodes.get(0));
            RelationalResult relationalTdsResult = new RelationalResult(sqlExecutionResult, relationalTdsInstantiationExecutionNode);
            if (this.executionState.inAllocation) {
                if (!this.executionState.transformAllocation) {
                    return relationalTdsResult;
                }
                RealizedRelationalResult realizedRelationalResult = (RealizedRelationalResult) relationalTdsResult.realizeInMemory();
                List<Map<String, Object>> rowValueMaps = realizedRelationalResult.getRowValueMaps(false);
                Result res = RelationalExecutor.evaluateAdditionalExtractors(this.resultInterpreterExtensions, this.executionState, rowValueMaps);
                if (res != null) {
                    return res;
                } else {
                    return new ConstantResult(rowValueMaps);
                }
            }
            return relationalTdsResult;
        } catch (Exception e) {
            if (sqlExecutionResult != null) {
                sqlExecutionResult.close();
            }
            throw e;
        }
    } else if (executionNode instanceof RelationalClassInstantiationExecutionNode) {
        RelationalClassInstantiationExecutionNode node = (RelationalClassInstantiationExecutionNode) executionNode;
        SQLExecutionResult sqlExecutionResult = null;
        try {
            SQLExecutionNode innerNode = (SQLExecutionNode) node.executionNodes.get(0);
            sqlExecutionResult = (SQLExecutionResult) this.visit(innerNode);
            RelationalResult relationalResult = new RelationalResult(sqlExecutionResult, node);
            boolean realizeAsConstant = this.executionState.inAllocation && ExecutionNodeResultHelper.isResultSizeRangeSet(node) && ExecutionNodeResultHelper.isSingleRecordResult(node);
            if (realizeAsConstant) {
                RealizedRelationalResult realizedRelationalResult = (RealizedRelationalResult) relationalResult.realizeInMemory();
                List<Map<String, Object>> rowValueMaps = realizedRelationalResult.getRowValueMaps(false);
                if (rowValueMaps.size() == 1) {
                    return new ConstantResult(rowValueMaps.get(0));
                } else {
                    return new ConstantResult(rowValueMaps);
                }
            }
            return this.getStreamingObjectResultFromRelationalResult(node, relationalResult, innerNode.connection);
        } catch (Exception e) {
            if (sqlExecutionResult != null) {
                sqlExecutionResult.close();
            }
            throw (e instanceof RuntimeException) ? (RuntimeException) e : new RuntimeException(e);
        }
    } else if (executionNode instanceof RelationalRelationDataInstantiationExecutionNode) {
        RelationalRelationDataInstantiationExecutionNode node = (RelationalRelationDataInstantiationExecutionNode) executionNode;
        SQLExecutionResult sqlExecutionResult = null;
        try {
            sqlExecutionResult = (SQLExecutionResult) this.visit((SQLExecutionNode) node.executionNodes.get(0));
            return new RelationalResult(sqlExecutionResult, node);
        } catch (Exception e) {
            if (sqlExecutionResult != null) {
                sqlExecutionResult.close();
            }
            throw e;
        }
    } else if (executionNode instanceof RelationalDataTypeInstantiationExecutionNode) {
        RelationalDataTypeInstantiationExecutionNode node = (RelationalDataTypeInstantiationExecutionNode) executionNode;
        SQLExecutionResult sqlExecutionResult = null;
        try {
            sqlExecutionResult = (SQLExecutionResult) this.visit((SQLExecutionNode) node.executionNodes.get(0));
            RelationalResult relationalPrimitiveResult = new RelationalResult(sqlExecutionResult, node);
            if (this.executionState.inAllocation) {
                if ((ExecutionNodeResultHelper.isResultSizeRangeSet(node) && !ExecutionNodeResultHelper.isSingleRecordResult(node)) && !this.executionState.transformAllocation) {
                    return relationalPrimitiveResult;
                }
                if (relationalPrimitiveResult.getResultSet().next()) {
                    List<org.eclipse.collections.api.block.function.Function<Object, Object>> transformers = relationalPrimitiveResult.getTransformers();
                    Object convertedValue = transformers.get(0).valueOf(relationalPrimitiveResult.getResultSet().getObject(1));
                    return new ConstantResult(convertedValue);
                } else {
                    throw new RuntimeException("Result set is empty for allocation node");
                }
            }
            return relationalPrimitiveResult;
        } catch (Exception e) {
            if (sqlExecutionResult != null) {
                sqlExecutionResult.close();
            }
            throw (e instanceof RuntimeException) ? (RuntimeException) e : new RuntimeException(e);
        }
    } else if (executionNode instanceof RelationalRootQueryTempTableGraphFetchExecutionNode) {
        return this.executeRelationalRootQueryTempTableGraphFetchExecutionNode((RelationalRootQueryTempTableGraphFetchExecutionNode) executionNode);
    } else if (executionNode instanceof RelationalCrossRootQueryTempTableGraphFetchExecutionNode) {
        return this.executeRelationalCrossRootQueryTempTableGraphFetchExecutionNode((RelationalCrossRootQueryTempTableGraphFetchExecutionNode) executionNode);
    } else if (executionNode instanceof RelationalPrimitiveQueryGraphFetchExecutionNode) {
        return this.executeRelationalPrimitiveQueryGraphFetchExecutionNode((RelationalPrimitiveQueryGraphFetchExecutionNode) executionNode);
    } else if (executionNode instanceof RelationalClassQueryTempTableGraphFetchExecutionNode) {
        return this.executeRelationalClassQueryTempTableGraphFetchExecutionNode((RelationalClassQueryTempTableGraphFetchExecutionNode) executionNode);
    } else if (executionNode instanceof RelationalRootGraphFetchExecutionNode) {
        RelationalRootGraphFetchExecutionNode node = (RelationalRootGraphFetchExecutionNode) executionNode;
        /* Fetch info from execution state */
        GraphExecutionState graphExecutionState = (GraphExecutionState) executionState;
        int batchSize = graphExecutionState.getBatchSize();
        SQLExecutionResult rootResult = (SQLExecutionResult) graphExecutionState.getRootResult();
        ResultSet rootResultSet = rootResult.getResultSet();
        /* Ensure all children run in the same connection */
        RelationalStoreExecutionState relationalStoreExecutionState = (RelationalStoreExecutionState) graphExecutionState.getStoreExecutionState(StoreType.Relational);
        BlockConnectionContext oldBlockConnectionContext = relationalStoreExecutionState.getBlockConnectionContext();
        boolean oldRetainConnectionFlag = relationalStoreExecutionState.retainConnection();
        relationalStoreExecutionState.setBlockConnectionContext(new BlockConnectionContext());
        relationalStoreExecutionState.setRetainConnection(true);
        try (Scope ignored1 = GlobalTracer.get().buildSpan("Graph Query Relational: Execute Relational Root").startActive(true)) {
            String databaseTimeZone = rootResult.getDatabaseTimeZone();
            String databaseConnectionString = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(rootResult.getSQLExecutionNode().connection);
            /* Get Java executor */
            Class<?> executeClass = this.getExecuteClass(node);
            if (Arrays.asList(executeClass.getInterfaces()).contains(IRelationalRootGraphNodeExecutor.class)) {
                IRelationalRootGraphNodeExecutor executor = (IRelationalRootGraphNodeExecutor) executeClass.getConstructor().newInstance();
                List<Method> primaryKeyGetters = executor.primaryKeyGetters();
                int primaryKeyCount = primaryKeyGetters.size();
                /* Check if caching is enabled and fetch the cache if required */
                boolean cachingEnabledForNode = false;
                ExecutionCache<GraphFetchCacheKey, Object> graphCache = null;
                RelationalGraphFetchUtils.RelationalSQLResultGraphFetchCacheKey rootResultCacheKey = null;
                if ((this.executionState.graphFetchCaches != null) && executor.supportsCaching()) {
                    GraphFetchCacheByEqualityKeys graphFetchCacheByEqualityKeys = RelationalGraphFetchUtils.findCacheByEqualityKeys(node.graphFetchTree, executor.getMappingId(rootResultSet, databaseTimeZone, databaseConnectionString), executor.getInstanceSetId(rootResultSet, databaseTimeZone, databaseConnectionString), this.executionState.graphFetchCaches);
                    if (graphFetchCacheByEqualityKeys != null) {
                        List<String> parentSQLKeyColumns = executor.primaryKeyColumns();
                        List<Integer> parentPrimaryKeyIndices = FastList.newList();
                        for (String pkCol : parentSQLKeyColumns) {
                            parentPrimaryKeyIndices.add(rootResultSet.findColumn(pkCol));
                        }
                        cachingEnabledForNode = true;
                        graphCache = graphFetchCacheByEqualityKeys.getExecutionCache();
                        rootResultCacheKey = new RelationalGraphFetchUtils.RelationalSQLResultGraphFetchCacheKey(rootResult, parentPrimaryKeyIndices);
                    }
                }
                /* Get the next batch of root records */
                List<Object> resultObjectsBatch = new ArrayList<>();
                List<org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?>> instancesToDeepFetch = new ArrayList<>();
                int objectCount = 0;
                try (Scope ignored2 = GlobalTracer.get().buildSpan("Graph Query Relational: Read Next Batch").startActive(true)) {
                    while (rootResultSet.next()) {
                        graphExecutionState.incrementRowCount();
                        boolean shouldDeepFetchOnThisInstance = true;
                        if (cachingEnabledForNode) {
                            Object cachedObject = graphCache.getIfPresent(rootResultCacheKey);
                            if (cachedObject != null) {
                                resultObjectsBatch.add(executor.deepCopy(cachedObject));
                                shouldDeepFetchOnThisInstance = false;
                            }
                        }
                        if (shouldDeepFetchOnThisInstance) {
                            org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?> wrappedObject = executor.getObjectFromResultSet(rootResultSet, databaseTimeZone, databaseConnectionString);
                            instancesToDeepFetch.add(wrappedObject);
                            resultObjectsBatch.add(wrappedObject.getValue());
                        }
                        objectCount += 1;
                        if (objectCount >= batchSize) {
                            break;
                        }
                    }
                }
                if (!instancesToDeepFetch.isEmpty()) {
                    boolean childrenExist = node.children != null && !node.children.isEmpty();
                    String tempTableName = node.tempTableName;
                    RealizedRelationalResult realizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.columns);
                    /* Create and populate double strategy map with key being object with its PK getters */
                    DoubleStrategyHashMap<Object, Object, SQLExecutionResult> rootMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
                    for (org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?> rootGraphInstance : instancesToDeepFetch) {
                        Object rootObject = rootGraphInstance.getValue();
                        rootMap.put(rootObject, rootObject);
                        graphExecutionState.addObjectMemoryUtilization(rootGraphInstance.instanceSize());
                        if (childrenExist) {
                            this.addKeyRowToRealizedRelationalResult(rootObject, primaryKeyGetters, realizedRelationalResult);
                        }
                    }
                    /* Execute store local children */
                    if (childrenExist) {
                        this.executeRelationalChildren(node, tempTableName, realizedRelationalResult, rootResult.getSQLExecutionNode().connection, rootResult.getDatabaseType(), databaseTimeZone, rootMap, primaryKeyGetters);
                    }
                }
                graphExecutionState.setObjectsForNodeIndex(node.nodeIndex, resultObjectsBatch);
                if (cachingEnabledForNode) {
                    for (org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?> deepFetchedInstance : instancesToDeepFetch) {
                        Object objectClone = executor.deepCopy(deepFetchedInstance.getValue());
                        graphCache.put(new RelationalGraphFetchUtils.RelationalObjectGraphFetchCacheKey(objectClone, primaryKeyGetters), objectClone);
                    }
                }
                return new ConstantResult(resultObjectsBatch);
            } else {
                throw new RuntimeException("Unknown execute class " + executeClass.getCanonicalName());
            }
        } catch (RuntimeException e) {
            throw e;
        } catch (Exception e) {
            throw new RuntimeException(e);
        } finally {
            relationalStoreExecutionState.getBlockConnectionContext().unlockAllBlockConnections();
            relationalStoreExecutionState.getBlockConnectionContext().closeAllBlockConnectionsAsync();
            relationalStoreExecutionState.setBlockConnectionContext(oldBlockConnectionContext);
            relationalStoreExecutionState.setRetainConnection(oldRetainConnectionFlag);
        }
    } else if (executionNode instanceof RelationalCrossRootGraphFetchExecutionNode) {
        RelationalCrossRootGraphFetchExecutionNode node = (RelationalCrossRootGraphFetchExecutionNode) executionNode;
        GraphExecutionState graphExecutionState = (GraphExecutionState) executionState;
        List<?> parentObjects = graphExecutionState.getObjectsToGraphFetch();
        List<Object> childObjects = FastList.newList();
        graphExecutionState.setObjectsForNodeIndex(node.nodeIndex, childObjects);
        RelationalStoreExecutionState relationalStoreExecutionState = (RelationalStoreExecutionState) graphExecutionState.getStoreExecutionState(StoreType.Relational);
        BlockConnectionContext oldBlockConnectionContext = relationalStoreExecutionState.getBlockConnectionContext();
        boolean oldRetainConnectionFlag = relationalStoreExecutionState.retainConnection();
        relationalStoreExecutionState.setBlockConnectionContext(new BlockConnectionContext());
        relationalStoreExecutionState.setRetainConnection(true);
        SQLExecutionResult childResult = null;
        try (Scope ignored1 = GlobalTracer.get().buildSpan("Graph Query Relational: Execute Relational Cross Root").startActive(true)) {
            /* Get Java executor */
            Class<?> executeClass = this.getExecuteClass(node);
            if (Arrays.asList(executeClass.getInterfaces()).contains(IRelationalCrossRootGraphNodeExecutor.class)) {
                IRelationalCrossRootGraphNodeExecutor executor = (IRelationalCrossRootGraphNodeExecutor) executeClass.getConstructor().newInstance();
                if (!parentObjects.isEmpty()) {
                    String parentTempTableName = node.parentTempTableName;
                    RealizedRelationalResult parentRealizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.parentTempTableColumns);
                    List<Method> crossKeyGetters = executor.parentCrossKeyGetters();
                    int parentKeyCount = crossKeyGetters.size();
                    for (Object parentObject : parentObjects) {
                        this.addKeyRowToRealizedRelationalResult(parentObject, crossKeyGetters, parentRealizedRelationalResult);
                    }
                    graphExecutionState.addResult(parentTempTableName, parentRealizedRelationalResult);
                    /* Execute relational node corresponding to the cross root */
                    childResult = (SQLExecutionResult) node.relationalNode.accept(new ExecutionNodeExecutor(this.profiles, graphExecutionState));
                    ResultSet childResultSet = childResult.getResultSet();
                    boolean childrenExist = node.children != null && !node.children.isEmpty();
                    String tempTableName = childrenExist ? node.tempTableName : null;
                    RealizedRelationalResult realizedRelationalResult = childrenExist ? RealizedRelationalResult.emptyRealizedRelationalResult(node.columns) : null;
                    DatabaseConnection databaseConnection = childResult.getSQLExecutionNode().connection;
                    String databaseType = childResult.getDatabaseType();
                    String databaseTimeZone = childResult.getDatabaseTimeZone();
                    List<String> parentSQLKeyColumns = executor.parentSQLColumnsInResultSet(childResult.getResultColumns().stream().map(ResultColumn::getNonQuotedLabel).collect(Collectors.toList()));
                    List<Integer> parentCrossKeyIndices = FastList.newList();
                    for (String pkCol : parentSQLKeyColumns) {
                        parentCrossKeyIndices.add(childResultSet.findColumn(pkCol));
                    }
                    DoubleStrategyHashMap<Object, List<Object>, SQLExecutionResult> parentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategy(crossKeyGetters, parentCrossKeyIndices));
                    for (Object parentObject : parentObjects) {
                        List<Object> mapped = parentMap.get(parentObject);
                        if (mapped == null) {
                            parentMap.put(parentObject, FastList.newListWith(parentObject));
                        } else {
                            mapped.add(parentObject);
                        }
                    }
                    List<Method> primaryKeyGetters = executor.primaryKeyGetters();
                    final int primaryKeyCount = primaryKeyGetters.size();
                    DoubleStrategyHashMap<Object, Object, SQLExecutionResult> currentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
                    String databaseConnectionString = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(childResult.getSQLExecutionNode().connection);
                    while (childResultSet.next()) {
                        graphExecutionState.incrementRowCount();
                        List<Object> parents = parentMap.getWithSecondKey(childResult);
                        if (parents == null) {
                            throw new RuntimeException("No parent");
                        }
                        org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance<?> childGraphInstance = executor.getObjectFromResultSet(childResultSet, childResult.getDatabaseTimeZone(), databaseConnectionString);
                        Object child = childGraphInstance.getValue();
                        Object mapObject = currentMap.putIfAbsent(child, child);
                        if (mapObject == null) {
                            mapObject = child;
                            childObjects.add(mapObject);
                            graphExecutionState.addObjectMemoryUtilization(childGraphInstance.instanceSize());
                            if (childrenExist) {
                                this.addKeyRowToRealizedRelationalResult(child, primaryKeyGetters, realizedRelationalResult);
                            }
                        }
                        for (Object parent : parents) {
                            executor.addChildToParent(parent, mapObject, DefaultExecutionNodeContext.factory().create(graphExecutionState, null));
                        }
                    }
                    childResult.close();
                    childResult = null;
                    if (childrenExist) {
                        this.executeRelationalChildren(node, tempTableName, realizedRelationalResult, databaseConnection, databaseType, databaseTimeZone, currentMap, primaryKeyGetters);
                    }
                }
            } else {
                throw new RuntimeException("Unknown execute class " + executeClass.getCanonicalName());
            }
        } catch (RuntimeException e) {
            throw e;
        } catch (Exception e) {
            throw new RuntimeException(e);
        } finally {
            if (childResult != null) {
                childResult.close();
            }
            relationalStoreExecutionState.getBlockConnectionContext().unlockAllBlockConnections();
            relationalStoreExecutionState.getBlockConnectionContext().closeAllBlockConnectionsAsync();
            relationalStoreExecutionState.setBlockConnectionContext(oldBlockConnectionContext);
            relationalStoreExecutionState.setRetainConnection(oldRetainConnectionFlag);
        }
        return new ConstantResult(childObjects);
    }
    throw new RuntimeException("Not implemented!");
}
Also used : org.finos.legend.engine.plan.dependencies.store.relational.graphFetch(org.finos.legend.engine.plan.dependencies.store.relational.graphFetch) DatabaseManager(org.finos.legend.engine.plan.execution.stores.relational.connection.driver.DatabaseManager) DefaultExecutionNodeContext(org.finos.legend.engine.plan.execution.nodes.helpers.platform.DefaultExecutionNodeContext) Connection(java.sql.Connection) RelationalDatabaseCommandsVisitorBuilder(org.finos.legend.engine.plan.execution.stores.relational.RelationalDatabaseCommandsVisitorBuilder) BlockConnection(org.finos.legend.engine.plan.execution.stores.relational.blockConnection.BlockConnection) GraphFetchCacheKey(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheKey) ClassResultType(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.result.ClassResultType) MutableList(org.eclipse.collections.api.list.MutableList) InMemoryCrossStoreGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryCrossStoreGraphFetchExecutionNode) ClassUtils(org.apache.commons.lang3.ClassUtils) FunctionHelper(org.finos.legend.engine.plan.execution.stores.relational.result.FunctionHelper) GraphFetchTree(org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.graph.GraphFetchTree) ExecutionNodeExecutor(org.finos.legend.engine.plan.execution.nodes.ExecutionNodeExecutor) ResultSet(java.sql.ResultSet) StoreStreamReadingExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.StoreStreamReadingExecutionNode) Tuples(org.eclipse.collections.impl.tuple.Tuples) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) AggregationAwareActivity(org.finos.legend.engine.plan.execution.stores.relational.activity.AggregationAwareActivity) ResultColumn(org.finos.legend.engine.plan.execution.stores.relational.result.ResultColumn) Method(java.lang.reflect.Method) RelationalExecutor(org.finos.legend.engine.plan.execution.stores.relational.RelationalExecutor) IRelationalResult(org.finos.legend.engine.plan.dependencies.store.relational.IRelationalResult) RelationalGraphObjectsBatch(org.finos.legend.engine.plan.execution.stores.relational.result.graphFetch.RelationalGraphObjectsBatch) GraphExecutionState(org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState) Iterate(org.eclipse.collections.impl.utility.Iterate) TempTableStreamingResult(org.finos.legend.engine.plan.execution.stores.relational.result.TempTableStreamingResult) Collectors(java.util.stream.Collectors) InvocationTargetException(java.lang.reflect.InvocationTargetException) org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch) InMemoryRootGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryRootGraphFetchExecutionNode) DatabaseConnection(org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection) Stream(java.util.stream.Stream) RelationalDatabaseCommands(org.finos.legend.engine.plan.execution.stores.relational.connection.driver.commands.RelationalDatabaseCommands) ObjectMapperFactory(org.finos.legend.engine.shared.core.ObjectMapperFactory) PreparedTempTableResult(org.finos.legend.engine.plan.execution.stores.relational.result.PreparedTempTableResult) Span(io.opentracing.Span) Scope(io.opentracing.Scope) SQLExecutionResult(org.finos.legend.engine.plan.execution.stores.relational.result.SQLExecutionResult) StoreType(org.finos.legend.engine.plan.execution.stores.StoreType) RootGraphFetchTree(org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.graph.RootGraphFetchTree) java.util(java.util) CommonProfile(org.pac4j.core.profile.CommonProfile) IGraphInstance(org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance) DoubleStrategyHashMap(org.finos.legend.engine.shared.core.collectionsExtensions.DoubleStrategyHashMap) ExecutionState(org.finos.legend.engine.plan.execution.nodes.state.ExecutionState) org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes) Lists(org.eclipse.collections.api.factory.Lists) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) ResultInterpreterExtension(org.finos.legend.engine.plan.execution.stores.relational.result.ResultInterpreterExtension) ExecutionNodeResultHelper(org.finos.legend.engine.plan.execution.nodes.helpers.ExecutionNodeResultHelper) PropertyGraphFetchTree(org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.graph.PropertyGraphFetchTree) Function(java.util.function.Function) Supplier(java.util.function.Supplier) FastList(org.eclipse.collections.impl.list.mutable.FastList) ExecutionCache(org.finos.legend.engine.plan.execution.cache.ExecutionCache) SQLException(java.sql.SQLException) RelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RelationalResult) Function2(org.eclipse.collections.api.block.function.Function2) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) JavaHelper(org.finos.legend.engine.plan.execution.nodes.helpers.platform.JavaHelper) JsonIgnore(com.fasterxml.jackson.annotation.JsonIgnore) GraphFetchCacheByEqualityKeys(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheByEqualityKeys) StreamSupport(java.util.stream.StreamSupport) Pair(org.eclipse.collections.api.tuple.Pair) BasicChecked(org.finos.legend.engine.plan.dependencies.domain.dataQuality.BasicChecked) IReferencedObject(org.finos.legend.engine.plan.dependencies.store.shared.IReferencedObject) RealizedRelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RealizedRelationalResult) Result(org.finos.legend.engine.plan.execution.result.Result) ExecutionNodeJavaPlatformHelper(org.finos.legend.engine.plan.execution.nodes.helpers.platform.ExecutionNodeJavaPlatformHelper) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) BlockConnectionContext(org.finos.legend.engine.plan.execution.stores.relational.blockConnection.BlockConnectionContext) GlobalTracer(io.opentracing.util.GlobalTracer) ClassBuilder(org.finos.legend.engine.plan.execution.result.builder._class.ClassBuilder) GraphObjectsBatch(org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch) InMemoryPropertyGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryPropertyGraphFetchExecutionNode) Consumer(java.util.function.Consumer) AtomicLong(java.util.concurrent.atomic.AtomicLong) IRelationalClassInstantiationNodeExecutor(org.finos.legend.engine.plan.dependencies.store.relational.classResult.IRelationalClassInstantiationNodeExecutor) IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics(org.finos.legend.engine.plan.dependencies.store.relational.IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics) GraphExecutionState(org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState) ExecutionState(org.finos.legend.engine.plan.execution.nodes.state.ExecutionState) IGraphInstance(org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance) ExecutionNodeExecutor(org.finos.legend.engine.plan.execution.nodes.ExecutionNodeExecutor) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) IRelationalResult(org.finos.legend.engine.plan.dependencies.store.relational.IRelationalResult) TempTableStreamingResult(org.finos.legend.engine.plan.execution.stores.relational.result.TempTableStreamingResult) PreparedTempTableResult(org.finos.legend.engine.plan.execution.stores.relational.result.PreparedTempTableResult) SQLExecutionResult(org.finos.legend.engine.plan.execution.stores.relational.result.SQLExecutionResult) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) RelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RelationalResult) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) RealizedRelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RealizedRelationalResult) Result(org.finos.legend.engine.plan.execution.result.Result) ResultSet(java.sql.ResultSet) Stream(java.util.stream.Stream) MutableList(org.eclipse.collections.api.list.MutableList) FastList(org.eclipse.collections.impl.list.mutable.FastList) org.finos.legend.engine.plan.dependencies.store.relational.graphFetch(org.finos.legend.engine.plan.dependencies.store.relational.graphFetch) org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch) GraphExecutionState(org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) DoubleStrategyHashMap(org.finos.legend.engine.shared.core.collectionsExtensions.DoubleStrategyHashMap) Method(java.lang.reflect.Method) IReferencedObject(org.finos.legend.engine.plan.dependencies.store.shared.IReferencedObject) DoubleStrategyHashMap(org.finos.legend.engine.shared.core.collectionsExtensions.DoubleStrategyHashMap) SQLExecutionResult(org.finos.legend.engine.plan.execution.stores.relational.result.SQLExecutionResult) SQLException(java.sql.SQLException) TempTableStreamingResult(org.finos.legend.engine.plan.execution.stores.relational.result.TempTableStreamingResult) Span(io.opentracing.Span) BlockConnectionContext(org.finos.legend.engine.plan.execution.stores.relational.blockConnection.BlockConnectionContext) DatabaseConnection(org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection) RealizedRelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RealizedRelationalResult) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) Connection(java.sql.Connection) BlockConnection(org.finos.legend.engine.plan.execution.stores.relational.blockConnection.BlockConnection) DatabaseConnection(org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection) RelationalDatabaseCommands(org.finos.legend.engine.plan.execution.stores.relational.connection.driver.commands.RelationalDatabaseCommands) InvocationTargetException(java.lang.reflect.InvocationTargetException) SQLException(java.sql.SQLException) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) GraphFetchCacheKey(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheKey) Scope(io.opentracing.Scope) IRelationalResult(org.finos.legend.engine.plan.dependencies.store.relational.IRelationalResult) RelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RelationalResult) RealizedRelationalResult(org.finos.legend.engine.plan.execution.stores.relational.result.RealizedRelationalResult) GraphFetchCacheByEqualityKeys(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheByEqualityKeys) IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics(org.finos.legend.engine.plan.dependencies.store.relational.IRelationalCreateAndPopulateTempTableExecutionNodeSpecifics)

Aggregations

JsonIgnore (com.fasterxml.jackson.annotation.JsonIgnore)1 JsonProcessingException (com.fasterxml.jackson.core.JsonProcessingException)1 Scope (io.opentracing.Scope)1 Span (io.opentracing.Span)1 GlobalTracer (io.opentracing.util.GlobalTracer)1 InvocationTargetException (java.lang.reflect.InvocationTargetException)1 Method (java.lang.reflect.Method)1 Connection (java.sql.Connection)1 ResultSet (java.sql.ResultSet)1 SQLException (java.sql.SQLException)1 java.util (java.util)1 AtomicLong (java.util.concurrent.atomic.AtomicLong)1 Consumer (java.util.function.Consumer)1 Function (java.util.function.Function)1 Supplier (java.util.function.Supplier)1 Collectors (java.util.stream.Collectors)1 Stream (java.util.stream.Stream)1 StreamSupport (java.util.stream.StreamSupport)1 ClassUtils (org.apache.commons.lang3.ClassUtils)1 Function2 (org.eclipse.collections.api.block.function.Function2)1