Examples with StoreType org.finos.legend.engine.plan.execution.stores.StoreType used on opensource projects

Example 1 with StoreType

use of org.finos.legend.engine.plan.execution.stores.StoreType in project legend-engine by finos.

the class RelationalExecutionNodeExecutor method executeRelationalClassQueryTempTableGraphFetchExecutionNode.

private Result executeRelationalClassQueryTempTableGraphFetchExecutionNode(RelationalClassQueryTempTableGraphFetchExecutionNode node) {
    boolean isLeaf = node.children == null || node.children.isEmpty();
    List<Object> childObjects = new ArrayList<>();
    List<Pair<IGraphInstance<? extends IReferencedObject>, ExecutionCache<GraphFetchCacheKey, Object>>> childInstancesToDeepFetchAndCache = new ArrayList<>();
    Result childResult = null;
    try (Scope ignored = GlobalTracer.get().buildSpan("local store property graph fetch").withTag("storeType", "relational").withTag("property", ((PropertyGraphFetchTree) node.graphFetchTree).property).startActive(true)) {
        RelationalGraphObjectsBatch relationalGraphObjectsBatch = (RelationalGraphObjectsBatch) this.executionState.graphObjectsBatch;
        childResult = node.executionNodes.get(0).accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
        SQLExecutionResult childSqlResult = (SQLExecutionResult) childResult;
        DatabaseConnection databaseConnection = childSqlResult.getSQLExecutionNode().connection;
        ResultSet childResultSet = childSqlResult.getResultSet();
        IRelationalClassQueryTempTableGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.profiles);
        /* Check if caching is enabled and fetch caches if required */
        List<Pair<String, String>> allInstanceSetImplementations = nodeSpecifics.allInstanceSetImplementations();
        int setIdCount = allInstanceSetImplementations.size();
        boolean isUnion = setIdCount > 1;
        RelationalMultiSetExecutionCacheWrapper multiSetCache = new RelationalMultiSetExecutionCacheWrapper(setIdCount);
        boolean cachingEnabledForNode = this.checkForCachingAndPopulateCachingHelpers(allInstanceSetImplementations, nodeSpecifics.supportsCaching(), node.graphFetchTree, childSqlResult, nodeSpecifics::primaryKeyColumns, multiSetCache);
        DoubleStrategyHashMap<Object, Object, SQLExecutionResult> parentMap = this.switchedParentHashMapPerChildResult(relationalGraphObjectsBatch, node.parentIndex, childResultSet, () -> nodeSpecifics.parentPrimaryKeyColumns(childSqlResult.getResultColumns().stream().map(ResultColumn::getNonQuotedLabel).collect(Collectors.toList())));
        List<Method> primaryKeyGetters = nodeSpecifics.primaryKeyGetters();
        DoubleStrategyHashMap<Object, Object, SQLExecutionResult> currentMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
        RealizedRelationalResult realizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.columns);
        /* Prepare for reading */
        nodeSpecifics.prepare(childResultSet, childSqlResult.getDatabaseTimeZone(), ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(databaseConnection));
        while (childResultSet.next()) {
            relationalGraphObjectsBatch.incrementRowCount();
            Object child;
            int setIndex = isUnion ? childResultSet.getInt(1) : 0;
            Object cachedObject = RelationalExecutionNodeExecutor.this.checkAndReturnCachedObject(cachingEnabledForNode, setIndex, multiSetCache);
            boolean shouldDeepFetchOnThisInstance = cachedObject == null;
            if (shouldDeepFetchOnThisInstance) {
                IGraphInstance<? extends IReferencedObject> wrappedObject = nodeSpecifics.nextGraphInstance();
                Object wrappedValue = wrappedObject.getValue();
                Object mapObject = currentMap.putIfAbsent(wrappedValue, wrappedValue);
                if (mapObject == null) {
                    child = wrappedValue;
                    childInstancesToDeepFetchAndCache.add(Tuples.pair(wrappedObject, multiSetCache.setCaches.get(setIndex)));
                    relationalGraphObjectsBatch.addObjectMemoryUtilization(wrappedObject.instanceSize());
                    childObjects.add(child);
                    if (!isLeaf) {
                        this.addKeyRowToRealizedRelationalResult(child, primaryKeyGetters, realizedRelationalResult);
                    }
                } else {
                    child = mapObject;
                }
            } else {
                childObjects.add(cachedObject);
                child = cachedObject;
            }
            Object parent = parentMap.getWithSecondKey(childSqlResult);
            if (parent == null) {
                throw new RuntimeException("Cannot find the parent for child");
            }
            nodeSpecifics.addChildToParent(parent, child, DefaultExecutionNodeContext.factory().create(this.executionState, null));
        }
        relationalGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, childObjects);
        childResult.close();
        childResult = null;
        /* Execute store local children */
        if (!isLeaf) {
            this.executeTempTableNodeChildren(node, realizedRelationalResult, databaseConnection, childSqlResult.getDatabaseType(), childSqlResult.getDatabaseTimeZone(), currentMap, primaryKeyGetters, this.executionState);
        }
        childInstancesToDeepFetchAndCache.stream().filter(x -> x.getTwo() != null).forEach(x -> {
            Object object = x.getOne().getValue();
            x.getTwo().put(new RelationalGraphFetchUtils.RelationalObjectGraphFetchCacheKey(object, primaryKeyGetters), object);
        });
        return new ConstantResult(childObjects);
    } catch (RuntimeException e) {
        throw e;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        if (childResult != null) {
            childResult.close();
        }
    }
}

Example 2 with StoreType

use of org.finos.legend.engine.plan.execution.stores.StoreType in project legend-engine by finos.

the class PlanExecutor method buildStoreExecutor.

private static Optional<StoreExecutor> buildStoreExecutor(StoreType storeType, ImmutableList<StoreExecutorConfiguration> configurations, ImmutableList<StoreExecutorBuilder> builders) {
    if (builders.size() == 0) {
        return Optional.empty();
    }
    if (builders.size() > 1) {
        List<String> builderClasses = builders.stream().map(builder -> builder.getClass().getCanonicalName()).collect(Collectors.toList());
        String message = String.format("Found more than one builder for store type %s. Builders=%s", storeType, builderClasses);
        throw new RuntimeException(message);
    }
    if (configurations.size() > 1) {
        List<String> configurationClasses = configurations.stream().map(config -> config.getClass().getCanonicalName()).collect(Collectors.toList());
        String message = String.format("Found more than one configuration for store type %s. Configuration object types=%s", storeType, configurationClasses.size());
        throw new RuntimeException(message);
    }
    StoreExecutorBuilder builder = builders.get(0);
    StoreExecutorConfiguration configuration = configurations.size() == 1 ? configurations.get(0) : null;
    StoreExecutor storeExecutor = configuration == null ? builder.build() : builder.build(configuration);
    return Optional.of(storeExecutor);
}

Example 3 with StoreType

use of org.finos.legend.engine.plan.execution.stores.StoreType in project legend-engine by finos.

the class RelationalExecutionNodeExecutor method executeRelationalRootQueryTempTableGraphFetchExecutionNode.

private Result executeRelationalRootQueryTempTableGraphFetchExecutionNode(RelationalRootQueryTempTableGraphFetchExecutionNode node) {
    int batchSize = node.batchSize == null ? 1000 : node.batchSize;
    boolean isLeaf = node.children == null || node.children.isEmpty();
    Result rootResult = null;
    Span graphFetchSpan = GlobalTracer.get().buildSpan("graph fetch").withTag("rootStoreType", "relational").withTag("batchSizeConfig", batchSize).start();
    GlobalTracer.get().activateSpan(graphFetchSpan);
    try {
        rootResult = node.executionNodes.get(0).accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
        SQLExecutionResult sqlExecutionResult = (SQLExecutionResult) rootResult;
        DatabaseConnection databaseConnection = sqlExecutionResult.getSQLExecutionNode().connection;
        ResultSet rootResultSet = ((SQLExecutionResult) rootResult).getResultSet();
        IRelationalRootQueryTempTableGraphFetchExecutionNodeSpecifics nodeSpecifics = ExecutionNodeJavaPlatformHelper.getNodeSpecificsInstance(node, this.executionState, this.profiles);
        List<Method> primaryKeyGetters = nodeSpecifics.primaryKeyGetters();
        /* Check if caching is enabled and fetch caches if required */
        List<Pair<String, String>> allInstanceSetImplementations = nodeSpecifics.allInstanceSetImplementations();
        int setIdCount = allInstanceSetImplementations.size();
        RelationalMultiSetExecutionCacheWrapper multiSetCache = new RelationalMultiSetExecutionCacheWrapper(setIdCount);
        boolean cachingEnabledForNode = this.checkForCachingAndPopulateCachingHelpers(allInstanceSetImplementations, nodeSpecifics.supportsCaching(), node.graphFetchTree, sqlExecutionResult, nodeSpecifics::primaryKeyColumns, multiSetCache);
        /* Prepare for reading */
        nodeSpecifics.prepare(rootResultSet, sqlExecutionResult.getDatabaseTimeZone(), ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().writeValueAsString(databaseConnection));
        boolean isUnion = setIdCount > 1;
        AtomicLong batchIndex = new AtomicLong(0L);
        Spliterator<GraphObjectsBatch> graphObjectsBatchSpliterator = new Spliterators.AbstractSpliterator<GraphObjectsBatch>(Long.MAX_VALUE, Spliterator.ORDERED) {

            @Override
            public boolean tryAdvance(Consumer<? super GraphObjectsBatch> action) {
                /* Ensure all children run in the same connection */
                RelationalStoreExecutionState relationalStoreExecutionState = (RelationalStoreExecutionState) executionState.getStoreExecutionState(StoreType.Relational);
                BlockConnectionContext oldBlockConnectionContext = relationalStoreExecutionState.getBlockConnectionContext();
                boolean oldRetainConnectionFlag = relationalStoreExecutionState.retainConnection();
                relationalStoreExecutionState.setBlockConnectionContext(new BlockConnectionContext());
                relationalStoreExecutionState.setRetainConnection(true);
                long currentBatch = batchIndex.incrementAndGet();
                try (Scope ignored = GlobalTracer.get().buildSpan("graph fetch batch").withTag("storeType", "relational").withTag("batchIndex", currentBatch).withTag("class", ((RootGraphFetchTree) node.graphFetchTree)._class).asChildOf(graphFetchSpan).startActive(true)) {
                    RelationalGraphObjectsBatch relationalGraphObjectsBatch = new RelationalGraphObjectsBatch(currentBatch);
                    List<Object> resultObjects = new ArrayList<>();
                    List<Pair<IGraphInstance<? extends IReferencedObject>, ExecutionCache<GraphFetchCacheKey, Object>>> instancesToDeepFetchAndCache = new ArrayList<>();
                    int objectCount = 0;
                    while ((!rootResultSet.isClosed()) && rootResultSet.next()) {
                        relationalGraphObjectsBatch.incrementRowCount();
                        int setIndex = isUnion ? rootResultSet.getInt(1) : 0;
                        Object cachedObject = RelationalExecutionNodeExecutor.this.checkAndReturnCachedObject(cachingEnabledForNode, setIndex, multiSetCache);
                        boolean shouldDeepFetchOnThisInstance = cachedObject == null;
                        Object object;
                        if (shouldDeepFetchOnThisInstance) {
                            IGraphInstance<? extends IReferencedObject> wrappedObject = nodeSpecifics.nextGraphInstance();
                            instancesToDeepFetchAndCache.add(Tuples.pair(wrappedObject, multiSetCache.setCaches.get(setIndex)));
                            object = wrappedObject.getValue();
                        } else {
                            object = cachedObject;
                        }
                        if (node.checked != null && node.checked) {
                            resultObjects.add(BasicChecked.newChecked(object, null));
                        } else {
                            resultObjects.add(object);
                        }
                        objectCount += 1;
                        if (objectCount >= batchSize) {
                            break;
                        }
                    }
                    relationalGraphObjectsBatch.setObjectsForNodeIndex(node.nodeIndex, resultObjects);
                    if (!instancesToDeepFetchAndCache.isEmpty()) {
                        RealizedRelationalResult realizedRelationalResult = RealizedRelationalResult.emptyRealizedRelationalResult(node.columns);
                        DoubleStrategyHashMap<Object, Object, SQLExecutionResult> rootMap = new DoubleStrategyHashMap<>(RelationalGraphFetchUtils.objectSQLResultDoubleHashStrategyWithEmptySecondStrategy(primaryKeyGetters));
                        for (Pair<IGraphInstance<? extends IReferencedObject>, ExecutionCache<GraphFetchCacheKey, Object>> instanceAndCache : instancesToDeepFetchAndCache) {
                            IGraphInstance<? extends IReferencedObject> rootGraphInstance = instanceAndCache.getOne();
                            Object rootObject = rootGraphInstance.getValue();
                            rootMap.put(rootObject, rootObject);
                            relationalGraphObjectsBatch.addObjectMemoryUtilization(rootGraphInstance.instanceSize());
                            if (!isLeaf) {
                                RelationalExecutionNodeExecutor.this.addKeyRowToRealizedRelationalResult(rootObject, primaryKeyGetters, realizedRelationalResult);
                            }
                        }
                        /* Execute store local children */
                        if (!isLeaf) {
                            ExecutionState newState = new ExecutionState(executionState);
                            newState.graphObjectsBatch = relationalGraphObjectsBatch;
                            RelationalExecutionNodeExecutor.this.executeTempTableNodeChildren(node, realizedRelationalResult, databaseConnection, sqlExecutionResult.getDatabaseType(), sqlExecutionResult.getDatabaseTimeZone(), rootMap, primaryKeyGetters, newState);
                        }
                    }
                    instancesToDeepFetchAndCache.stream().filter(x -> x.getTwo() != null).forEach(x -> {
                        Object object = x.getOne().getValue();
                        x.getTwo().put(new RelationalGraphFetchUtils.RelationalObjectGraphFetchCacheKey(object, primaryKeyGetters), object);
                    });
                    action.accept(relationalGraphObjectsBatch);
                    return !resultObjects.isEmpty();
                } catch (SQLException | InvocationTargetException | IllegalAccessException e) {
                    throw new RuntimeException(e);
                } finally {
                    relationalStoreExecutionState.getBlockConnectionContext().unlockAllBlockConnections();
                    relationalStoreExecutionState.getBlockConnectionContext().closeAllBlockConnectionsAsync();
                    relationalStoreExecutionState.setBlockConnectionContext(oldBlockConnectionContext);
                    relationalStoreExecutionState.setRetainConnection(oldRetainConnectionFlag);
                }
            }
        };
        Stream<GraphObjectsBatch> graphObjectsBatchStream = StreamSupport.stream(graphObjectsBatchSpliterator, false);
        return new GraphFetchResult(graphObjectsBatchStream, rootResult).withGraphFetchSpan(graphFetchSpan);
    } catch (RuntimeException e) {
        if (rootResult != null) {
            rootResult.close();
        }
        if (graphFetchSpan != null) {
            graphFetchSpan.finish();
        }
        throw e;
    } catch (Exception e) {
        if (rootResult != null) {
            rootResult.close();
        }
        if (graphFetchSpan != null) {
            graphFetchSpan.finish();
        }
        throw new RuntimeException(e);
    }
}