Examples with PartialClassBuilder org.finos.legend.engine.plan.execution.result.builder._class.PartialClassBuilder used on opensource projects

Example 1 with PartialClassBuilder

use of org.finos.legend.engine.plan.execution.result.builder._class.PartialClassBuilder in project legend-engine by finos.

the class ExecutionNodeExecutor method visit.

@Override
public Result visit(GlobalGraphFetchExecutionNode globalGraphFetchExecutionNode) {
    final Span topSpan = GlobalTracer.get().activeSpan();
    final boolean isGraphRoot = globalGraphFetchExecutionNode.parentIndex == null;
    if (isGraphRoot) {
        final boolean enableConstraints = globalGraphFetchExecutionNode.enableConstraints == null ? false : globalGraphFetchExecutionNode.enableConstraints;
        final boolean checked = globalGraphFetchExecutionNode.checked == null ? false : globalGraphFetchExecutionNode.checked;
        // Handle batching at root level
        final AtomicLong rowCount = new AtomicLong(0L);
        final AtomicLong objectCount = new AtomicLong(0L);
        final DoubleSummaryStatistics memoryStatistics = new DoubleSummaryStatistics();
        GraphFetchResult graphFetchResult = (GraphFetchResult) globalGraphFetchExecutionNode.localGraphFetchExecutionNode.accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
        Stream<?> objectStream = graphFetchResult.getGraphObjectsBatchStream().map(batch -> {
            List<?> parentObjects = batch.getObjectsForNodeIndex(0);
            boolean nonEmptyObjectList = !parentObjects.isEmpty();
            ExecutionState newState = new ExecutionState(this.executionState).setGraphObjectsBatch(batch);
            if (globalGraphFetchExecutionNode.children != null && !globalGraphFetchExecutionNode.children.isEmpty() && nonEmptyObjectList) {
                globalGraphFetchExecutionNode.children.forEach(c -> c.accept(new ExecutionNodeExecutor(this.profiles, newState)));
            }
            rowCount.addAndGet(batch.getRowCount());
            if (nonEmptyObjectList) {
                long currentObjectCount = objectCount.addAndGet(parentObjects.size());
                memoryStatistics.accept(batch.getTotalObjectMemoryUtilization() / (parentObjects.size() * 1.0));
                if (graphFetchResult.getGraphFetchSpan() != null) {
                    Span graphFetchSpan = graphFetchResult.getGraphFetchSpan();
                    graphFetchSpan.setTag("batchCount", memoryStatistics.getCount());
                    graphFetchSpan.setTag("objectCount", currentObjectCount);
                    graphFetchSpan.setTag("avgMemoryUtilizationInBytesPerObject", memoryStatistics.getAverage());
                }
            }
            if (!nonEmptyObjectList) {
                if (topSpan != null && rowCount.get() > 0) {
                    topSpan.setTag("lastQueryRowCount", rowCount);
                }
            }
            if (checked) {
                return parentObjects.stream().map(x -> (IChecked<?>) x).map(x -> x.getValue() instanceof Constrained ? ((Constrained<?>) x.getValue()).toChecked(x.getSource(), enableConstraints) : x).collect(Collectors.toList());
            }
            if (enableConstraints) {
                return parentObjects.stream().map(x -> x instanceof Constrained ? ((Constrained<?>) x).withConstraintsApplied() : x).collect(Collectors.toList());
            }
            return parentObjects;
        }).flatMap(Collection::stream);
        boolean realizeAsConstant = this.executionState.inAllocation && ExecutionNodeResultHelper.isResultSizeRangeSet(globalGraphFetchExecutionNode) && ExecutionNodeResultHelper.isSingleRecordResult(globalGraphFetchExecutionNode);
        if (realizeAsConstant) {
            return new ConstantResult(objectStream.findFirst().orElseThrow(() -> new RuntimeException("Constant value not found")));
        }
        return new StreamingObjectResult<>(objectStream, new PartialClassBuilder(globalGraphFetchExecutionNode), graphFetchResult);
    } else {
        GraphObjectsBatch graphObjectsBatch = this.executionState.graphObjectsBatch;
        List<?> parentObjects = graphObjectsBatch.getObjectsForNodeIndex(globalGraphFetchExecutionNode.parentIndex);
        if ((parentObjects != null) && !parentObjects.isEmpty()) {
            if (globalGraphFetchExecutionNode.xStorePropertyFetchDetails != null && globalGraphFetchExecutionNode.xStorePropertyFetchDetails.supportsCaching && this.executionState.graphFetchCaches != null) {
                graphObjectsBatch.setXStorePropertyCachesForNodeIndex(globalGraphFetchExecutionNode.localGraphFetchExecutionNode.nodeIndex, findGraphFetchCacheByTargetCrossKeys(globalGraphFetchExecutionNode));
            }
            globalGraphFetchExecutionNode.localGraphFetchExecutionNode.accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
            if (globalGraphFetchExecutionNode.children != null && !globalGraphFetchExecutionNode.children.isEmpty()) {
                globalGraphFetchExecutionNode.children.forEach(c -> c.accept(new ExecutionNodeExecutor(this.profiles, this.executionState)));
            }
        }
        return new ConstantResult(parentObjects);
    }
}

Example 2 with PartialClassBuilder

use of org.finos.legend.engine.plan.execution.result.builder._class.PartialClassBuilder in project legend-engine by finos.

the class ExecutionNodeExecutor method visit.

@Deprecated
@Override
public Result visit(GraphFetchExecutionNode graphFetchExecutionNode) {
    final Span topSpan = GlobalTracer.get().activeSpan();
    try (Scope ignored1 = GlobalTracer.get().buildSpan("Graph Query: Execute").startActive(true)) {
        Result rootResult;
        try (Scope ignored2 = GlobalTracer.get().buildSpan("Graph Query: Execute Root").startActive(true)) {
            rootResult = graphFetchExecutionNode.rootExecutionNode.accept(new ExecutionNodeExecutor(profiles, executionState));
        }
        if (graphFetchExecutionNode.implementation != null) {
            if (!(rootResult instanceof StreamingObjectResult)) {
                throw new RuntimeException("Unexpected result : " + rootResult.getClass().getName());
            }
            StreamingObjectResult<?> objectResult = (StreamingObjectResult<?>) rootResult;
            try {
                if (!(graphFetchExecutionNode.implementation instanceof JavaPlatformImplementation)) {
                    throw new RuntimeException("Only Java implementations are currently supported, found: " + graphFetchExecutionNode.implementation);
                }
                JavaPlatformImplementation javaPlatformImpl = (JavaPlatformImplementation) graphFetchExecutionNode.implementation;
                String executionClassName = JavaHelper.getExecutionClassFullName(javaPlatformImpl);
                String executionMethodName = JavaHelper.getExecutionMethodName(javaPlatformImpl);
                Stream<?> transformedResult = ExecutionNodeJavaPlatformHelper.executeStaticJavaMethod(graphFetchExecutionNode, executionClassName, executionMethodName, Arrays.asList(StreamingObjectResult.class, ExecutionNode.class, ExecutionState.class, ProfileManager.class), Arrays.asList(objectResult, graphFetchExecutionNode, this.executionState, this.profiles), this.executionState, this.profiles);
                return new StreamingObjectResult<>(transformedResult, objectResult.getResultBuilder(), objectResult);
            } catch (Exception e) {
                objectResult.close();
                throw e;
            }
        } else {
            final long maxMemoryBytesForGraph = 52_428_800L;
            /* 50MB - 50 * 1024 * 1024 */
            final int batchSize = graphFetchExecutionNode.batchSize;
            final GlobalGraphFetchExecutionNode rootGlobalNode = graphFetchExecutionNode.globalGraphFetchExecutionNode;
            final LocalGraphFetchExecutionNode rootLocalNode = rootGlobalNode.localGraphFetchExecutionNode;
            final AtomicInteger batchIndex = new AtomicInteger(1);
            final AtomicLong rowCount = new AtomicLong(0);
            try {
                Stream<List<?>> batchedStoreLocalObjectStream = StreamSupport.stream(new Spliterators.AbstractSpliterator<List<?>>(Long.MAX_VALUE, Spliterator.ORDERED) {

                    @Override
                    public boolean tryAdvance(Consumer<? super List<?>> action) {
                        int currentBatch = batchIndex.getAndIncrement();
                        try (Scope scope = GlobalTracer.get().buildSpan("Graph Query: Execute Batch " + currentBatch).startActive(true)) {
                            GraphExecutionState graphExecutionState = new GraphExecutionState(executionState, batchSize, rootResult, maxMemoryBytesForGraph);
                            ConstantResult constantResult = (ConstantResult) rootLocalNode.accept(new ExecutionNodeExecutor(ExecutionNodeExecutor.this.profiles, graphExecutionState));
                            List<?> objects = (List<?>) constantResult.getValue();
                            boolean nonEmptyObjectList = !objects.isEmpty();
                            if (rootGlobalNode.children != null && (rootGlobalNode.children.size() > 0) && nonEmptyObjectList) {
                                try (Scope ignored3 = GlobalTracer.get().buildSpan("Graph Query: Execute Cross Store Children").startActive(true)) {
                                    for (GlobalGraphFetchExecutionNode crossChild : rootGlobalNode.children) {
                                        try (Scope ignored4 = GlobalTracer.get().buildSpan("Graph Query: Execute Cross Store Child").startActive(true)) {
                                            graphExecutionState.setObjectsToGraphFetch(objects);
                                            ExecutionNodeExecutor.this.executeGlobalGraphOperation(crossChild, graphExecutionState);
                                        }
                                    }
                                }
                            }
                            rowCount.addAndGet(graphExecutionState.getRowCount());
                            action.accept(objects);
                            if (nonEmptyObjectList) {
                                scope.span().setTag("batchObjectCount", objects.size());
                                scope.span().setTag("batchMemoryUtilization", graphExecutionState.getTotalObjectMemoryUtilization());
                            } else {
                                if (topSpan != null) {
                                    topSpan.setTag("lastQueryRowCount", rowCount);
                                }
                            }
                            return nonEmptyObjectList && (objects.size() >= batchSize);
                        }
                    }
                }, false);
                Stream<?> globalObjectStream = batchedStoreLocalObjectStream.flatMap(Collection::stream);
                return new StreamingObjectResult<>(globalObjectStream, new PartialClassBuilder(graphFetchExecutionNode), rootResult);
            } catch (Exception e) {
                rootResult.close();
                throw e;
            }
        }
    }
}

Example 3 with PartialClassBuilder

use of org.finos.legend.engine.plan.execution.result.builder._class.PartialClassBuilder in project legend-engine by finos.

the class RelationalResult method buildTransformersAndBuilder.

private void buildTransformersAndBuilder(ExecutionNode node, DatabaseConnection databaseConnection) throws SQLException {
    boolean isDatabaseIdentifiersCaseSensitive = databaseConnection.accept(new DatabaseIdentifiersCaseSensitiveVisitor());
    if (ExecutionNodeTDSResultHelper.isResultTDS(node)) {
        List<TransformerInput<Integer>> transformerInputs = Lists.mutable.empty();
        for (int columnIndex = 1; columnIndex <= this.columnCount; columnIndex++) {
            TDSColumn c = ExecutionNodeTDSResultHelper.getTDSColumn(node, this.resultSetMetaData.getColumnLabel(columnIndex), isDatabaseIdentifiersCaseSensitive);
            transformerInputs.add(new TransformerInput<>(columnIndex, c.type, (index) -> {
                try {
                    return ExecutionNodeTDSResultHelper.isTDSColumnEnum(node, this.resultSetMetaData.getColumnLabel(index), isDatabaseIdentifiersCaseSensitive);
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }, (index) -> {
                try {
                    return ExecutionNodeTDSResultHelper.getTDSEnumTransformer(node, this.resultSetMetaData.getColumnLabel(index), isDatabaseIdentifiersCaseSensitive);
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }));
        }
        setTransformers.add(new SetImplTransformers(transformerInputs));
        this.builder = new TDSBuilder(node, this.sqlColumns, isDatabaseIdentifiersCaseSensitive);
        this.columnListForSerializer = ListIterate.collect(((TDSBuilder) this.builder).columns, col -> col.name);
    } else if (ExecutionNodeClassResultHelper.isClassResult(node)) {
        List<? extends ClassMappingInfo> classMappings = ExecutionNodeClassResultHelper.getClassMappingInfoFromClassResult(node);
        for (ClassMappingInfo classMappingInfo : classMappings) {
            List<TransformerInput<String>> transformerInputs = Lists.mutable.empty();
            for (int i = 1; i <= this.columnCount; i++) {
                final String colName = this.resultSetMetaData.getColumnLabel(i);
                PropertyInfo profiles = ListIterate.select(classMappingInfo.properties, p -> isDatabaseIdentifiersCaseSensitive ? p.property.equals(colName) : p.property.equalsIgnoreCase(colName)).getFirst();
                transformerInputs.add(new TransformerInput<>(profiles != null ? profiles.property : colName, resolveType(profiles, colName), (colNameX) -> {
                    try {
                        return !TEMPORAL_DATE_ALIASES.contains(colNameX) && ExecutionNodeClassResultHelper.isClassPropertyEnum(node, classMappingInfo.setImplementationId, colNameX);
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }, (colNameX) -> {
                    try {
                        return ExecutionNodeClassResultHelper.getClassEnumTransformer(node, classMappingInfo.setImplementationId, colNameX);
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }));
            }
            setTransformers.add(new SetImplTransformers(transformerInputs));
            if (ExecutionNodePartialClassResultHelper.isPartialClassResult(node)) {
                this.builder = new PartialClassBuilder(node);
            } else {
                this.builder = new ClassBuilder(node);
            }
        }
    } else if (ExecutionNodeRelationalResultHelper.isRelationResult(node)) {
        SetImplTransformers setImpl = new SetImplTransformers();
        for (int columnIndex = 1; columnIndex <= this.columnCount; columnIndex++) {
            setImpl.transformers.add(SetImplTransformers.TEMPORARY_DATATYPE_TRANSFORMER);
        }
        setTransformers.add(setImpl);
        this.builder = new RelationBuilder(node);
    } else {
        SetImplTransformers setImpl = new SetImplTransformers();
        for (int i = 1; i <= this.columnCount; i++) {
            setImpl.transformers.add(SetImplTransformers.TEMPORARY_DATATYPE_TRANSFORMER);
        }
        setTransformers.add(setImpl);
        this.builder = new DataTypeBuilder(node);
    }
}