Search in sources :

Example 1 with GraphFetchCache

use of org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCache in project legend-engine by finos.

the class ExecutionNodeExecutor method findGraphFetchCacheByTargetCrossKeys.

private ExecutionCache<GraphFetchCacheKey, List<Object>> findGraphFetchCacheByTargetCrossKeys(GlobalGraphFetchExecutionNode globalGraphFetchExecutionNode) {
    List<GraphFetchCache> graphFetchCaches = this.executionState.graphFetchCaches;
    XStorePropertyFetchDetails fetchDetails = globalGraphFetchExecutionNode.xStorePropertyFetchDetails;
    return graphFetchCaches.stream().filter(GraphFetchCacheByTargetCrossKeys.class::isInstance).map(GraphFetchCacheByTargetCrossKeys.class::cast).filter(cache -> cache.getGraphFetchCrossAssociationKeys() != null).filter(cache -> {
        GraphFetchCrossAssociationKeys c = cache.getGraphFetchCrossAssociationKeys();
        return c.getPropertyPath().equals(fetchDetails.propertyPath) && c.getSourceMappingId().equals(fetchDetails.sourceMappingId) && c.getSourceSetId().equals(fetchDetails.sourceSetId) && c.getTargetMappingId().equals(fetchDetails.targetMappingId) && c.getTargetSetId().equals(fetchDetails.targetSetId) && c.getTargetPropertiesOrdered().equals(fetchDetails.targetPropertiesOrdered) && c.getSubTree().equals(fetchDetails.subTree);
    }).map(GraphFetchCacheByTargetCrossKeys::getExecutionCache).findFirst().orElse(null);
}
Also used : Maps(org.eclipse.collections.impl.factory.Maps) ExecutionNodeVisitor(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ExecutionNodeVisitor) Arrays(java.util.Arrays) DefaultExecutionNodeContext(org.finos.legend.engine.plan.execution.nodes.helpers.platform.DefaultExecutionNodeContext) Spliterators(java.util.Spliterators) GraphFetchCacheKey(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheKey) ExecutionNodeSerializerHelper(org.finos.legend.engine.plan.execution.nodes.helpers.ExecutionNodeSerializerHelper) DoubleSummaryStatistics(java.util.DoubleSummaryStatistics) MutableList(org.eclipse.collections.api.list.MutableList) InMemoryCrossStoreGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryCrossStoreGraphFetchExecutionNode) ErrorExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ErrorExecutionNode) IPlatformPureExpressionExecutionNodeSerializeSpecifics(org.finos.legend.engine.plan.dependencies.store.platform.IPlatformPureExpressionExecutionNodeSerializeSpecifics) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) StoreStreamReadingExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.StoreStreamReadingExecutionNode) Map(java.util.Map) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) ExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ExecutionNode) FunctionParametersValidationNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.FunctionParametersValidationNode) AppliedFunction(org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.application.AppliedFunction) AllocationExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.AllocationExecutionNode) ConstantExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ConstantExecutionNode) Collection(java.util.Collection) GraphExecutionState(org.finos.legend.engine.plan.execution.nodes.state.GraphExecutionState) FunctionParametersParametersValidation(org.finos.legend.engine.plan.execution.validation.FunctionParametersParametersValidation) SerializationConfig(org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.SerializationConfig) Collectors(java.util.stream.Collectors) Constrained(org.finos.legend.engine.plan.dependencies.domain.dataQuality.Constrained) IExecutionNodeContext(org.finos.legend.engine.plan.dependencies.store.shared.IExecutionNodeContext) Objects(java.util.Objects) InMemoryRootGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryRootGraphFetchExecutionNode) List(java.util.List) Stream(java.util.stream.Stream) Lists(org.eclipse.collections.impl.factory.Lists) Span(io.opentracing.Span) Scope(io.opentracing.Scope) FreeMarkerConditionalExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.FreeMarkerConditionalExecutionNode) Spliterator(java.util.Spliterator) FreeMarkerExecutor(org.finos.legend.engine.plan.execution.nodes.helpers.freemarker.FreeMarkerExecutor) IPlatformPureExpressionExecutionNodeGraphFetchUnionSpecifics(org.finos.legend.engine.plan.dependencies.store.platform.IPlatformPureExpressionExecutionNodeGraphFetchUnionSpecifics) StoreType(org.finos.legend.engine.plan.execution.stores.StoreType) MultiResultSequenceExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.MultiResultSequenceExecutionNode) CommonProfile(org.pac4j.core.profile.CommonProfile) ExecutionState(org.finos.legend.engine.plan.execution.nodes.state.ExecutionState) GraphFetchCache(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCache) AggregationAwareExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.AggregationAwareExecutionNode) IPlatformPureExpressionExecutionNodeGraphFetchMergeSpecifics(org.finos.legend.engine.plan.dependencies.store.platform.IPlatformPureExpressionExecutionNodeGraphFetchMergeSpecifics) PartialClassBuilder(org.finos.legend.engine.plan.execution.result.builder._class.PartialClassBuilder) GraphFetchResult(org.finos.legend.engine.plan.execution.result.graphFetch.GraphFetchResult) GraphFetchM2MExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.GraphFetchM2MExecutionNode) ExecutionNodeResultHelper(org.finos.legend.engine.plan.execution.nodes.helpers.ExecutionNodeResultHelper) GraphFetchCrossAssociationKeys(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCrossAssociationKeys) ExecutionCache(org.finos.legend.engine.plan.execution.cache.ExecutionCache) PureExpressionPlatformExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.PureExpressionPlatformExecutionNode) GraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.GraphFetchExecutionNode) StreamingObjectResult(org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult) JavaHelper(org.finos.legend.engine.plan.execution.nodes.helpers.platform.JavaHelper) StreamSupport(java.util.stream.StreamSupport) IChecked(org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked) JavaPlatformImplementation(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.JavaPlatformImplementation) ErrorResult(org.finos.legend.engine.plan.execution.result.ErrorResult) SequenceExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.SequenceExecutionNode) Result(org.finos.legend.engine.plan.execution.result.Result) ResultVisitor(org.finos.legend.engine.plan.execution.result.ResultVisitor) XStorePropertyFetchDetails(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.XStorePropertyFetchDetails) GraphFetchCacheByTargetCrossKeys(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheByTargetCrossKeys) ExecutionNodeJavaPlatformHelper(org.finos.legend.engine.plan.execution.nodes.helpers.platform.ExecutionNodeJavaPlatformHelper) GlobalTracer(io.opentracing.util.GlobalTracer) GraphObjectsBatch(org.finos.legend.engine.plan.execution.result.graphFetch.GraphObjectsBatch) InMemoryPropertyGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.store.inMemory.InMemoryPropertyGraphFetchExecutionNode) ProfileManager(org.pac4j.core.profile.ProfileManager) Consumer(java.util.function.Consumer) AtomicLong(java.util.concurrent.atomic.AtomicLong) MultiResult(org.finos.legend.engine.plan.execution.result.MultiResult) GlobalGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.GlobalGraphFetchExecutionNode) LocalGraphFetchExecutionNode(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.LocalGraphFetchExecutionNode) XStorePropertyFetchDetails(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.graphFetch.XStorePropertyFetchDetails) GraphFetchCacheByTargetCrossKeys(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheByTargetCrossKeys) GraphFetchCache(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCache) GraphFetchCrossAssociationKeys(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCrossAssociationKeys)

Example 2 with GraphFetchCache

use of org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCache in project legend-engine by finos.

the class AbstractServicePlanExecutor method execute.

protected Result execute(Map<String, ?> parameters, ServiceRunnerInput serviceRunnerInput, StreamProvider streamProvider) {
    MutableList<CommonProfile> profiles = Lists.mutable.empty();
    PlanExecutionContext planExecutionContext = null;
    if (serviceRunnerInput != null) {
        Identity identity = serviceRunnerInput.getIdentity();
        if (identity != null) {
            profiles.addAll(IdentityFactoryProvider.getInstance().adapt(identity));
        }
        if (serviceRunnerInput.getOperationalContext() != null && serviceRunnerInput.getOperationalContext().getGraphFetchCrossAssociationKeysCacheConfig() != null) {
            List<GraphFetchCache> graphFetchCaches = serviceRunnerInput.getOperationalContext().getGraphFetchCrossAssociationKeysCacheConfig().entrySet().stream().map(e -> ExecutionCacheBuilder.buildGraphFetchCacheByTargetCrossKeysFromExecutionCache(e.getValue(), e.getKey())).collect(Collectors.toList());
            planExecutionContext = new PlanExecutionContext(graphFetchCaches);
        }
    }
    return this.executor.execute(this.plan, parameters, streamProvider, profiles, planExecutionContext);
}
Also used : Identity(org.finos.legend.engine.shared.core.identity.Identity) Assert(org.finos.legend.engine.shared.core.operational.Assert) CommonProfile(org.pac4j.core.profile.CommonProfile) URL(java.net.URL) GraphFetchCache(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCache) Lists(org.eclipse.collections.api.factory.Lists) SerializationFormat(org.finos.legend.engine.plan.execution.result.serialization.SerializationFormat) HashMap(java.util.HashMap) StreamProvider(org.finos.legend.engine.shared.core.url.StreamProvider) MutableList(org.eclipse.collections.api.list.MutableList) GraphFetchCrossAssociationKeys(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCrossAssociationKeys) StoreExecutor(org.finos.legend.engine.plan.execution.stores.StoreExecutor) PlanExecutor(org.finos.legend.engine.plan.execution.PlanExecutor) ByteArrayInputStream(java.io.ByteArrayInputStream) StoreExecutorConfiguration(org.finos.legend.engine.plan.execution.stores.StoreExecutorConfiguration) Map(java.util.Map) ConstantResult(org.finos.legend.engine.plan.execution.result.ConstantResult) Multiplicity(org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Multiplicity) Path(java.nio.file.Path) ErrorResult(org.finos.legend.engine.plan.execution.result.ErrorResult) PlanExecutorInfo(org.finos.legend.engine.plan.execution.PlanExecutorInfo) OutputStream(java.io.OutputStream) Result(org.finos.legend.engine.plan.execution.result.Result) Files(java.nio.file.Files) StreamingResult(org.finos.legend.engine.plan.execution.result.StreamingResult) IOException(java.io.IOException) Reader(java.io.Reader) ExecutionCacheBuilder(org.finos.legend.engine.plan.execution.cache.ExecutionCacheBuilder) IdentityFactoryProvider(org.finos.legend.engine.shared.core.identity.factory.IdentityFactoryProvider) InputStreamReader(java.io.InputStreamReader) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) Objects(java.util.Objects) List(java.util.List) PlanExecutionContext(org.finos.legend.engine.plan.execution.PlanExecutionContext) ObjectMapperFactory(org.finos.legend.engine.shared.core.ObjectMapperFactory) ExecutionPlan(org.finos.legend.engine.protocol.pure.v1.model.executionPlan.ExecutionPlan) BufferedReader(java.io.BufferedReader) Collections(java.util.Collections) InputStream(java.io.InputStream) CommonProfile(org.pac4j.core.profile.CommonProfile) GraphFetchCache(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCache) Identity(org.finos.legend.engine.shared.core.identity.Identity) PlanExecutionContext(org.finos.legend.engine.plan.execution.PlanExecutionContext)

Example 3 with GraphFetchCache

use of org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCache in project legend-engine by finos.

the class RelationalGraphFetchUtils method findCacheByEqualityKeys.

static GraphFetchCacheByEqualityKeys findCacheByEqualityKeys(GraphFetchTree graphFetchTree, String mappingId, String instanceSetId, List<GraphFetchCache> graphFetchCaches) {
    if (!subTreeValidForCaching(graphFetchTree)) {
        return null;
    }
    String subTree = getSubTreeString(graphFetchTree);
    GraphFetchCacheByEqualityKeys matchingUtilizedCache = null;
    for (GraphFetchCache c : graphFetchCaches) {
        if (c instanceof GraphFetchCacheByEqualityKeys) {
            GraphFetchCacheByEqualityKeys ce = (GraphFetchCacheByEqualityKeys) c;
            if (ce.isCacheUtilized() && mappingId.equals(ce.getMappingId()) && instanceSetId.equals(ce.getInstanceSetId()) && subTree.equals(ce.getSubTree())) {
                matchingUtilizedCache = ce;
                break;
            }
        }
    }
    if (matchingUtilizedCache == null) {
        GraphFetchCacheByEqualityKeys unUtilizedCache = null;
        for (GraphFetchCache c : graphFetchCaches) {
            if (c instanceof GraphFetchCacheByEqualityKeys) {
                GraphFetchCacheByEqualityKeys ce = (GraphFetchCacheByEqualityKeys) c;
                if (!ce.isCacheUtilized() && mappingId.equals(ce.getMappingId()) && instanceSetId.equals(ce.getInstanceSetId())) {
                    unUtilizedCache = ce;
                    break;
                }
            }
        }
        if (unUtilizedCache != null) {
            unUtilizedCache.setSubTree(subTree);
            return unUtilizedCache;
        }
        return null;
    }
    return matchingUtilizedCache;
}
Also used : GraphFetchCache(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCache) GraphFetchCacheByEqualityKeys(org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCacheByEqualityKeys) CString(org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.CString)

Aggregations

GraphFetchCache (org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCache)3 List (java.util.List)2 Map (java.util.Map)2 Objects (java.util.Objects)2 Collectors (java.util.stream.Collectors)2 MutableList (org.eclipse.collections.api.list.MutableList)2 GraphFetchCrossAssociationKeys (org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCrossAssociationKeys)2 ConstantResult (org.finos.legend.engine.plan.execution.result.ConstantResult)2 ErrorResult (org.finos.legend.engine.plan.execution.result.ErrorResult)2 Result (org.finos.legend.engine.plan.execution.result.Result)2 CommonProfile (org.pac4j.core.profile.CommonProfile)2 Scope (io.opentracing.Scope)1 Span (io.opentracing.Span)1 GlobalTracer (io.opentracing.util.GlobalTracer)1 BufferedReader (java.io.BufferedReader)1 ByteArrayInputStream (java.io.ByteArrayInputStream)1 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 InputStreamReader (java.io.InputStreamReader)1 OutputStream (java.io.OutputStream)1