Search in sources :

Example 1 with QueryPlanContext

use of com.bakdata.conquery.models.query.QueryPlanContext in project conquery by bakdata.

the class CQConcept method createQueryPlan.

@Override
public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) {
    final List<Aggregator<?>> conceptAggregators = createAggregators(plan, selects);
    List<QPNode> tableNodes = new ArrayList<>();
    for (CQTable table : tables) {
        List<FilterNode<?>> filters = table.getFilters().stream().map(FilterValue::createNode).collect(Collectors.toList());
        // add filter to children
        List<Aggregator<?>> aggregators = new ArrayList<>();
        aggregators.addAll(conceptAggregators);
        final List<Aggregator<?>> connectorAggregators = createAggregators(plan, table.getSelects());
        // Exists aggregators hold a reference to their parent FiltersNode so they need to be treated separately.
        // They also don't need aggregation as they simply imitate their reference.
        List<ExistsAggregator> existsAggregators = connectorAggregators.stream().filter(ExistsAggregator.class::isInstance).map(ExistsAggregator.class::cast).collect(Collectors.toList());
        aggregators.addAll(connectorAggregators);
        aggregators.removeIf(ExistsAggregator.class::isInstance);
        List<Aggregator<CDateSet>> eventDateUnionAggregators = aggregateEventDates ? List.of(new EventDateUnionAggregator(Set.of(table.getConnector().getTable()))) : Collections.emptyList();
        aggregators.addAll(eventDateUnionAggregators);
        final QPNode filtersNode = getConcept().createConceptQuery(context, filters, aggregators, eventDateUnionAggregators);
        // Link up the ExistsAggregators to the node
        existsAggregators.forEach(agg -> agg.setReference(filtersNode));
        // Select if matching secondaryId available
        final boolean hasSelectedSecondaryId = Arrays.stream(table.getConnector().getTable().getColumns()).map(Column::getSecondaryId).filter(Objects::nonNull).anyMatch(o -> Objects.equals(context.getSelectedSecondaryId(), o));
        final Column validityDateColumn = selectValidityDateColumn(table);
        final ConceptNode node = new ConceptNode(// TODO Don't set validity node, when no validity column exists. See workaround for this and remove it: https://github.com/bakdata/conquery/pull/1362
        new ValidityDateNode(validityDateColumn, filtersNode), elements, table, // if the node is excluded, don't pass it into the Node.
        !excludeFromSecondaryId && hasSelectedSecondaryId ? context.getSelectedSecondaryId() : null);
        tableNodes.add(node);
    }
    // We always merge on concept level
    final QPNode outNode = OrNode.of(tableNodes, aggregateEventDates ? DateAggregationAction.MERGE : DateAggregationAction.BLOCK);
    // Link concept-level Exists-select to outer node.
    conceptAggregators.stream().filter(aggregator -> aggregator instanceof ExistsAggregator).forEach(aggregator -> ((ExistsAggregator) aggregator).setReference(outNode));
    return outNode;
}
Also used : ConceptNode(com.bakdata.conquery.models.query.queryplan.specific.ConceptNode) JsonProperty(com.fasterxml.jackson.annotation.JsonProperty) Arrays(java.util.Arrays) ConceptElement(com.bakdata.conquery.models.datasets.concepts.ConceptElement) EventDateUnionAggregator(com.bakdata.conquery.models.query.queryplan.aggregators.specific.EventDateUnionAggregator) NotEmpty(javax.validation.constraints.NotEmpty) Valid(javax.validation.Valid) JsonAlias(com.fasterxml.jackson.annotation.JsonAlias) Locale(java.util.Locale) QPNode(com.bakdata.conquery.models.query.queryplan.QPNode) ToString(lombok.ToString) CQTable(com.bakdata.conquery.apiv1.query.concept.filter.CQTable) InternalOnly(com.bakdata.conquery.io.jackson.InternalOnly) ConceptQueryPlan(com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan) NamespacedIdentifiableHolding(com.bakdata.conquery.models.query.NamespacedIdentifiableHolding) CPSType(com.bakdata.conquery.io.cps.CPSType) SelectResultInfo(com.bakdata.conquery.models.query.resultinfo.SelectResultInfo) NsIdRefCollection(com.bakdata.conquery.io.jackson.serializer.NsIdRefCollection) Concept(com.bakdata.conquery.models.datasets.concepts.Concept) Set(java.util.Set) ConceptNode(com.bakdata.conquery.models.query.queryplan.specific.ConceptNode) NotNull(javax.validation.constraints.NotNull) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) FilterValue(com.bakdata.conquery.apiv1.query.concept.filter.FilterValue) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) JsonManagedReference(com.fasterxml.jackson.annotation.JsonManagedReference) Column(com.bakdata.conquery.models.datasets.Column) Setter(lombok.Setter) Getter(lombok.Getter) NamespacedIdentifiable(com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable) ArrayList(java.util.ArrayList) Select(com.bakdata.conquery.models.datasets.concepts.select.Select) JsonIgnore(com.fasterxml.jackson.annotation.JsonIgnore) DateAggregationMode(com.bakdata.conquery.models.query.DateAggregationMode) FilterNode(com.bakdata.conquery.models.query.queryplan.filter.FilterNode) ResultInfo(com.bakdata.conquery.models.query.resultinfo.ResultInfo) DateAggregationAction(com.bakdata.conquery.models.query.queryplan.DateAggregationAction) ValidityDateNode(com.bakdata.conquery.models.query.queryplan.specific.ValidityDateNode) ExportForm(com.bakdata.conquery.apiv1.forms.export_form.ExportForm) QueryPlanContext(com.bakdata.conquery.models.query.QueryPlanContext) ValidationMethod(io.dropwizard.validation.ValidationMethod) QueryResolveContext(com.bakdata.conquery.models.query.QueryResolveContext) OrNode(com.bakdata.conquery.models.query.queryplan.specific.OrNode) Aggregator(com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator) CQElement(com.bakdata.conquery.apiv1.query.CQElement) ExistsAggregator(com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator) Collections(java.util.Collections) CDateSet(com.bakdata.conquery.models.common.CDateSet) QPNode(com.bakdata.conquery.models.query.queryplan.QPNode) CQTable(com.bakdata.conquery.apiv1.query.concept.filter.CQTable) FilterNode(com.bakdata.conquery.models.query.queryplan.filter.FilterNode) EventDateUnionAggregator(com.bakdata.conquery.models.query.queryplan.aggregators.specific.EventDateUnionAggregator) ArrayList(java.util.ArrayList) EventDateUnionAggregator(com.bakdata.conquery.models.query.queryplan.aggregators.specific.EventDateUnionAggregator) Aggregator(com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator) ExistsAggregator(com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator) Column(com.bakdata.conquery.models.datasets.Column) ValidityDateNode(com.bakdata.conquery.models.query.queryplan.specific.ValidityDateNode) ExistsAggregator(com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator)

Example 2 with QueryPlanContext

use of com.bakdata.conquery.models.query.QueryPlanContext in project conquery by bakdata.

the class ExecuteForm method react.

@Override
public void react(Worker worker) throws Exception {
    log.info("Started Form {}", formId);
    // Execution might have been cancelled before so we uncancel it here.
    final QueryExecutor queryExecutor = worker.getQueryExecutor();
    queryExecutor.unsetQueryCancelled(formId);
    // Execute all plans.
    for (Entry<ManagedExecutionId, Query> entry : queries.entrySet()) {
        final Query query = entry.getValue();
        ShardResult result = createResult(worker, entry.getKey());
        // Before we start the query, we create it once to test if it will succeed before creating it multiple times for evaluation per core.
        try {
            query.createQueryPlan(new QueryPlanContext(worker));
        } catch (Exception e) {
            ConqueryError err = asConqueryError(e);
            log.warn("Failed to create query plans for {}.", formId, err);
            queryExecutor.sendFailureToManagerNode(result, err);
            return;
        }
        final QueryExecutionContext subQueryContext = new QueryExecutionContext(formId, queryExecutor, worker.getStorage(), worker.getBucketManager());
        if (!queryExecutor.execute(query, subQueryContext, result)) {
            return;
        }
    }
}
Also used : ConqueryError.asConqueryError(com.bakdata.conquery.models.error.ConqueryError.asConqueryError) ConqueryError(com.bakdata.conquery.models.error.ConqueryError) Query(com.bakdata.conquery.apiv1.query.Query) QueryExecutionContext(com.bakdata.conquery.models.query.QueryExecutionContext) QueryExecutor(com.bakdata.conquery.models.query.QueryExecutor) ManagedExecutionId(com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId) FormShardResult(com.bakdata.conquery.models.query.results.FormShardResult) ShardResult(com.bakdata.conquery.models.query.results.ShardResult) QueryPlanContext(com.bakdata.conquery.models.query.QueryPlanContext)

Example 3 with QueryPlanContext

use of com.bakdata.conquery.models.query.QueryPlanContext in project conquery by bakdata.

the class ExecuteQuery method react.

@Override
public void react(Worker worker) throws Exception {
    final ManagedExecutionId executionId = id;
    log.info("Started {} {}", query.getClass().getSimpleName(), executionId);
    // Execution might have been cancelled before so we uncancel it here.
    final QueryExecutor queryExecutor = worker.getQueryExecutor();
    queryExecutor.unsetQueryCancelled(executionId);
    final ShardResult result = createShardResult(worker);
    // Before we start the query, we create it once to test if it will succeed before creating it multiple times for evaluation per core.
    try {
        query.createQueryPlan(new QueryPlanContext(worker));
    } catch (Exception e) {
        ConqueryError err = asConqueryError(e);
        log.warn("Failed to create query plans for {}.", executionId, err);
        queryExecutor.sendFailureToManagerNode(result, err);
        return;
    }
    final QueryExecutionContext executionContext = new QueryExecutionContext(executionId, queryExecutor, worker.getStorage(), worker.getBucketManager());
    queryExecutor.execute(query, executionContext, result);
}
Also used : ConqueryError.asConqueryError(com.bakdata.conquery.models.error.ConqueryError.asConqueryError) ConqueryError(com.bakdata.conquery.models.error.ConqueryError) QueryExecutionContext(com.bakdata.conquery.models.query.QueryExecutionContext) QueryExecutor(com.bakdata.conquery.models.query.QueryExecutor) ManagedExecutionId(com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId) ShardResult(com.bakdata.conquery.models.query.results.ShardResult) QueryPlanContext(com.bakdata.conquery.models.query.QueryPlanContext)

Aggregations

QueryPlanContext (com.bakdata.conquery.models.query.QueryPlanContext)3 ConqueryError (com.bakdata.conquery.models.error.ConqueryError)2 ConqueryError.asConqueryError (com.bakdata.conquery.models.error.ConqueryError.asConqueryError)2 ManagedExecutionId (com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId)2 QueryExecutionContext (com.bakdata.conquery.models.query.QueryExecutionContext)2 QueryExecutor (com.bakdata.conquery.models.query.QueryExecutor)2 ShardResult (com.bakdata.conquery.models.query.results.ShardResult)2 ExportForm (com.bakdata.conquery.apiv1.forms.export_form.ExportForm)1 CQElement (com.bakdata.conquery.apiv1.query.CQElement)1 Query (com.bakdata.conquery.apiv1.query.Query)1 CQTable (com.bakdata.conquery.apiv1.query.concept.filter.CQTable)1 FilterValue (com.bakdata.conquery.apiv1.query.concept.filter.FilterValue)1 CPSType (com.bakdata.conquery.io.cps.CPSType)1 InternalOnly (com.bakdata.conquery.io.jackson.InternalOnly)1 NsIdRefCollection (com.bakdata.conquery.io.jackson.serializer.NsIdRefCollection)1 CDateSet (com.bakdata.conquery.models.common.CDateSet)1 Column (com.bakdata.conquery.models.datasets.Column)1 Concept (com.bakdata.conquery.models.datasets.concepts.Concept)1 ConceptElement (com.bakdata.conquery.models.datasets.concepts.ConceptElement)1 Select (com.bakdata.conquery.models.datasets.concepts.select.Select)1