Search in sources :

Example 1 with CQReusedQuery

use of com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery in project conquery by bakdata.

the class ManagedQuery method makeDefaultLabel.

/**
 * Creates a default label based on the submitted {@link QueryDescription}.
 * The Label is customized by mentioning that a description contained a
 * {@link CQExternal}, {@link CQReusedQuery} or {@link CQConcept}, in this order.
 * In case of one ore more {@link CQConcept} the distinct labels of the concepts are chosen
 * and concatinated until a length of {@value #MAX_CONCEPT_LABEL_CONCAT_LENGTH} is reached.
 * All further labels are dropped.
 */
@Override
protected String makeDefaultLabel(PrintSettings cfg) {
    final StringBuilder sb = new StringBuilder();
    final Map<Class<? extends Visitable>, List<Visitable>> sortedContents = Visitable.stream(query).collect(Collectors.groupingBy(Visitable::getClass));
    int sbStartSize = sb.length();
    // Check for CQExternal
    List<Visitable> externals = sortedContents.getOrDefault(CQExternal.class, Collections.emptyList());
    if (!externals.isEmpty()) {
        if (sb.length() > 0) {
            sb.append(" ");
        }
        sb.append(C10N.get(CQElementC10n.class, I18n.LOCALE.get()).external());
    }
    // Check for CQReused
    if (sortedContents.containsKey(CQReusedQuery.class)) {
        if (sb.length() > 0) {
            sb.append(" ");
        }
        sb.append(C10N.get(CQElementC10n.class, I18n.LOCALE.get()).reused());
    }
    // Check for CQConcept
    if (sortedContents.containsKey(CQConcept.class)) {
        if (sb.length() > 0) {
            sb.append(" ");
        }
        // Track length of text we are appending for concepts.
        final AtomicInteger length = new AtomicInteger();
        sortedContents.get(CQConcept.class).stream().map(CQConcept.class::cast).map(c -> makeLabelWithRootAndChild(c, cfg)).filter(Predicate.not(Strings::isNullOrEmpty)).distinct().takeWhile(elem -> length.addAndGet(elem.length()) < MAX_CONCEPT_LABEL_CONCAT_LENGTH).forEach(label -> sb.append(label).append(" "));
        // Last entry will output one Space that we don't want
        if (sb.length() > 0) {
            sb.deleteCharAt(sb.length() - 1);
        }
        // If not all Concept could be included in the name, point that out
        if (length.get() > MAX_CONCEPT_LABEL_CONCAT_LENGTH) {
            sb.append(" ").append(C10N.get(CQElementC10n.class, I18n.LOCALE.get()).furtherConcepts());
        }
    }
    // Fallback to id if nothing could be extracted from the query description
    if (sbStartSize == sb.length()) {
        sb.append(getId().getExecution());
    }
    return sb.toString();
}
Also used : ConqueryConfig(com.bakdata.conquery.models.config.ConqueryConfig) ExecutionStatus(com.bakdata.conquery.apiv1.ExecutionStatus) ManagedExecution(com.bakdata.conquery.models.execution.ManagedExecution) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Locale(java.util.Locale) Map(java.util.Map) ToString(lombok.ToString) QueryDescription(com.bakdata.conquery.apiv1.query.QueryDescription) ResultType(com.bakdata.conquery.models.externalservice.ResultType) ShardResult(com.bakdata.conquery.models.query.results.ShardResult) WorkerMessage(com.bakdata.conquery.models.messages.namespaces.WorkerMessage) CPSType(com.bakdata.conquery.io.cps.CPSType) Predicate(java.util.function.Predicate) NonNull(lombok.NonNull) SecondaryIdQuery(com.bakdata.conquery.apiv1.query.SecondaryIdQuery) Set(java.util.Set) User(com.bakdata.conquery.models.auth.entities.User) Collectors(java.util.stream.Collectors) Dataset(com.bakdata.conquery.models.datasets.Dataset) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) Stream(java.util.stream.Stream) C10N(c10n.C10N) CQReusedQuery(com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery) MetaStorage(com.bakdata.conquery.io.storage.MetaStorage) Namespace(com.bakdata.conquery.models.worker.Namespace) Setter(lombok.Setter) Getter(lombok.Getter) ExecutionState(com.bakdata.conquery.models.execution.ExecutionState) Subject(com.bakdata.conquery.models.auth.entities.Subject) EntityResult(com.bakdata.conquery.models.query.results.EntityResult) NamespacedIdentifiable(com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable) ArrayList(java.util.ArrayList) Strings(com.google.common.base.Strings) ExecuteQuery(com.bakdata.conquery.models.messages.namespaces.specific.ExecuteQuery) CQElementC10n(com.bakdata.conquery.internationalization.CQElementC10n) JsonIgnore(com.fasterxml.jackson.annotation.JsonIgnore) I18n(com.bakdata.conquery.models.i18n.I18n) NamespacedIdentifiableCollector(com.bakdata.conquery.util.QueryUtils.NamespacedIdentifiableCollector) ResultInfo(com.bakdata.conquery.models.query.resultinfo.ResultInfo) UniqueNamer(com.bakdata.conquery.models.query.resultinfo.UniqueNamer) CQExternal(com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal) Consumer(java.util.function.Consumer) FullExecutionStatus(com.bakdata.conquery.apiv1.FullExecutionStatus) DatasetRegistry(com.bakdata.conquery.models.worker.DatasetRegistry) Query(com.bakdata.conquery.apiv1.query.Query) Preconditions(com.google.common.base.Preconditions) CQConcept(com.bakdata.conquery.apiv1.query.concept.specific.CQConcept) Collections(java.util.Collections) NoArgsConstructor(lombok.NoArgsConstructor) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) List(java.util.List) ArrayList(java.util.ArrayList) CQConcept(com.bakdata.conquery.apiv1.query.concept.specific.CQConcept) Strings(com.google.common.base.Strings)

Example 2 with CQReusedQuery

use of com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery in project conquery by bakdata.

the class QueryCleanupTask method execute.

@Override
public void execute(Map<String, List<String>> parameters, PrintWriter output) throws Exception {
    Duration queryExpiration = this.queryExpiration;
    if (parameters.containsKey(EXPIRATION_PARAM)) {
        if (parameters.get(EXPIRATION_PARAM).size() > 1) {
            log.warn("Will not respect more than one expiration time. Have `{}`", parameters.get(EXPIRATION_PARAM));
        }
        queryExpiration = Duration.parse(parameters.get(EXPIRATION_PARAM).get(0));
    }
    if (queryExpiration == null) {
        throw new IllegalArgumentException("Query Expiration may not be null");
    }
    log.info("Starting deletion of queries older than {} of {}", queryExpiration, storage.getAllExecutions().size());
    // Iterate for as long as no changes are needed (this is because queries can be referenced by other queries)
    while (true) {
        final QueryUtils.AllReusedFinder reusedChecker = new QueryUtils.AllReusedFinder();
        Set<ManagedExecution<?>> toDelete = new HashSet<>();
        for (ManagedExecution<?> execution : storage.getAllExecutions()) {
            // Gather all referenced queries via reused checker.
            if (execution instanceof ManagedQuery) {
                ((ManagedQuery) execution).getQuery().visit(reusedChecker);
            } else if (execution instanceof ManagedForm) {
                ((ManagedForm) execution).getFlatSubQueries().values().forEach(q -> q.getQuery().visit(reusedChecker));
            }
            if (execution.isShared()) {
                continue;
            }
            log.trace("{} is not shared", execution.getId());
            if (ArrayUtils.isNotEmpty(execution.getTags())) {
                continue;
            }
            log.trace("{} has no tags", execution.getId());
            if (execution.getLabel() != null && !isDefaultLabel(execution.getLabel())) {
                continue;
            }
            log.trace("{} has no label", execution.getId());
            if (LocalDateTime.now().minus(queryExpiration).isBefore(execution.getCreationTime())) {
                continue;
            }
            log.trace("{} is not older than {}.", execution.getId(), queryExpiration);
            toDelete.add(execution);
        }
        // remove all queries referenced in reused queries.
        final Collection<ManagedExecution<?>> referenced = reusedChecker.getReusedElements().stream().map(CQReusedQuery::getQueryId).map(storage::getExecution).collect(Collectors.toSet());
        toDelete.removeAll(referenced);
        if (toDelete.isEmpty()) {
            log.info("No queries to delete");
            break;
        }
        log.info("Deleting {} Executions", toDelete.size());
        for (ManagedExecution<?> execution : toDelete) {
            log.trace("Deleting Execution[{}]", execution.getId());
            storage.removeExecution(execution.getId());
        }
    }
}
Also used : PrintWriter(java.io.PrintWriter) ManagedQuery(com.bakdata.conquery.models.query.ManagedQuery) ManagedForm(com.bakdata.conquery.models.forms.managed.ManagedForm) Predicate(java.util.function.Predicate) Collection(java.util.Collection) LocalDateTime(java.time.LocalDateTime) Set(java.util.Set) ArrayUtils(org.apache.commons.lang3.ArrayUtils) Collectors(java.util.stream.Collectors) HashSet(java.util.HashSet) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) ManagedExecution(com.bakdata.conquery.models.execution.ManagedExecution) Task(io.dropwizard.servlets.tasks.Task) Duration(java.time.Duration) Map(java.util.Map) QueryUtils(com.bakdata.conquery.util.QueryUtils) CQReusedQuery(com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery) Pattern(java.util.regex.Pattern) MetaStorage(com.bakdata.conquery.io.storage.MetaStorage) CQReusedQuery(com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery) Duration(java.time.Duration) QueryUtils(com.bakdata.conquery.util.QueryUtils) ManagedForm(com.bakdata.conquery.models.forms.managed.ManagedForm) ManagedQuery(com.bakdata.conquery.models.query.ManagedQuery) ManagedExecution(com.bakdata.conquery.models.execution.ManagedExecution) HashSet(java.util.HashSet)

Example 3 with CQReusedQuery

use of com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery in project conquery by bakdata.

the class DefaultLabelTest method autoLabelComplexQuery.

@ParameterizedTest
@CsvSource({ "de,Hochgeladene-Liste Anfrage Concept1 Concept2 und weitere", "en,Uploaded-List Query Concept1 Concept2 and further" })
void autoLabelComplexQuery(Locale locale, String autoLabel) {
    I18n.LOCALE.set(locale);
    final ManagedQuery managedQuery = new ManagedQuery(null, null, DATASET);
    managedQuery.setQueryId(UUID.randomUUID());
    CQAnd and = new CQAnd();
    CQConcept concept1 = makeCQConcept("Concept1");
    CQConcept concept2 = makeCQConcept("Concept2");
    CQConcept concept3 = makeCQConcept("Concept3veryveryveryveryveryveryveryverylooooooooooooooooooooonglabel");
    and.setChildren(List.of(new CQExternal(List.of(), new String[0][0]), new CQReusedQuery(managedQuery.getId()), concept1, concept2, concept3));
    ConceptQuery cq = new ConceptQuery(and);
    ManagedQuery mQuery = cq.toManagedExecution(user, DATASET);
    mQuery.setLabel(mQuery.makeAutoLabel(getPrintSettings(locale)));
    assertThat(mQuery.getLabel()).isEqualTo(autoLabel + AUTO_LABEL_SUFFIX);
    assertThat(mQuery.getLabelWithoutAutoLabelSuffix()).isEqualTo(autoLabel);
}
Also used : CQReusedQuery(com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery) CQConcept(com.bakdata.conquery.apiv1.query.concept.specific.CQConcept) CQExternal(com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal) ManagedQuery(com.bakdata.conquery.models.query.ManagedQuery) CQAnd(com.bakdata.conquery.apiv1.query.concept.specific.CQAnd) ConceptQuery(com.bakdata.conquery.apiv1.query.ConceptQuery) CsvSource(org.junit.jupiter.params.provider.CsvSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 4 with CQReusedQuery

use of com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery in project conquery by bakdata.

the class QueryCleanupTaskTest method reusedNoNames.

@Test
void reusedNoNames() throws Exception {
    assertThat(STORAGE.getAllExecutions()).isEmpty();
    final ManagedQuery managedQuery = createManagedQuery();
    final ManagedQuery managedQueryReused = createManagedQuery();
    managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId())));
    new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null);
    assertThat(STORAGE.getAllExecutions()).isEmpty();
}
Also used : CQReusedQuery(com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery) ManagedQuery(com.bakdata.conquery.models.query.ManagedQuery) ConceptQuery(com.bakdata.conquery.apiv1.query.ConceptQuery) Test(org.junit.jupiter.api.Test)

Example 5 with CQReusedQuery

use of com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery in project conquery by bakdata.

the class DefaultLabelTest method autoLabelReusedQuery.

@ParameterizedTest
@CsvSource({ "de,Anfrage", "en,Query" })
void autoLabelReusedQuery(Locale locale, String autoLabel) {
    I18n.LOCALE.set(locale);
    final ManagedQuery managedQuery = new ManagedQuery(null, null, DATASET);
    managedQuery.setQueryId(UUID.randomUUID());
    CQReusedQuery reused = new CQReusedQuery(managedQuery.getId());
    ConceptQuery cq = new ConceptQuery(reused);
    ManagedQuery mQuery = cq.toManagedExecution(user, DATASET);
    mQuery.setLabel(mQuery.makeAutoLabel(getPrintSettings(locale)));
    assertThat(mQuery.getLabel()).isEqualTo(autoLabel + AUTO_LABEL_SUFFIX);
    assertThat(mQuery.getLabelWithoutAutoLabelSuffix()).isEqualTo(autoLabel);
}
Also used : CQReusedQuery(com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery) ManagedQuery(com.bakdata.conquery.models.query.ManagedQuery) ConceptQuery(com.bakdata.conquery.apiv1.query.ConceptQuery) CsvSource(org.junit.jupiter.params.provider.CsvSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Aggregations

CQReusedQuery (com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery)12 ManagedQuery (com.bakdata.conquery.models.query.ManagedQuery)11 ConceptQuery (com.bakdata.conquery.apiv1.query.ConceptQuery)10 Test (org.junit.jupiter.api.Test)5 CQConcept (com.bakdata.conquery.apiv1.query.concept.specific.CQConcept)4 CQAnd (com.bakdata.conquery.apiv1.query.concept.specific.CQAnd)3 CQExternal (com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal)3 MetaStorage (com.bakdata.conquery.io.storage.MetaStorage)3 FullExecutionStatus (com.bakdata.conquery.apiv1.FullExecutionStatus)2 SecondaryIdQuery (com.bakdata.conquery.apiv1.query.SecondaryIdQuery)2 User (com.bakdata.conquery.models.auth.entities.User)2 Dataset (com.bakdata.conquery.models.datasets.Dataset)2 ManagedExecution (com.bakdata.conquery.models.execution.ManagedExecution)2 List (java.util.List)2 Map (java.util.Map)2 Set (java.util.Set)2 Predicate (java.util.function.Predicate)2 Collectors (java.util.stream.Collectors)2 Slf4j (lombok.extern.slf4j.Slf4j)2 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)2