Search in sources :

Example 1 with ManagedForm

use of com.bakdata.conquery.models.forms.managed.ManagedForm in project conquery by bakdata.

the class QueryCleanupTask method execute.

@Override
public void execute(Map<String, List<String>> parameters, PrintWriter output) throws Exception {
    Duration queryExpiration = this.queryExpiration;
    if (parameters.containsKey(EXPIRATION_PARAM)) {
        if (parameters.get(EXPIRATION_PARAM).size() > 1) {
            log.warn("Will not respect more than one expiration time. Have `{}`", parameters.get(EXPIRATION_PARAM));
        }
        queryExpiration = Duration.parse(parameters.get(EXPIRATION_PARAM).get(0));
    }
    if (queryExpiration == null) {
        throw new IllegalArgumentException("Query Expiration may not be null");
    }
    log.info("Starting deletion of queries older than {} of {}", queryExpiration, storage.getAllExecutions().size());
    // Iterate for as long as no changes are needed (this is because queries can be referenced by other queries)
    while (true) {
        final QueryUtils.AllReusedFinder reusedChecker = new QueryUtils.AllReusedFinder();
        Set<ManagedExecution<?>> toDelete = new HashSet<>();
        for (ManagedExecution<?> execution : storage.getAllExecutions()) {
            // Gather all referenced queries via reused checker.
            if (execution instanceof ManagedQuery) {
                ((ManagedQuery) execution).getQuery().visit(reusedChecker);
            } else if (execution instanceof ManagedForm) {
                ((ManagedForm) execution).getFlatSubQueries().values().forEach(q -> q.getQuery().visit(reusedChecker));
            }
            if (execution.isShared()) {
                continue;
            }
            log.trace("{} is not shared", execution.getId());
            if (ArrayUtils.isNotEmpty(execution.getTags())) {
                continue;
            }
            log.trace("{} has no tags", execution.getId());
            if (execution.getLabel() != null && !isDefaultLabel(execution.getLabel())) {
                continue;
            }
            log.trace("{} has no label", execution.getId());
            if (LocalDateTime.now().minus(queryExpiration).isBefore(execution.getCreationTime())) {
                continue;
            }
            log.trace("{} is not older than {}.", execution.getId(), queryExpiration);
            toDelete.add(execution);
        }
        // remove all queries referenced in reused queries.
        final Collection<ManagedExecution<?>> referenced = reusedChecker.getReusedElements().stream().map(CQReusedQuery::getQueryId).map(storage::getExecution).collect(Collectors.toSet());
        toDelete.removeAll(referenced);
        if (toDelete.isEmpty()) {
            log.info("No queries to delete");
            break;
        }
        log.info("Deleting {} Executions", toDelete.size());
        for (ManagedExecution<?> execution : toDelete) {
            log.trace("Deleting Execution[{}]", execution.getId());
            storage.removeExecution(execution.getId());
        }
    }
}
Also used : PrintWriter(java.io.PrintWriter) ManagedQuery(com.bakdata.conquery.models.query.ManagedQuery) ManagedForm(com.bakdata.conquery.models.forms.managed.ManagedForm) Predicate(java.util.function.Predicate) Collection(java.util.Collection) LocalDateTime(java.time.LocalDateTime) Set(java.util.Set) ArrayUtils(org.apache.commons.lang3.ArrayUtils) Collectors(java.util.stream.Collectors) HashSet(java.util.HashSet) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) ManagedExecution(com.bakdata.conquery.models.execution.ManagedExecution) Task(io.dropwizard.servlets.tasks.Task) Duration(java.time.Duration) Map(java.util.Map) QueryUtils(com.bakdata.conquery.util.QueryUtils) CQReusedQuery(com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery) Pattern(java.util.regex.Pattern) MetaStorage(com.bakdata.conquery.io.storage.MetaStorage) CQReusedQuery(com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery) Duration(java.time.Duration) QueryUtils(com.bakdata.conquery.util.QueryUtils) ManagedForm(com.bakdata.conquery.models.forms.managed.ManagedForm) ManagedQuery(com.bakdata.conquery.models.query.ManagedQuery) ManagedExecution(com.bakdata.conquery.models.execution.ManagedExecution) HashSet(java.util.HashSet)

Example 2 with ManagedForm

use of com.bakdata.conquery.models.forms.managed.ManagedForm in project conquery by bakdata.

the class ResultArrowProcessor method getArrowResult.

public static <E extends ManagedExecution<?> & SingleTableResult> Response getArrowResult(Function<OutputStream, Function<VectorSchemaRoot, ArrowWriter>> writerProducer, Subject subject, E exec, Dataset dataset, DatasetRegistry datasetRegistry, boolean pretty, String fileExtension, MediaType mediaType, ConqueryConfig config) {
    final Namespace namespace = datasetRegistry.get(dataset.getId());
    ConqueryMDC.setLocation(subject.getName());
    log.info("Downloading results for {} on dataset {}", exec, dataset);
    subject.authorize(dataset, Ability.READ);
    subject.authorize(dataset, Ability.DOWNLOAD);
    subject.authorize(exec, Ability.READ);
    // Check if subject is permitted to download on all datasets that were referenced by the query
    authorizeDownloadDatasets(subject, exec);
    if (!(exec instanceof ManagedQuery || (exec instanceof ManagedForm && ((ManagedForm) exec).getSubQueries().size() == 1))) {
        return Response.status(HttpStatus.SC_UNPROCESSABLE_ENTITY, "Execution result is not a single Table").build();
    }
    // Get the locale extracted by the LocaleFilter
    IdPrinter idPrinter = config.getFrontend().getQueryUpload().getIdPrinter(subject, exec, namespace);
    final Locale locale = I18n.LOCALE.get();
    PrintSettings settings = new PrintSettings(pretty, locale, datasetRegistry, config, idPrinter::createId);
    // Collect ResultInfos for id columns and result columns
    final List<ResultInfo> resultInfosId = config.getFrontend().getQueryUpload().getIdResultInfos();
    final List<ResultInfo> resultInfosExec = exec.getResultInfos();
    StreamingOutput out = output -> renderToStream(writerProducer.apply(output), settings, config.getArrow().getBatchSize(), resultInfosId, resultInfosExec, exec.streamResults());
    return makeResponseWithFileName(out, exec.getLabelWithoutAutoLabelSuffix(), fileExtension, mediaType, ResultUtil.ContentDispositionOption.ATTACHMENT);
}
Also used : IdPrinter(com.bakdata.conquery.models.identifiable.mapping.IdPrinter) Locale(java.util.Locale) ManagedQuery(com.bakdata.conquery.models.query.ManagedQuery) ConqueryConfig(com.bakdata.conquery.models.config.ConqueryConfig) ManagedForm(com.bakdata.conquery.models.forms.managed.ManagedForm) Subject(com.bakdata.conquery.models.auth.entities.Subject) AuthorizationHelper.authorizeDownloadDatasets(com.bakdata.conquery.models.auth.AuthorizationHelper.authorizeDownloadDatasets) HttpStatus(org.apache.http.HttpStatus) Function(java.util.function.Function) ArrayList(java.util.ArrayList) PrintSettings(com.bakdata.conquery.models.query.PrintSettings) UtilityClass(lombok.experimental.UtilityClass) MediaType(javax.ws.rs.core.MediaType) ManagedExecution(com.bakdata.conquery.models.execution.ManagedExecution) ArrowWriter(org.apache.arrow.vector.ipc.ArrowWriter) Locale(java.util.Locale) ArrowRenderer.renderToStream(com.bakdata.conquery.io.result.arrow.ArrowRenderer.renderToStream) I18n(com.bakdata.conquery.models.i18n.I18n) IdPrinter(com.bakdata.conquery.models.identifiable.mapping.IdPrinter) ResultInfo(com.bakdata.conquery.models.query.resultinfo.ResultInfo) OutputStream(java.io.OutputStream) ResultUtil(com.bakdata.conquery.io.result.ResultUtil) ConqueryMDC(com.bakdata.conquery.util.io.ConqueryMDC) ResultUtil.makeResponseWithFileName(com.bakdata.conquery.io.result.ResultUtil.makeResponseWithFileName) VectorSchemaRoot(org.apache.arrow.vector.VectorSchemaRoot) StreamingOutput(javax.ws.rs.core.StreamingOutput) SingleTableResult(com.bakdata.conquery.models.query.SingleTableResult) Dataset(com.bakdata.conquery.models.datasets.Dataset) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) Response(javax.ws.rs.core.Response) Ability(com.bakdata.conquery.models.auth.permissions.Ability) DatasetRegistry(com.bakdata.conquery.models.worker.DatasetRegistry) Namespace(com.bakdata.conquery.models.worker.Namespace) ManagedForm(com.bakdata.conquery.models.forms.managed.ManagedForm) PrintSettings(com.bakdata.conquery.models.query.PrintSettings) StreamingOutput(javax.ws.rs.core.StreamingOutput) ManagedQuery(com.bakdata.conquery.models.query.ManagedQuery) ResultInfo(com.bakdata.conquery.models.query.resultinfo.ResultInfo) Namespace(com.bakdata.conquery.models.worker.Namespace)

Example 3 with ManagedForm

use of com.bakdata.conquery.models.forms.managed.ManagedForm in project conquery by bakdata.

the class DefaultLabelTest method autoLabelExportForm.

@ParameterizedTest
@CsvSource({ "de,Datenexport 2020-10-30 12:37", "en,Data Export 2020-10-30 12:37" })
void autoLabelExportForm(Locale locale, String autoLabel) {
    I18n.LOCALE.set(locale);
    ExportForm form = new ExportForm();
    ManagedForm mForm = form.toManagedExecution(user, DATASET);
    mForm.setCreationTime(LocalDateTime.of(2020, 10, 30, 12, 37));
    mForm.setLabel(mForm.makeAutoLabel(getPrintSettings(locale)));
    assertThat(mForm.getLabel()).isEqualTo(autoLabel + AUTO_LABEL_SUFFIX);
    assertThat(mForm.getLabelWithoutAutoLabelSuffix()).isEqualTo(autoLabel);
}
Also used : ManagedForm(com.bakdata.conquery.models.forms.managed.ManagedForm) ExportForm(com.bakdata.conquery.apiv1.forms.export_form.ExportForm) CsvSource(org.junit.jupiter.params.provider.CsvSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Aggregations

ManagedForm (com.bakdata.conquery.models.forms.managed.ManagedForm)3 ManagedExecution (com.bakdata.conquery.models.execution.ManagedExecution)2 ManagedQuery (com.bakdata.conquery.models.query.ManagedQuery)2 List (java.util.List)2 Slf4j (lombok.extern.slf4j.Slf4j)2 ExportForm (com.bakdata.conquery.apiv1.forms.export_form.ExportForm)1 CQReusedQuery (com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery)1 ResultUtil (com.bakdata.conquery.io.result.ResultUtil)1 ResultUtil.makeResponseWithFileName (com.bakdata.conquery.io.result.ResultUtil.makeResponseWithFileName)1 ArrowRenderer.renderToStream (com.bakdata.conquery.io.result.arrow.ArrowRenderer.renderToStream)1 MetaStorage (com.bakdata.conquery.io.storage.MetaStorage)1 AuthorizationHelper.authorizeDownloadDatasets (com.bakdata.conquery.models.auth.AuthorizationHelper.authorizeDownloadDatasets)1 Subject (com.bakdata.conquery.models.auth.entities.Subject)1 Ability (com.bakdata.conquery.models.auth.permissions.Ability)1 ConqueryConfig (com.bakdata.conquery.models.config.ConqueryConfig)1 Dataset (com.bakdata.conquery.models.datasets.Dataset)1 I18n (com.bakdata.conquery.models.i18n.I18n)1 IdPrinter (com.bakdata.conquery.models.identifiable.mapping.IdPrinter)1 PrintSettings (com.bakdata.conquery.models.query.PrintSettings)1 SingleTableResult (com.bakdata.conquery.models.query.SingleTableResult)1