use of com.bakdata.conquery.models.forms.managed.ManagedForm in project conquery by bakdata.
the class QueryCleanupTask method execute.
@Override
public void execute(Map<String, List<String>> parameters, PrintWriter output) throws Exception {
Duration queryExpiration = this.queryExpiration;
if (parameters.containsKey(EXPIRATION_PARAM)) {
if (parameters.get(EXPIRATION_PARAM).size() > 1) {
log.warn("Will not respect more than one expiration time. Have `{}`", parameters.get(EXPIRATION_PARAM));
}
queryExpiration = Duration.parse(parameters.get(EXPIRATION_PARAM).get(0));
}
if (queryExpiration == null) {
throw new IllegalArgumentException("Query Expiration may not be null");
}
log.info("Starting deletion of queries older than {} of {}", queryExpiration, storage.getAllExecutions().size());
// Iterate for as long as no changes are needed (this is because queries can be referenced by other queries)
while (true) {
final QueryUtils.AllReusedFinder reusedChecker = new QueryUtils.AllReusedFinder();
Set<ManagedExecution<?>> toDelete = new HashSet<>();
for (ManagedExecution<?> execution : storage.getAllExecutions()) {
// Gather all referenced queries via reused checker.
if (execution instanceof ManagedQuery) {
((ManagedQuery) execution).getQuery().visit(reusedChecker);
} else if (execution instanceof ManagedForm) {
((ManagedForm) execution).getFlatSubQueries().values().forEach(q -> q.getQuery().visit(reusedChecker));
}
if (execution.isShared()) {
continue;
}
log.trace("{} is not shared", execution.getId());
if (ArrayUtils.isNotEmpty(execution.getTags())) {
continue;
}
log.trace("{} has no tags", execution.getId());
if (execution.getLabel() != null && !isDefaultLabel(execution.getLabel())) {
continue;
}
log.trace("{} has no label", execution.getId());
if (LocalDateTime.now().minus(queryExpiration).isBefore(execution.getCreationTime())) {
continue;
}
log.trace("{} is not older than {}.", execution.getId(), queryExpiration);
toDelete.add(execution);
}
// remove all queries referenced in reused queries.
final Collection<ManagedExecution<?>> referenced = reusedChecker.getReusedElements().stream().map(CQReusedQuery::getQueryId).map(storage::getExecution).collect(Collectors.toSet());
toDelete.removeAll(referenced);
if (toDelete.isEmpty()) {
log.info("No queries to delete");
break;
}
log.info("Deleting {} Executions", toDelete.size());
for (ManagedExecution<?> execution : toDelete) {
log.trace("Deleting Execution[{}]", execution.getId());
storage.removeExecution(execution.getId());
}
}
}
use of com.bakdata.conquery.models.forms.managed.ManagedForm in project conquery by bakdata.
the class ResultArrowProcessor method getArrowResult.
public static <E extends ManagedExecution<?> & SingleTableResult> Response getArrowResult(Function<OutputStream, Function<VectorSchemaRoot, ArrowWriter>> writerProducer, Subject subject, E exec, Dataset dataset, DatasetRegistry datasetRegistry, boolean pretty, String fileExtension, MediaType mediaType, ConqueryConfig config) {
final Namespace namespace = datasetRegistry.get(dataset.getId());
ConqueryMDC.setLocation(subject.getName());
log.info("Downloading results for {} on dataset {}", exec, dataset);
subject.authorize(dataset, Ability.READ);
subject.authorize(dataset, Ability.DOWNLOAD);
subject.authorize(exec, Ability.READ);
// Check if subject is permitted to download on all datasets that were referenced by the query
authorizeDownloadDatasets(subject, exec);
if (!(exec instanceof ManagedQuery || (exec instanceof ManagedForm && ((ManagedForm) exec).getSubQueries().size() == 1))) {
return Response.status(HttpStatus.SC_UNPROCESSABLE_ENTITY, "Execution result is not a single Table").build();
}
// Get the locale extracted by the LocaleFilter
IdPrinter idPrinter = config.getFrontend().getQueryUpload().getIdPrinter(subject, exec, namespace);
final Locale locale = I18n.LOCALE.get();
PrintSettings settings = new PrintSettings(pretty, locale, datasetRegistry, config, idPrinter::createId);
// Collect ResultInfos for id columns and result columns
final List<ResultInfo> resultInfosId = config.getFrontend().getQueryUpload().getIdResultInfos();
final List<ResultInfo> resultInfosExec = exec.getResultInfos();
StreamingOutput out = output -> renderToStream(writerProducer.apply(output), settings, config.getArrow().getBatchSize(), resultInfosId, resultInfosExec, exec.streamResults());
return makeResponseWithFileName(out, exec.getLabelWithoutAutoLabelSuffix(), fileExtension, mediaType, ResultUtil.ContentDispositionOption.ATTACHMENT);
}
use of com.bakdata.conquery.models.forms.managed.ManagedForm in project conquery by bakdata.
the class DefaultLabelTest method autoLabelExportForm.
@ParameterizedTest
@CsvSource({ "de,Datenexport 2020-10-30 12:37", "en,Data Export 2020-10-30 12:37" })
void autoLabelExportForm(Locale locale, String autoLabel) {
I18n.LOCALE.set(locale);
ExportForm form = new ExportForm();
ManagedForm mForm = form.toManagedExecution(user, DATASET);
mForm.setCreationTime(LocalDateTime.of(2020, 10, 30, 12, 37));
mForm.setLabel(mForm.makeAutoLabel(getPrintSettings(locale)));
assertThat(mForm.getLabel()).isEqualTo(autoLabel + AUTO_LABEL_SUFFIX);
assertThat(mForm.getLabelWithoutAutoLabelSuffix()).isEqualTo(autoLabel);
}
Aggregations