Search in sources :

Example 1 with ManagedExecutionId

use of com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId in project conquery by bakdata.

the class AbstractQueryEngineTest method executeTest.

@Override
public void executeTest(StandaloneSupport standaloneSupport) throws IOException {
    Query query = getQuery();
    assertThat(standaloneSupport.getValidator().validate(query)).describedAs("Query Validation Errors").isEmpty();
    log.info("{} QUERY INIT", getLabel());
    final User testUser = standaloneSupport.getTestUser();
    final ManagedExecutionId executionId = IntegrationUtils.assertQueryResult(standaloneSupport, query, -1, ExecutionState.DONE, testUser, 201);
    final ManagedQuery execution = (ManagedQuery) standaloneSupport.getMetaStorage().getExecution(executionId);
    // check result info size
    List<ResultInfo> resultInfos = execution.getResultInfos();
    assertThat(execution.streamResults().flatMap(EntityResult::streamValues)).as("Should have same size as result infos").allSatisfy(v -> assertThat(v).hasSameSizeAs(resultInfos));
    // Get the actual response and compare with expected result.
    final Response csvResponse = standaloneSupport.getClient().target(HierarchyHelper.hierarchicalPath(standaloneSupport.defaultApiURIBuilder(), ResultCsvResource.class, "getAsCsv").buildFromMap(Map.of(DATASET, standaloneSupport.getDataset().getName(), QUERY, execution.getId().toString()))).queryParam("pretty", false).request(AdditionalMediaTypes.CSV).acceptLanguage(Locale.ENGLISH).get();
    List<String> actual = In.stream(((InputStream) csvResponse.getEntity())).readLines();
    ResourceFile expectedCsv = getExpectedCsv();
    List<String> expected = In.stream(expectedCsv.stream()).readLines();
    assertThat(actual).as("Results for %s are not as expected.", this).containsExactlyInAnyOrderElementsOf(expected);
    // check that getLastResultCount returns the correct size
    if (execution.streamResults().noneMatch(MultilineEntityResult.class::isInstance)) {
        assertThat(execution.getLastResultCount()).as("Result count for %s is not as expected.", this).isEqualTo(expected.size() - 1);
    }
    log.info("INTEGRATION TEST SUCCESSFUL {} {} on {} rows", getClass().getSimpleName(), this, expected.size());
}
Also used : Response(javax.ws.rs.core.Response) ResourceFile(com.bakdata.conquery.integration.common.ResourceFile) User(com.bakdata.conquery.models.auth.entities.User) ManagedQuery(com.bakdata.conquery.models.query.ManagedQuery) Query(com.bakdata.conquery.apiv1.query.Query) ManagedExecutionId(com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId) ManagedQuery(com.bakdata.conquery.models.query.ManagedQuery) ResultInfo(com.bakdata.conquery.models.query.resultinfo.ResultInfo) MultilineEntityResult(com.bakdata.conquery.models.query.results.MultilineEntityResult)

Example 2 with ManagedExecutionId

use of com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId in project conquery by bakdata.

the class StoredQueriesProcessorTest method mockUser.

private static User mockUser(int id, List<ManagedExecutionId> allowedQueryIds) {
    final User user = new User("user" + id, null, STORAGE);
    STORAGE.addUser(user);
    for (ManagedExecutionId queryId : allowedQueryIds) {
        user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, queryId));
    }
    return user;
}
Also used : User(com.bakdata.conquery.models.auth.entities.User) ManagedExecutionId(com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId)

Example 3 with ManagedExecutionId

use of com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId in project conquery by bakdata.

the class PermissionCleanupTask method deleteQueryPermissionsWithMissingRef.

/**
 * Deletes permission that reference non-existing executions.
 *
 * @return The number of deleted permissions.
 */
public static int deleteQueryPermissionsWithMissingRef(MetaStorage storage, Iterable<? extends PermissionOwner<?>> owners) {
    int countDeleted = 0;
    // Do the loop-di-loop
    for (PermissionOwner<?> owner : owners) {
        Set<ConqueryPermission> permissions = owner.getPermissions();
        for (Permission permission : permissions) {
            WildcardPermission wpermission = getAsWildcardPermission(permission);
            if (wpermission == null) {
                continue;
            }
            if (!wpermission.getDomains().contains(ExecutionPermission.DOMAIN.toLowerCase())) {
                // Skip Permissions that do not reference an Execution/Query
                continue;
            }
            // Handle multiple references to instances
            Set<String> validRef = new HashSet<>();
            for (String sId : wpermission.getInstances()) {
                ManagedExecutionId mId = ManagedExecutionId.Parser.INSTANCE.parse(sId);
                if (storage.getExecution(mId) != null) {
                    // Execution exists -- it is a valid reference
                    validRef.add(mId.toString());
                }
            }
            if (!validRef.isEmpty()) {
                if (wpermission.getInstances().size() == validRef.size()) {
                    // All are valid, nothing changed proceed with the next permission
                    continue;
                }
                // Create a new Permission that only contains valid references
                WildcardPermission reducedPermission = new WildcardPermission(List.of(wpermission.getDomains(), wpermission.getAbilities(), validRef), wpermission.getCreationTime());
                owner.addPermission(reducedPermission);
            }
            // Delete the old permission that containes both valid and invalid references
            owner.removePermission(wpermission);
            countDeleted++;
        }
    }
    return countDeleted;
}
Also used : ConqueryPermission(com.bakdata.conquery.models.auth.permissions.ConqueryPermission) ConqueryPermission(com.bakdata.conquery.models.auth.permissions.ConqueryPermission) Permission(org.apache.shiro.authz.Permission) WildcardPermission(com.bakdata.conquery.models.auth.permissions.WildcardPermission) ExecutionPermission(com.bakdata.conquery.models.auth.permissions.ExecutionPermission) FormConfigPermission(com.bakdata.conquery.models.auth.permissions.FormConfigPermission) ManagedExecutionId(com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId) WildcardPermission(com.bakdata.conquery.models.auth.permissions.WildcardPermission) HashSet(java.util.HashSet)

Example 4 with ManagedExecutionId

use of com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId in project conquery by bakdata.

the class QueryProcessor method patchQuery.

public void patchQuery(Subject subject, ManagedExecution<?> execution, MetaDataPatch patch) {
    log.info("Patching {} ({}) with patch: {}", execution.getClass().getSimpleName(), execution, patch);
    patch.applyTo(execution, storage, subject);
    storage.updateExecution(execution);
    // TODO remove this, since we don't translate anymore
    // Patch this query in other datasets
    List<Dataset> remainingDatasets = datasetRegistry.getAllDatasets();
    remainingDatasets.remove(execution.getDataset());
    for (Dataset dataset : remainingDatasets) {
        ManagedExecutionId id = new ManagedExecutionId(dataset.getId(), execution.getQueryId());
        final ManagedExecution<?> otherExecution = storage.getExecution(id);
        if (otherExecution == null) {
            continue;
        }
        log.trace("Patching {} ({}) with patch: {}", execution.getClass().getSimpleName(), id, patch);
        patch.applyTo(otherExecution, storage, subject);
        storage.updateExecution(execution);
    }
}
Also used : Dataset(com.bakdata.conquery.models.datasets.Dataset) ManagedExecutionId(com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId)

Example 5 with ManagedExecutionId

use of com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId in project conquery by bakdata.

the class QueryProcessor method postQuery.

/**
 * Creates a query for all datasets, then submits it for execution on the
 * intended dataset.
 */
public ManagedExecution<?> postQuery(Dataset dataset, QueryDescription query, Subject subject) {
    log.info("Query posted on Dataset[{}] by User[{{}].", dataset.getId(), subject.getId());
    // This maps works as long as we have query visitors that are not configured in anyway.
    // So adding a visitor twice would replace the previous one but both would have yielded the same result.
    // For the future a better data structure might be desired that also regards similar QueryVisitors of different configuration
    ClassToInstanceMap<QueryVisitor> visitors = MutableClassToInstanceMap.create();
    query.addVisitors(visitors);
    // Initialize checks that need to traverse the query tree
    visitors.putInstance(QueryUtils.OnlyReusingChecker.class, new QueryUtils.OnlyReusingChecker());
    visitors.putInstance(NamespacedIdentifiableCollector.class, new NamespacedIdentifiableCollector());
    final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(subject, storage).map(Group::getName).orElse("none");
    visitors.putInstance(ExecutionMetrics.QueryMetricsReporter.class, new ExecutionMetrics.QueryMetricsReporter(primaryGroupName));
    // Chain all Consumers
    Consumer<Visitable> consumerChain = QueryUtils.getNoOpEntryPoint();
    for (QueryVisitor visitor : visitors.values()) {
        consumerChain = consumerChain.andThen(visitor);
    }
    // Apply consumers to the query tree
    query.visit(consumerChain);
    query.authorize(subject, dataset, visitors);
    // After all authorization checks we can now use the actual subject to invoke the query and do not to bubble down the Userish in methods
    ExecutionMetrics.reportNamespacedIds(visitors.getInstance(NamespacedIdentifiableCollector.class).getIdentifiables(), primaryGroupName);
    ExecutionMetrics.reportQueryClassUsage(query.getClass(), primaryGroupName);
    final Namespace namespace = datasetRegistry.get(dataset.getId());
    final ExecutionManager executionManager = namespace.getExecutionManager();
    // If this is only a re-executing query, try to execute the underlying query instead.
    {
        final Optional<ManagedExecutionId> executionId = visitors.getInstance(QueryUtils.OnlyReusingChecker.class).getOnlyReused();
        final Optional<ManagedExecution<?>> execution = executionId.map(id -> tryReuse(query, id, datasetRegistry, config, executionManager, subject.getUser()));
        if (execution.isPresent()) {
            return execution.get();
        }
    }
    // Execute the query
    return executionManager.runQuery(datasetRegistry, query, subject.getUser(), dataset, config);
}
Also used : ManagedQuery(com.bakdata.conquery.models.query.ManagedQuery) ConqueryConfig(com.bakdata.conquery.models.config.ConqueryConfig) URL(java.net.URL) ExternalUpload(com.bakdata.conquery.apiv1.query.ExternalUpload) RequiredArgsConstructor(lombok.RequiredArgsConstructor) ManagedExecution(com.bakdata.conquery.models.execution.ManagedExecution) Map(java.util.Map) DatasetId(com.bakdata.conquery.models.identifiable.ids.specific.DatasetId) BadRequestException(javax.ws.rs.BadRequestException) UriBuilder(javax.ws.rs.core.UriBuilder) QueryDescription(com.bakdata.conquery.apiv1.query.QueryDescription) CQAnd(com.bakdata.conquery.apiv1.query.concept.specific.CQAnd) ExecutionMetrics(com.bakdata.conquery.metrics.ExecutionMetrics) Collection(java.util.Collection) SecondaryIdQuery(com.bakdata.conquery.apiv1.query.SecondaryIdQuery) Visitable(com.bakdata.conquery.models.query.Visitable) Set(java.util.Set) User(com.bakdata.conquery.models.auth.entities.User) Collectors(java.util.stream.Collectors) Dataset(com.bakdata.conquery.models.datasets.Dataset) ConceptQuery(com.bakdata.conquery.apiv1.query.ConceptQuery) ExternalUploadResult(com.bakdata.conquery.apiv1.query.ExternalUploadResult) SecondaryIdDescription(com.bakdata.conquery.models.datasets.SecondaryIdDescription) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) Stream(java.util.stream.Stream) Response(javax.ws.rs.core.Response) CancelQuery(com.bakdata.conquery.models.messages.namespaces.specific.CancelQuery) Ability(com.bakdata.conquery.models.auth.permissions.Ability) QueryUtils(com.bakdata.conquery.util.QueryUtils) Optional(java.util.Optional) MetaStorage(com.bakdata.conquery.io.storage.MetaStorage) Namespace(com.bakdata.conquery.models.worker.Namespace) ExecutionManager(com.bakdata.conquery.models.query.ExecutionManager) ResultRendererProvider(com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider) Getter(lombok.Getter) ExecutionState(com.bakdata.conquery.models.execution.ExecutionState) Subject(com.bakdata.conquery.models.auth.entities.Subject) ClassToInstanceMap(com.google.common.collect.ClassToInstanceMap) HttpServletRequest(javax.servlet.http.HttpServletRequest) ManagedExecutionId(com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId) QueryVisitor(com.bakdata.conquery.models.query.visitor.QueryVisitor) NamespacedIdentifiableCollector(com.bakdata.conquery.util.QueryUtils.NamespacedIdentifiableCollector) Group(com.bakdata.conquery.models.auth.entities.Group) CQExternal(com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal) AuthorizationHelper(com.bakdata.conquery.models.auth.AuthorizationHelper) Consumer(java.util.function.Consumer) MutableClassToInstanceMap(com.google.common.collect.MutableClassToInstanceMap) AuthorizationHelper.buildDatasetAbilityMap(com.bakdata.conquery.models.auth.AuthorizationHelper.buildDatasetAbilityMap) DatasetRegistry(com.bakdata.conquery.models.worker.DatasetRegistry) CQElement(com.bakdata.conquery.apiv1.query.CQElement) Query(com.bakdata.conquery.apiv1.query.Query) QueryVisitor(com.bakdata.conquery.models.query.visitor.QueryVisitor) ExecutionManager(com.bakdata.conquery.models.query.ExecutionManager) NamespacedIdentifiableCollector(com.bakdata.conquery.util.QueryUtils.NamespacedIdentifiableCollector) Optional(java.util.Optional) Visitable(com.bakdata.conquery.models.query.Visitable) ExecutionMetrics(com.bakdata.conquery.metrics.ExecutionMetrics) Namespace(com.bakdata.conquery.models.worker.Namespace) QueryUtils(com.bakdata.conquery.util.QueryUtils)

Aggregations

ManagedExecutionId (com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId)13 User (com.bakdata.conquery.models.auth.entities.User)6 Dataset (com.bakdata.conquery.models.datasets.Dataset)4 DatasetId (com.bakdata.conquery.models.identifiable.ids.specific.DatasetId)4 ManagedQuery (com.bakdata.conquery.models.query.ManagedQuery)4 Query (com.bakdata.conquery.apiv1.query.Query)3 MetaStorage (com.bakdata.conquery.io.storage.MetaStorage)3 Group (com.bakdata.conquery.models.auth.entities.Group)3 Role (com.bakdata.conquery.models.auth.entities.Role)3 Test (org.junit.jupiter.api.Test)3 ConceptQuery (com.bakdata.conquery.apiv1.query.ConceptQuery)2 SecondaryIdQuery (com.bakdata.conquery.apiv1.query.SecondaryIdQuery)2 CQAnd (com.bakdata.conquery.apiv1.query.concept.specific.CQAnd)2 ExecutionPermission (com.bakdata.conquery.models.auth.permissions.ExecutionPermission)2 ConqueryError (com.bakdata.conquery.models.error.ConqueryError)2 ConqueryError.asConqueryError (com.bakdata.conquery.models.error.ConqueryError.asConqueryError)2 CentralRegistry (com.bakdata.conquery.models.identifiable.CentralRegistry)2 QueryExecutionContext (com.bakdata.conquery.models.query.QueryExecutionContext)2 QueryExecutor (com.bakdata.conquery.models.query.QueryExecutor)2 QueryPlanContext (com.bakdata.conquery.models.query.QueryPlanContext)2