Examples with Dataset com.bakdata.conquery.models.datasets.Dataset used on opensource projects

Example 41 with Dataset

use of com.bakdata.conquery.models.datasets.Dataset in project conquery by bakdata.

the class CBlockTest method serialize.

@SneakyThrows
@Test
public void serialize() throws IOException, JSONException {
    final CentralRegistry registry = new CentralRegistry();
    final Dataset dataset = new Dataset();
    dataset.setName("dataset");
    final TreeConcept concept = new TreeConcept();
    concept.setDataset(dataset);
    concept.setName("concept");
    final ConceptTreeConnector connector = new ConceptTreeConnector();
    connector.setName("connector");
    connector.setConcept(concept);
    concept.setConnectors(List.of(connector));
    final Table table = new Table();
    table.setName("table");
    table.setDataset(dataset);
    final Import imp = new Import(table);
    imp.setName("import");
    final Bucket bucket = new Bucket(0, 0, 0, new ColumnStore[0], Collections.emptySet(), new int[10], new int[10], imp);
    final CBlock cBlock = CBlock.createCBlock(connector, bucket, 10);
    registry.register(dataset).register(table).register(concept).register(connector).register(bucket).register(imp);
    SerializationTestUtil.forType(CBlock.class).registry(registry).test(cBlock);
}

Example 42 with Dataset

use of com.bakdata.conquery.models.datasets.Dataset in project conquery by bakdata.

the class QueryDescription method authorize.

/**
 * Check implementation specific permissions. Is called after all visitors have been registered and executed.
 */
default void authorize(Subject subject, Dataset submittedDataset, @NonNull ClassToInstanceMap<QueryVisitor> visitors) {
    NamespacedIdentifiableCollector nsIdCollector = QueryUtils.getVisitor(visitors, NamespacedIdentifiableCollector.class);
    ExternalIdChecker externalIdChecker = QueryUtils.getVisitor(visitors, QueryUtils.ExternalIdChecker.class);
    if (nsIdCollector == null) {
        throw new IllegalStateException();
    }
    // Generate DatasetPermissions
    final Set<Dataset> datasets = nsIdCollector.getIdentifiables().stream().map(NamespacedIdentifiable::getDataset).collect(Collectors.toSet());
    subject.authorize(datasets, Ability.READ);
    // Generate ConceptPermissions
    final Set<Concept> concepts = nsIdCollector.getIdentifiables().stream().filter(ConceptElement.class::isInstance).map(ConceptElement.class::cast).map(ConceptElement::getConcept).collect(Collectors.toSet());
    subject.authorize(concepts, Ability.READ);
    subject.authorize(collectRequiredQueries(), Ability.READ);
    // Check if the query contains parts that require to resolve external IDs. If so the subject must have the preserve_id permission on the dataset.
    if (externalIdChecker.resolvesExternalIds()) {
        subject.authorize(submittedDataset, Ability.PRESERVE_ID);
    }
}

Example 43 with Dataset

use of com.bakdata.conquery.models.datasets.Dataset in project conquery by bakdata.

the class PermissionGroupHandlingTest method execute.

/**
 * This is a longer test that plays through different scenarios of permission
 * and role adding/deleting. Creating many objects here to avoid side effects.
 */
@Override
public void execute(StandaloneSupport conquery) throws Exception {
    MetaStorage storage = conquery.getMetaStorage();
    Dataset dataset1 = new Dataset();
    dataset1.setLabel("dataset1");
    ManagedExecutionId query1 = new ManagedExecutionId(dataset1.getId(), UUID.randomUUID());
    Role role1 = new Role("role", "role", storage);
    TestUser user1 = new TestUser(storage);
    Group group1 = new Group("company", "company", storage);
    try {
        storage.addRole(role1);
        storage.addUser(user1);
        storage.addGroup(group1);
        user1.addRole(role1);
        group1.addMember(user1);
        user1.addPermission(ExecutionPermission.onInstance(Ability.READ, query1));
        role1.addPermission(ExecutionPermission.onInstance(Ability.DELETE, query1));
        group1.addPermission(ExecutionPermission.onInstance(Ability.SHARE, query1));
        assertThat(user1.isPermitted(ExecutionPermission.onInstance(Ability.READ, query1))).isTrue();
        assertThat(user1.isPermitted(ExecutionPermission.onInstance(Ability.DELETE, query1))).isTrue();
        assertThat(user1.isPermitted(ExecutionPermission.onInstance(Ability.SHARE, query1))).isTrue();
        // remove user from group
        group1.removeMember(user1);
        assertThat(user1.isPermitted(ExecutionPermission.onInstance(Ability.READ, query1))).isTrue();
        assertThat(user1.isPermitted(ExecutionPermission.onInstance(Ability.DELETE, query1))).isTrue();
        assertThat(user1.isPermitted(ExecutionPermission.onInstance(Ability.SHARE, query1))).isFalse();
    } finally {
        storage.removeGroup(group1.getId());
        storage.removeUser(user1.getId());
        storage.removeRole(role1.getId());
    }
}

Example 44 with Dataset

use of com.bakdata.conquery.models.datasets.Dataset in project conquery by bakdata.

the class RestartTest method execute.

@Override
public void execute(String name, TestConquery testConquery) throws Exception {
    // read test specification
    String testJson = In.resource("/tests/query/RESTART_TEST_DATA/SIMPLE_TREECONCEPT_Query.json").withUTF8().readAll();
    Validator validator = Validators.newValidator();
    EntityIdMap entityIdMap = IdMapSerialisationTest.createTestPersistentMap();
    ManagerNode manager = testConquery.getStandaloneCommand().getManager();
    AdminDatasetProcessor adminDatasetProcessor = manager.getAdmin().getAdminDatasetProcessor();
    AdminProcessor adminProcessor = manager.getAdmin().getAdminProcessor();
    StandaloneSupport conquery = testConquery.getSupport(name);
    DatasetId dataset = conquery.getDataset().getId();
    ConqueryTestSpec test = JsonIntegrationTest.readJson(dataset, testJson);
    ValidatorHelper.failOnError(log, validator.validate(test));
    test.importRequiredData(conquery);
    test.executeTest(conquery);
    final int numberOfExecutions = conquery.getMetaStorage().getAllExecutions().size();
    // IDMapping Testing
    NamespaceStorage namespaceStorage = conquery.getNamespaceStorage();
    namespaceStorage.updateIdMapping(entityIdMap);
    final Dataset dataset1 = adminDatasetProcessor.addDataset(TEST_DATASET_1);
    final Dataset dataset2 = adminDatasetProcessor.addDataset(TEST_DATASET_2);
    final Dataset dataset3 = adminDatasetProcessor.addDataset(TEST_DATASET_3);
    final Dataset dataset4 = adminDatasetProcessor.addDataset(TEST_DATASET_4);
    final Dataset dataset5 = adminDatasetProcessor.addDataset(TEST_DATASET_5);
    final Dataset dataset6 = adminDatasetProcessor.addDataset(TEST_DATASET_6);
    MetaStorage storage = conquery.getMetaStorage();
    Role role = new Role("role", "ROLE", storage);
    Role roleToDelete = new Role("roleDelete", "ROLE_DELETE", storage);
    User user = new User("user@test.email", "USER", storage);
    User userToDelete = new User("userDelete@test.email", "USER_DELETE", storage);
    Group group = new Group("group", "GROUP", storage);
    Group groupToDelete = new Group("groupDelete", "GROUP_DELETE", storage);
    {
        // Auth testing (deletion and permission grant)
        // build constellation
        // TODO USE APIS
        adminProcessor.addUser(user);
        adminProcessor.addUser(userToDelete);
        adminProcessor.addRole(role);
        adminProcessor.addRole(roleToDelete);
        adminProcessor.addGroup(group);
        adminProcessor.addGroup(groupToDelete);
        adminProcessor.addRoleTo(user, role);
        adminProcessor.addRoleTo(user, roleToDelete);
        adminProcessor.addRoleTo(userToDelete, role);
        adminProcessor.addRoleTo(userToDelete, roleToDelete);
        adminProcessor.addRoleTo(group, role);
        adminProcessor.addRoleTo(group, roleToDelete);
        adminProcessor.addRoleTo(groupToDelete, role);
        adminProcessor.addRoleTo(groupToDelete, roleToDelete);
        adminProcessor.addUserToGroup(group, user);
        adminProcessor.addUserToGroup(group, userToDelete);
        adminProcessor.addUserToGroup(groupToDelete, user);
        adminProcessor.addUserToGroup(groupToDelete, userToDelete);
        // Adding Permissions
        adminProcessor.createPermission(user, dataset1.createPermission(Ability.READ.asSet()));
        adminProcessor.createPermission(userToDelete, dataset2.createPermission(Ability.READ.asSet()));
        adminProcessor.createPermission(role, dataset3.createPermission(Ability.READ.asSet()));
        adminProcessor.createPermission(roleToDelete, dataset4.createPermission(Ability.READ.asSet()));
        adminProcessor.createPermission(group, dataset5.createPermission(Ability.READ.asSet()));
        adminProcessor.createPermission(groupToDelete, dataset6.createPermission(Ability.READ.asSet()));
        // Delete entities
        // TODO use API
        adminProcessor.deleteUser(userToDelete);
        adminProcessor.deleteRole(roleToDelete);
        adminProcessor.deleteGroup(groupToDelete);
    }
    log.info("Shutting down for restart");
    testConquery.shutdown();
    log.info("Restarting");
    testConquery.beforeAll();
    final StandaloneSupport support = testConquery.openDataset(dataset);
    log.info("Restart complete");
    DatasetRegistry datasetRegistry = support.getDatasetsProcessor().getDatasetRegistry();
    assertThat(support.getMetaStorage().getAllExecutions().size()).as("Executions after restart").isEqualTo(numberOfExecutions);
    test.executeTest(support);
    {
        // Auth actual tests
        User userStored = storage.getUser(user.getId());
        assertThat(userStored).isEqualTo(user);
        assertThat(storage.getRole(role.getId())).isEqualTo(role);
        assertThat(storage.getGroup(group.getId())).isEqualTo(group);
        assertThat(storage.getUser(userToDelete.getId())).as("deleted user should stay deleted").isNull();
        assertThat(storage.getRole(roleToDelete.getId())).as("deleted role should stay deleted").isNull();
        assertThat(storage.getGroup(groupToDelete.getId())).as("deleted group should stay deleted").isNull();
        assertThat(userStored.isPermitted(datasetRegistry.get(TEST_DATASET_1.getId()).getDataset(), Ability.READ)).isTrue();
        assertThat(userStored.isPermitted(datasetRegistry.get(TEST_DATASET_2.getId()).getDataset(), Ability.READ)).isFalse();
        assertThat(userStored.isPermitted(datasetRegistry.get(TEST_DATASET_3.getId()).getDataset(), Ability.READ)).isTrue();
        assertThat(userStored.isPermitted(datasetRegistry.get(TEST_DATASET_4.getId()).getDataset(), Ability.READ)).isFalse();
        assertThat(userStored.isPermitted(datasetRegistry.get(TEST_DATASET_5.getId()).getDataset(), Ability.READ)).isTrue();
        assertThat(userStored.isPermitted(datasetRegistry.get(TEST_DATASET_6.getId()).getDataset(), Ability.READ)).isFalse();
    }
    EntityIdMap entityIdMapAfterRestart = conquery.getNamespaceStorage().getIdMapping();
    assertThat(entityIdMapAfterRestart).isEqualTo(entityIdMap);
    // We need to reassign the dataset processor because the instance prio to the restart became invalid
    adminDatasetProcessor = testConquery.getStandaloneCommand().getManager().getAdmin().getAdminDatasetProcessor();
    // Cleanup
    adminDatasetProcessor.deleteDataset(dataset1);
    adminDatasetProcessor.deleteDataset(dataset2);
    adminDatasetProcessor.deleteDataset(dataset3);
    adminDatasetProcessor.deleteDataset(dataset4);
    adminDatasetProcessor.deleteDataset(dataset5);
    adminDatasetProcessor.deleteDataset(dataset6);
}

Example 45 with Dataset

use of com.bakdata.conquery.models.datasets.Dataset in project conquery by bakdata.

the class ReusedQueryTest method execute.

@Override
public void execute(String name, TestConquery testConquery) throws Exception {
    final StandaloneSupport conquery = testConquery.getSupport(name);
    final String testJson = In.resource("/tests/query/SECONDARY_ID_MIXED/SECONDARY_IDS_MIXED.test.json").withUTF8().readAll();
    final Dataset dataset = conquery.getDataset();
    final QueryTest test = (QueryTest) JsonIntegrationTest.readJson(dataset, testJson);
    // Manually import data, so we can do our own work.
    {
        ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
        importSecondaryIds(conquery, test.getContent().getSecondaryIds());
        conquery.waitUntilWorkDone();
        LoadingUtil.importTables(conquery, test.getContent().getTables());
        conquery.waitUntilWorkDone();
        LoadingUtil.importConcepts(conquery, test.getRawConcepts());
        conquery.waitUntilWorkDone();
        LoadingUtil.importTableContents(conquery, test.getContent().getTables());
        conquery.waitUntilWorkDone();
    }
    final SecondaryIdQuery query = (SecondaryIdQuery) IntegrationUtils.parseQuery(conquery, test.getRawQuery());
    final ManagedExecutionId id = IntegrationUtils.assertQueryResult(conquery, query, 4L, ExecutionState.DONE, conquery.getTestUser(), 201);
    assertThat(id).isNotNull();
    final MetaStorage metaStorage = conquery.getMetaStorage();
    final ManagedQuery execution = (ManagedQuery) metaStorage.getExecution(id);
    // Normal reuse
    {
        final ConceptQuery reused = new ConceptQuery(new CQReusedQuery(execution.getId()));
        IntegrationUtils.assertQueryResult(conquery, reused, 2L, ExecutionState.DONE, conquery.getTestUser(), 201);
    }
    // Reuse by API
    {
        final URI reexecuteUri = HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), QueryResource.class, "reexecute").buildFromMap(Map.of(ResourceConstants.DATASET, conquery.getDataset().getName(), ResourceConstants.QUERY, execution.getId().toString()));
        final FullExecutionStatus status = conquery.getClient().target(reexecuteUri).request(MediaType.APPLICATION_JSON).post(Entity.entity(null, MediaType.APPLICATION_JSON_TYPE)).readEntity(FullExecutionStatus.class);
        assertThat(status.getStatus()).isIn(ExecutionState.RUNNING, ExecutionState.DONE);
    }
    // Reuse in SecondaryId
    {
        final SecondaryIdQuery reused = new SecondaryIdQuery();
        reused.setRoot(new CQReusedQuery(execution.getId()));
        reused.setSecondaryId(query.getSecondaryId());
        IntegrationUtils.assertQueryResult(conquery, reused, 4L, ExecutionState.DONE, conquery.getTestUser(), 201);
    }
    // Reuse in SecondaryId, but do exclude
    {
        final SecondaryIdQuery reused = new SecondaryIdQuery();
        final CQAnd root = new CQAnd();
        reused.setRoot(root);
        final CQReusedQuery reuse = new CQReusedQuery(execution.getId());
        reuse.setExcludeFromSecondaryId(true);
        // We select only a single event of the query by the exact filtering.
        final CQConcept cqConcept = new CQConcept();
        final ConceptId conceptId = new ConceptId(conquery.getDataset().getId(), "concept");
        final Concept<?> concept = conquery.getNamespaceStorage().getConcept(conceptId);
        cqConcept.setElements(List.of(concept));
        final CQTable cqTable = new CQTable();
        cqTable.setConcept(cqConcept);
        final CentralRegistry centralRegistry = conquery.getNamespaceStorage().getCentralRegistry();
        final Connector connector = centralRegistry.resolve(new ConnectorId(conceptId, "connector1"));
        cqTable.setConnector(connector);
        cqTable.setFilters(List.of(new FilterValue.CQRealRangeFilter((Filter<Range<BigDecimal>>) centralRegistry.resolve(new FilterId(connector.getId(), "filter")), new Range<>(BigDecimal.valueOf(1.01d), BigDecimal.valueOf(1.01d)))));
        cqConcept.setTables(List.of(cqTable));
        cqConcept.setExcludeFromSecondaryId(false);
        root.setChildren(List.of(reuse, cqConcept));
        reused.setSecondaryId(query.getSecondaryId());
        IntegrationUtils.assertQueryResult(conquery, reused, 1L, ExecutionState.DONE, conquery.getTestUser(), 201);
    }
    // Reuse Multiple times with different query types
    {
        final SecondaryIdQuery reused1 = new SecondaryIdQuery();
        reused1.setRoot(new CQReusedQuery(execution.getId()));
        reused1.setSecondaryId(query.getSecondaryId());
        final ManagedExecutionId reused1Id = IntegrationUtils.assertQueryResult(conquery, reused1, 4L, ExecutionState.DONE, conquery.getTestUser(), 201);
        final ManagedQuery execution1 = (ManagedQuery) metaStorage.getExecution(reused1Id);
        {
            final SecondaryIdQuery reused2 = new SecondaryIdQuery();
            reused2.setRoot(new CQReusedQuery(execution1.getId()));
            reused2.setSecondaryId(query.getSecondaryId());
            final ManagedExecutionId reused2Id = IntegrationUtils.assertQueryResult(conquery, reused2, 4L, ExecutionState.DONE, conquery.getTestUser(), 201);
            final ManagedQuery execution2 = (ManagedQuery) metaStorage.getExecution(reused2Id);
            assertThat(reused2Id).as("Query should be reused.").isEqualTo(reused1Id);
            // Now we change to ConceptQuery
            final ConceptQuery reused3 = new ConceptQuery(new CQReusedQuery(execution2.getId()));
            IntegrationUtils.assertQueryResult(conquery, reused3, 2L, ExecutionState.DONE, conquery.getTestUser(), 201);
        }
        {
            final SecondaryIdQuery reusedDiffId = new SecondaryIdQuery();
            reusedDiffId.setRoot(new CQReusedQuery(execution1.getId()));
            // ignored is a single global value and therefore the same as by-PID
            reusedDiffId.setSecondaryId(conquery.getNamespace().getStorage().getSecondaryId(new SecondaryIdDescriptionId(conquery.getDataset().getId(), "ignored")));
            final ManagedExecutionId executionId = IntegrationUtils.assertQueryResult(conquery, reusedDiffId, 2L, ExecutionState.DONE, conquery.getTestUser(), 201);
            assertThat(executionId).as("Query should NOT be reused.").isNotEqualTo(reused1Id);
        }
        {
            // Reuse by another user (create a copy of the actual query)
            final SecondaryIdQuery reused = new SecondaryIdQuery();
            reused.setRoot(new CQReusedQuery(execution.getId()));
            reused.setSecondaryId(query.getSecondaryId());
            User shareHolder = new User("shareholder", "ShareHolder", conquery.getMetaStorage());
            conquery.getMetaProcessor().addUser(shareHolder);
            shareHolder.addPermissions(Set.of(dataset.createPermission(Set.of(Ability.READ)), execution.createPermission(Set.of(Ability.READ))));
            ManagedExecutionId copyId = IntegrationUtils.assertQueryResult(conquery, reused, 4L, ExecutionState.DONE, shareHolder, 201);
            ManagedExecution<?> copy = metaStorage.getExecution(copyId);
            // Contentwise the label and tags should be the same
            assertThat(copy).usingRecursiveComparison().comparingOnlyFields("label", "tags").isEqualTo(execution);
            // However the Object holding the tags must be different, so the two are not linked here
            assertThat(copy.getTags()).isNotSameAs(execution.getTags());
            // And the ids must be different
            assertThat(copy.getId()).isNotSameAs(execution.getId());
        }
    }
}