use of com.bakdata.conquery.io.storage.MetaStorage in project conquery by bakdata.
the class SuperPermissionTest method execute.
@Override
public void execute(StandaloneSupport conquery) throws Exception {
Dataset dataset1 = new Dataset();
dataset1.setLabel("dataset1");
MetaStorage storage = conquery.getMetaStorage();
Role role1 = new Role("company", "company", storage);
TestUser user1 = new TestUser(storage);
storage.addRole(role1);
try {
user1.addRole(role1);
// Add SuperPermission to User
user1.addPermission(SuperPermission.onDomain());
assertThat(user1.isPermitted(DatasetPermission.onInstance(Ability.READ, dataset1.getId()))).isTrue();
assertThat(user1.isPermitted(DatasetPermission.onInstance(Ability.DOWNLOAD, dataset1.getId()))).isTrue();
// Add SuperPermission to mandator and remove from user
user1.removePermission(SuperPermission.onDomain());
role1.addPermission(SuperPermission.onDomain());
assertThat(user1.isPermitted(DatasetPermission.onInstance(Ability.READ, dataset1.getId()))).isTrue();
assertThat(user1.isPermitted(DatasetPermission.onInstance(Ability.DOWNLOAD, dataset1.getId()))).isTrue();
// Add SuperPermission to mandator and remove from user
role1.removePermission(SuperPermission.onDomain());
assertThat(user1.isPermitted(DatasetPermission.onInstance(Ability.READ, dataset1.getId()))).isFalse();
assertThat(user1.isPermitted(DatasetPermission.onInstance(Ability.DOWNLOAD, dataset1.getId()))).isFalse();
} finally {
storage.removeUser(user1.getId());
storage.removeRole(role1.getId());
}
}
use of com.bakdata.conquery.io.storage.MetaStorage in project conquery by bakdata.
the class DatasetDeletionTest method execute.
@Override
public void execute(String name, TestConquery testConquery) throws Exception {
final StandaloneSupport conquery = testConquery.getSupport(name);
final MetaStorage storage = conquery.getMetaStorage();
final Dataset dataset = conquery.getDataset();
Namespace namespace = conquery.getNamespace();
final String testJson = In.resource("/tests/query/DELETE_IMPORT_TESTS/SIMPLE_TREECONCEPT_Query.test.json").withUTF8().readAll();
final QueryTest test = (QueryTest) JsonIntegrationTest.readJson(dataset, testJson);
// Manually import data, so we can do our own work.
{
ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
importSecondaryIds(conquery, test.getContent().getSecondaryIds());
conquery.waitUntilWorkDone();
LoadingUtil.importTables(conquery, test.getContent().getTables());
conquery.waitUntilWorkDone();
LoadingUtil.importConcepts(conquery, test.getRawConcepts());
conquery.waitUntilWorkDone();
LoadingUtil.importTableContents(conquery, test.getContent().getTables());
conquery.waitUntilWorkDone();
}
final Query query = IntegrationUtils.parseQuery(conquery, test.getRawQuery());
final int nImports = namespace.getStorage().getAllImports().size();
log.info("Checking state before deletion");
// Assert state before deletion.
{
// Must contain the import.
assertThat(namespace.getStorage().getCentralRegistry().getOptional(dataset.getId())).isNotEmpty();
for (ShardNode node : conquery.getShardNodes()) {
for (Worker value : node.getWorkers().getWorkers().values()) {
if (!value.getInfo().getDataset().equals(dataset.getId())) {
continue;
}
final ModificationShieldedWorkerStorage workerStorage = value.getStorage();
assertThat(workerStorage.getAllCBlocks()).describedAs("CBlocks for Worker %s", value.getInfo().getId()).isNotEmpty();
assertThat(workerStorage.getAllBuckets()).describedAs("Buckets for Worker %s", value.getInfo().getId()).isNotEmpty();
}
}
log.info("Executing query before deletion");
IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201);
}
// Delete Dataset.
{
log.info("Issuing deletion of import {}", dataset);
// Delete the import.
// But, we do not allow deletion of tables with associated connectors, so this should throw!
assertThatThrownBy(() -> conquery.getDatasetsProcessor().deleteDataset(dataset)).isInstanceOf(WebApplicationException.class);
// TODO use api
conquery.getNamespace().getStorage().getTables().forEach(tableId -> conquery.getDatasetsProcessor().deleteTable(tableId, true));
conquery.waitUntilWorkDone();
// Finally delete dataset
conquery.getDatasetsProcessor().deleteDataset(dataset);
conquery.waitUntilWorkDone();
assertThat(storage.getCentralRegistry().getOptional(dataset.getId())).isEmpty();
}
// State after deletion.
{
log.info("Checking state after deletion");
// We have deleted an import now there should be two less!
assertThat(namespace.getStorage().getAllImports().size()).isEqualTo(0);
// The deleted import should not be found.
assertThat(namespace.getStorage().getAllImports()).filteredOn(imp -> imp.getId().getTable().getDataset().equals(dataset.getId())).isEmpty();
for (ShardNode node : conquery.getShardNodes()) {
for (Worker value : node.getWorkers().getWorkers().values()) {
if (!value.getInfo().getDataset().equals(dataset.getId())) {
continue;
}
final ModificationShieldedWorkerStorage workerStorage = value.getStorage();
// No bucket should be found referencing the import.
assertThat(workerStorage.getAllBuckets()).describedAs("Buckets for Worker %s", value.getInfo().getId()).filteredOn(bucket -> bucket.getTable().getDataset().getId().equals(dataset.getId())).isEmpty();
// No CBlock associated with import may exist
assertThat(workerStorage.getAllCBlocks()).describedAs("CBlocks for Worker %s", value.getInfo().getId()).filteredOn(cBlock -> cBlock.getBucket().getTable().getDataset().getId().equals(dataset.getId())).isEmpty();
}
}
// It's not exactly possible to issue a query for a non-existant dataset, so we assert that parsing the fails.
assertThatThrownBy(() -> {
IntegrationUtils.parseQuery(conquery, test.getRawQuery());
}).isNotNull();
IntegrationUtils.assertQueryResult(conquery, query, 0, ExecutionState.FAILED, conquery.getTestUser(), 404);
}
// Reload the dataset and assert the state.
// We have to do some weird trix with StandaloneSupport to open it with another Dataset
final StandaloneSupport conqueryReimport = testConquery.getSupport(namespace.getDataset().getName());
{
// only import the deleted import/table
LoadingUtil.importTables(conqueryReimport, test.getContent().getTables());
assertThat(conqueryReimport.getNamespace().getStorage().getTables()).isNotEmpty();
conqueryReimport.waitUntilWorkDone();
LoadingUtil.importTableContents(conqueryReimport, test.getContent().getTables());
conqueryReimport.waitUntilWorkDone();
LoadingUtil.importConcepts(conqueryReimport, test.getRawConcepts());
conqueryReimport.waitUntilWorkDone();
assertThat(conqueryReimport.getDatasetsProcessor().getDatasetRegistry().get(conqueryReimport.getDataset().getId())).describedAs("Dataset after re-import.").isNotNull();
assertThat(conqueryReimport.getNamespace().getStorage().getAllImports().size()).isEqualTo(nImports);
for (ShardNode node : conqueryReimport.getShardNodes()) {
assertThat(node.getWorkers().getWorkers().values()).filteredOn(w -> w.getInfo().getDataset().equals(conqueryReimport.getDataset().getId())).describedAs("Workers for node {}", node.getName()).isNotEmpty();
}
log.info("Executing query after re-import");
final Query query2 = IntegrationUtils.parseQuery(conqueryReimport, test.getRawQuery());
// Issue a query and assert that it has the same content as the first time around.
IntegrationUtils.assertQueryResult(conqueryReimport, query2, 2L, ExecutionState.DONE, conqueryReimport.getTestUser(), 201);
}
// Finally, restart conquery and assert again, that the data is correct.
{
testConquery.shutdown();
// restart
testConquery.beforeAll();
final StandaloneSupport conqueryRestart = testConquery.openDataset(conqueryReimport.getDataset().getId());
log.info("Checking state after re-start");
assertThat(conqueryRestart.getNamespace().getStorage().getAllImports().size()).isEqualTo(2);
for (ShardNode node : conqueryRestart.getShardNodes()) {
for (Worker value : node.getWorkers().getWorkers().values()) {
if (!value.getInfo().getDataset().equals(dataset.getId())) {
continue;
}
final ModificationShieldedWorkerStorage workerStorage = value.getStorage();
assertThat(workerStorage.getAllBuckets().stream().filter(bucket -> bucket.getTable().getDataset().getId().equals(dataset.getId()))).describedAs("Buckets for Worker %s", value.getInfo().getId()).isNotEmpty();
}
}
log.info("Executing query after restart");
final Query query3 = IntegrationUtils.parseQuery(conqueryRestart, test.getRawQuery());
// Issue a query and assert that it has the same content as the first time around.
IntegrationUtils.assertQueryResult(conqueryRestart, query3, 2L, ExecutionState.DONE, conquery.getTestUser(), 201);
}
}
use of com.bakdata.conquery.io.storage.MetaStorage in project conquery by bakdata.
the class ConceptPermissionTest method execute.
@Override
public void execute(StandaloneSupport conquery) throws Exception {
final MetaStorage storage = conquery.getMetaStorage();
final Dataset dataset = conquery.getDataset();
final String testJson = In.resource("/tests/query/SIMPLE_TREECONCEPT_QUERY/SIMPLE_TREECONCEPT_Query.test.json").withUTF8().readAll();
final QueryTest test = (QueryTest) JsonIntegrationTest.readJson(dataset.getId(), testJson);
final QueryProcessor processor = new QueryProcessor(conquery.getDatasetRegistry(), storage, conquery.getConfig());
final User user = new User("testUser", "testUserLabel", storage);
// Manually import data, so we can do our own work.
{
ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
importSecondaryIds(conquery, test.getContent().getSecondaryIds());
conquery.waitUntilWorkDone();
LoadingUtil.importTables(conquery, test.getContent().getTables());
conquery.waitUntilWorkDone();
LoadingUtil.importConcepts(conquery, test.getRawConcepts());
conquery.waitUntilWorkDone();
LoadingUtil.importTableContents(conquery, test.getContent().getTables());
conquery.waitUntilWorkDone();
storage.addUser(user);
user.addPermission(DatasetPermission.onInstance(Ability.READ, dataset.getId()));
}
// Query cannot be deserialized without Namespace set up
final Query query = IntegrationUtils.parseQuery(conquery, test.getRawQuery());
// Id of the lone concept that is used in the test.
Concept<?> conceptId = conquery.getNamespace().getStorage().getAllConcepts().iterator().next();
IntegrationUtils.assertQueryResult(conquery, query, -1, ExecutionState.FAILED, user, 403);
// Add the necessary Permission
{
final ConqueryPermission permission = conceptId.createPermission(Ability.READ.asSet());
log.info("Adding the Permission[{}] to User[{}]", permission, user);
user.addPermission(permission);
}
// Only assert permissions
IntegrationUtils.assertQueryResult(conquery, query, -1, ExecutionState.DONE, user, 201);
conquery.waitUntilWorkDone();
// Clean up
{
storage.removeUser(user.getId());
}
}
use of com.bakdata.conquery.io.storage.MetaStorage in project conquery by bakdata.
the class ExecutionManager method execute.
public void execute(DatasetRegistry datasets, ManagedExecution<?> execution, ConqueryConfig config) {
// Initialize the query / create subqueries
try {
execution.initExecutable(datasets, config);
} catch (Exception e) {
log.error("Failed to initialize Query[{}]", execution.getId(), e);
// TODO we don't want to store completely faulty queries but is that right like this?
datasets.getMetaStorage().removeExecution(execution.getId());
throw e;
}
log.info("Executing Query[{}] in Datasets[{}]", execution.getQueryId(), execution.getRequiredDatasets());
execution.start();
final MetaStorage storage = datasets.getMetaStorage();
final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(execution.getOwner(), storage).map(Group::getName).orElse("none");
ExecutionMetrics.getRunningQueriesCounter(primaryGroupName).inc();
for (Namespace namespace : execution.getRequiredDatasets()) {
namespace.sendToAll(execution.createExecutionMessage());
}
}
use of com.bakdata.conquery.io.storage.MetaStorage in project conquery by bakdata.
the class CopyUserTest method testUserCopy.
@Test
void testUserCopy() {
final DatasetRegistry registry = new DatasetRegistry(0);
MetaStorage storage = new NonPersistentStoreFactory().createMetaStorage();
registry.setMetaStorage(storage);
// Create test role
Role role = new Role("role", "role", storage);
storage.addRole(role);
role.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("dataset0")));
// Create test group
Group group = new Group("group", "group", storage);
storage.addGroup(group);
group.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("dataset1")));
// Create original user with role and group mapping
User originUser = new User("user", "user", storage);
storage.addUser(originUser);
originUser.addRole(role);
group.addMember(originUser);
// Do copy
User copy = AuthorizationController.flatCopyUser(originUser, "copytest", storage);
// Check that it is not the same user
assertThat(copy).usingRecursiveComparison().isNotEqualTo(originUser);
// Check that the copy does not have any mappings
assertThat(group.containsMember(copy)).isFalse();
assertThat(copy.getRoles()).isEmpty();
// Check that the flat map worked
assertThat(copy.getPermissions()).containsExactlyInAnyOrderElementsOf(originUser.getEffectivePermissions());
}
Aggregations