Search in sources :

Example 11 with MetaStorage

use of com.bakdata.conquery.io.storage.MetaStorage in project conquery by bakdata.

the class TestConquery method beforeEach.

public void beforeEach() {
    final MetaStorage storage = standaloneCommand.getManager().getStorage();
    testUser = standaloneCommand.getManager().getConfig().getAuthorizationRealms().getInitialUsers().get(0).createOrOverwriteUser(storage);
    storage.updateUser(testUser);
}
Also used : MetaStorage(com.bakdata.conquery.io.storage.MetaStorage)

Example 12 with MetaStorage

use of com.bakdata.conquery.io.storage.MetaStorage in project conquery by bakdata.

the class ExecutionManager method handleQueryResult.

/**
 * Receive part of query result and store into query.
 *
 * @param result
 */
public <R extends ShardResult, E extends ManagedExecution<R>> void handleQueryResult(R result) {
    final MetaStorage storage = namespace.getNamespaces().getMetaStorage();
    final E query = (E) storage.getExecution(result.getQueryId());
    if (query.getState() != ExecutionState.RUNNING) {
        return;
    }
    query.addResult(storage, result);
    // State changed to DONE or FAILED
    if (query.getState() != ExecutionState.RUNNING) {
        final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(query.getOwner(), storage).map(Group::getName).orElse("none");
        ExecutionMetrics.getRunningQueriesCounter(primaryGroupName).dec();
        ExecutionMetrics.getQueryStateCounter(query.getState(), primaryGroupName).inc();
        ExecutionMetrics.getQueriesTimeHistogram(primaryGroupName).update(query.getExecutionTime().toMillis());
    }
}
Also used : MetaStorage(com.bakdata.conquery.io.storage.MetaStorage)

Example 13 with MetaStorage

use of com.bakdata.conquery.io.storage.MetaStorage in project conquery by bakdata.

the class AdminResource method getQueries.

@GET
@Path("/queries")
public FullExecutionStatus[] getQueries(@Auth Subject currentUser, @QueryParam("limit") OptionalLong limit, @QueryParam("since") Optional<String> since) {
    final MetaStorage storage = processor.getStorage();
    final DatasetRegistry datasetRegistry = processor.getDatasetRegistry();
    return storage.getAllExecutions().stream().map(t -> {
        try {
            return t.buildStatusFull(storage, currentUser, datasetRegistry, processor.getConfig());
        } catch (ConqueryError e) {
            // Initialization of execution probably failed, so we construct a status based on the overview status
            final FullExecutionStatus fullExecutionStatus = new FullExecutionStatus();
            t.setStatusBase(currentUser, fullExecutionStatus);
            fullExecutionStatus.setStatus(ExecutionState.FAILED);
            fullExecutionStatus.setError(e);
            return fullExecutionStatus;
        }
    }).filter(t -> t.getCreatedAt().toLocalDate().isEqual(since.map(LocalDate::parse).orElse(LocalDate.now()))).limit(limit.orElse(100)).toArray(FullExecutionStatus[]::new);
}
Also used : PathParam(javax.ws.rs.PathParam) Produces(javax.ws.rs.Produces) GET(javax.ws.rs.GET) ExecutionState(com.bakdata.conquery.models.execution.ExecutionState) JobManagerStatus(com.bakdata.conquery.models.jobs.JobManagerStatus) Path(javax.ws.rs.Path) Subject(com.bakdata.conquery.models.auth.entities.Subject) JOB_ID(com.bakdata.conquery.resources.ResourceConstants.JOB_ID) Auth(io.dropwizard.auth.Auth) NewCookie(javax.ws.rs.core.NewCookie) ContainerRequestContext(javax.ws.rs.container.ContainerRequestContext) Inject(javax.inject.Inject) OptionalLong(java.util.OptionalLong) MediaType(javax.ws.rs.core.MediaType) QueryParam(javax.ws.rs.QueryParam) Consumes(javax.ws.rs.Consumes) UriBuilder(javax.ws.rs.core.UriBuilder) AdminUIResource(com.bakdata.conquery.resources.admin.ui.AdminUIResource) POST(javax.ws.rs.POST) Context(javax.ws.rs.core.Context) ImmutableMap(com.google.common.collect.ImmutableMap) AuthenticationConfig(com.bakdata.conquery.models.config.auth.AuthenticationConfig) UUID(java.util.UUID) ConqueryError(com.bakdata.conquery.models.error.ConqueryError) ExtraMimeTypes(com.bakdata.conquery.io.jersey.ExtraMimeTypes) Objects(java.util.Objects) CancelJobMessage(com.bakdata.conquery.models.messages.network.specific.CancelJobMessage) Response(javax.ws.rs.core.Response) LocalDate(java.time.LocalDate) FullExecutionStatus(com.bakdata.conquery.apiv1.FullExecutionStatus) ShardNodeInformation(com.bakdata.conquery.models.worker.ShardNodeInformation) Optional(java.util.Optional) DatasetRegistry(com.bakdata.conquery.models.worker.DatasetRegistry) MetaStorage(com.bakdata.conquery.io.storage.MetaStorage) ConqueryError(com.bakdata.conquery.models.error.ConqueryError) MetaStorage(com.bakdata.conquery.io.storage.MetaStorage) DatasetRegistry(com.bakdata.conquery.models.worker.DatasetRegistry) FullExecutionStatus(com.bakdata.conquery.apiv1.FullExecutionStatus) Path(javax.ws.rs.Path) GET(javax.ws.rs.GET)

Example 14 with MetaStorage

use of com.bakdata.conquery.io.storage.MetaStorage in project conquery by bakdata.

the class ManagerNode method run.

public void run(ConqueryConfig config, Environment environment) throws InterruptedException {
    this.environment = environment;
    validator = environment.getValidator();
    client = new JerseyClientBuilder(environment).using(config.getJerseyClient()).build(getName());
    // Instantiate DatasetRegistry and MetaStorage so they are ready for injection into the object mapper (API + Storage)
    // The validator is already injected at this point see Conquery.java
    datasetRegistry = new DatasetRegistry(config.getCluster().getEntityBucketSize());
    storage = new MetaStorage(datasetRegistry);
    datasetRegistry.injectInto(environment.getObjectMapper());
    storage.injectInto(environment.getObjectMapper());
    jobManager = new JobManager("ManagerNode", config.isFailOnError());
    formScanner = new FormScanner();
    this.config = config;
    config.initialize(this);
    // Initialization of internationalization
    I18n.init();
    RESTServer.configure(config, environment.jersey().getResourceConfig());
    maintenanceService = environment.lifecycle().scheduledExecutorService("Maintenance Service").build();
    environment.lifecycle().manage(this);
    loadNamespaces();
    loadMetaStorage();
    authController = new AuthorizationController(storage, config.getAuthorizationRealms());
    environment.lifecycle().manage(authController);
    unprotectedAuthAdmin = AuthServlet.generalSetup(environment.metrics(), config, environment.admin(), environment.getObjectMapper());
    unprotectedAuthApi = AuthServlet.generalSetup(environment.metrics(), config, environment.servlets(), environment.getObjectMapper());
    // Create AdminServlet first to make it available to the realms
    admin = new AdminServlet(this);
    authController.externalInit(this, config.getAuthenticationRealms());
    // Register default components for the admin interface
    admin.register(this);
    log.info("Registering ResourcesProvider");
    for (Class<? extends ResourcesProvider> resourceProvider : CPSTypeIdResolver.listImplementations(ResourcesProvider.class)) {
        try {
            ResourcesProvider provider = resourceProvider.getConstructor().newInstance();
            provider.registerResources(this);
            providers.add(provider);
        } catch (Exception e) {
            log.error("Failed to register Resource {}", resourceProvider, e);
        }
    }
    try {
        formScanner.execute(null, null);
    } catch (Exception e) {
        Throwables.throwIfUnchecked(e);
        throw new RuntimeException(e);
    }
    environment.admin().addTask(formScanner);
    environment.admin().addTask(new QueryCleanupTask(storage, Duration.of(config.getQueries().getOldQueriesTime().getQuantity(), config.getQueries().getOldQueriesTime().getUnit().toChronoUnit())));
    environment.admin().addTask(new PermissionCleanupTask(storage));
    environment.admin().addTask(new ClearFilterSourceSearch());
    environment.admin().addTask(new ReportConsistencyTask(datasetRegistry));
    ShutdownTask shutdown = new ShutdownTask();
    environment.admin().addTask(shutdown);
    environment.lifecycle().addServerLifecycleListener(shutdown);
}
Also used : FormScanner(com.bakdata.conquery.models.forms.frontendconfiguration.FormScanner) ResourcesProvider(com.bakdata.conquery.resources.ResourcesProvider) JobManager(com.bakdata.conquery.models.jobs.JobManager) PermissionCleanupTask(com.bakdata.conquery.tasks.PermissionCleanupTask) ClearFilterSourceSearch(com.bakdata.conquery.tasks.ClearFilterSourceSearch) ReportConsistencyTask(com.bakdata.conquery.tasks.ReportConsistencyTask) JerseyClientBuilder(io.dropwizard.client.JerseyClientBuilder) QueryCleanupTask(com.bakdata.conquery.tasks.QueryCleanupTask) AuthorizationController(com.bakdata.conquery.models.auth.AuthorizationController) AdminServlet(com.bakdata.conquery.resources.admin.AdminServlet) MetaStorage(com.bakdata.conquery.io.storage.MetaStorage) ShutdownTask(com.bakdata.conquery.resources.admin.ShutdownTask) DatasetRegistry(com.bakdata.conquery.models.worker.DatasetRegistry)

Example 15 with MetaStorage

use of com.bakdata.conquery.io.storage.MetaStorage in project conquery by bakdata.

the class RoleHandlingOnGroupTest method execute.

@Override
public void execute(StandaloneSupport conquery) throws Exception {
    Dataset dataset1 = new Dataset();
    dataset1.setLabel("dataset1");
    MetaStorage storage = conquery.getMetaStorage();
    Group group1 = new Group("company", "company", storage);
    Role role = new Role("role1", "role1", storage);
    TestUser user1 = new TestUser(storage);
    try {
        storage.addRole(role);
        storage.addUser(user1);
        storage.addGroup(group1);
        role.addPermission(new DatasetPermission().instancePermission(Ability.READ, new DatasetId("testDataset")));
        // // Add user to group
        group1.addMember(user1);
        assertThat(user1.isPermitted(new DatasetPermission().instancePermission(Ability.READ, new DatasetId("testDataset")))).isFalse();
        // // Add role to group
        group1.addRole(role);
        assertThat(group1.getRoles()).containsExactlyInAnyOrder(role.getId());
        assertThat(user1.isPermitted(new DatasetPermission().instancePermission(Ability.READ, new DatasetId("testDataset")))).isTrue();
        // // Remove role from group
        group1.removeRole(role);
        assertThat(group1.getRoles()).isEmpty();
        assertThat(user1.isPermitted(new DatasetPermission().instancePermission(Ability.READ, new DatasetId("testDataset")))).isFalse();
    } finally {
        storage.removeGroup(group1.getId());
        storage.removeUser(user1.getId());
        storage.removeRole(role.getId());
    }
}
Also used : Role(com.bakdata.conquery.models.auth.entities.Role) Group(com.bakdata.conquery.models.auth.entities.Group) DatasetPermission(com.bakdata.conquery.models.auth.permissions.DatasetPermission) Dataset(com.bakdata.conquery.models.datasets.Dataset) MetaStorage(com.bakdata.conquery.io.storage.MetaStorage) DatasetId(com.bakdata.conquery.models.identifiable.ids.specific.DatasetId)

Aggregations

MetaStorage (com.bakdata.conquery.io.storage.MetaStorage)23 Dataset (com.bakdata.conquery.models.datasets.Dataset)12 User (com.bakdata.conquery.models.auth.entities.User)9 Role (com.bakdata.conquery.models.auth.entities.Role)8 QueryTest (com.bakdata.conquery.integration.json.QueryTest)7 StandaloneSupport (com.bakdata.conquery.util.support.StandaloneSupport)6 Response (javax.ws.rs.core.Response)6 Query (com.bakdata.conquery.apiv1.query.Query)5 Group (com.bakdata.conquery.models.auth.entities.Group)5 ExecutionState (com.bakdata.conquery.models.execution.ExecutionState)5 DatasetId (com.bakdata.conquery.models.identifiable.ids.specific.DatasetId)5 Namespace (com.bakdata.conquery.models.worker.Namespace)5 ShardNode (com.bakdata.conquery.commands.ShardNode)4 IntegrationUtils (com.bakdata.conquery.integration.common.IntegrationUtils)4 LoadingUtil (com.bakdata.conquery.integration.common.LoadingUtil)4 LoadingUtil.importSecondaryIds (com.bakdata.conquery.integration.common.LoadingUtil.importSecondaryIds)4 JsonIntegrationTest (com.bakdata.conquery.integration.json.JsonIntegrationTest)4 ModificationShieldedWorkerStorage (com.bakdata.conquery.io.storage.ModificationShieldedWorkerStorage)4 ValidatorHelper (com.bakdata.conquery.models.exceptions.ValidatorHelper)4 DatasetRegistry (com.bakdata.conquery.models.worker.DatasetRegistry)4