Search in sources :

Example 1 with JobManager

use of com.bakdata.conquery.models.jobs.JobManager in project conquery by bakdata.

the class ManagerNode method run.

public void run(ConqueryConfig config, Environment environment) throws InterruptedException {
    this.environment = environment;
    validator = environment.getValidator();
    client = new JerseyClientBuilder(environment).using(config.getJerseyClient()).build(getName());
    // Instantiate DatasetRegistry and MetaStorage so they are ready for injection into the object mapper (API + Storage)
    // The validator is already injected at this point see Conquery.java
    datasetRegistry = new DatasetRegistry(config.getCluster().getEntityBucketSize());
    storage = new MetaStorage(datasetRegistry);
    datasetRegistry.injectInto(environment.getObjectMapper());
    storage.injectInto(environment.getObjectMapper());
    jobManager = new JobManager("ManagerNode", config.isFailOnError());
    formScanner = new FormScanner();
    this.config = config;
    config.initialize(this);
    // Initialization of internationalization
    I18n.init();
    RESTServer.configure(config, environment.jersey().getResourceConfig());
    maintenanceService = environment.lifecycle().scheduledExecutorService("Maintenance Service").build();
    environment.lifecycle().manage(this);
    loadNamespaces();
    loadMetaStorage();
    authController = new AuthorizationController(storage, config.getAuthorizationRealms());
    environment.lifecycle().manage(authController);
    unprotectedAuthAdmin = AuthServlet.generalSetup(environment.metrics(), config, environment.admin(), environment.getObjectMapper());
    unprotectedAuthApi = AuthServlet.generalSetup(environment.metrics(), config, environment.servlets(), environment.getObjectMapper());
    // Create AdminServlet first to make it available to the realms
    admin = new AdminServlet(this);
    authController.externalInit(this, config.getAuthenticationRealms());
    // Register default components for the admin interface
    admin.register(this);
    log.info("Registering ResourcesProvider");
    for (Class<? extends ResourcesProvider> resourceProvider : CPSTypeIdResolver.listImplementations(ResourcesProvider.class)) {
        try {
            ResourcesProvider provider = resourceProvider.getConstructor().newInstance();
            provider.registerResources(this);
            providers.add(provider);
        } catch (Exception e) {
            log.error("Failed to register Resource {}", resourceProvider, e);
        }
    }
    try {
        formScanner.execute(null, null);
    } catch (Exception e) {
        Throwables.throwIfUnchecked(e);
        throw new RuntimeException(e);
    }
    environment.admin().addTask(formScanner);
    environment.admin().addTask(new QueryCleanupTask(storage, Duration.of(config.getQueries().getOldQueriesTime().getQuantity(), config.getQueries().getOldQueriesTime().getUnit().toChronoUnit())));
    environment.admin().addTask(new PermissionCleanupTask(storage));
    environment.admin().addTask(new ClearFilterSourceSearch());
    environment.admin().addTask(new ReportConsistencyTask(datasetRegistry));
    ShutdownTask shutdown = new ShutdownTask();
    environment.admin().addTask(shutdown);
    environment.lifecycle().addServerLifecycleListener(shutdown);
}
Also used : FormScanner(com.bakdata.conquery.models.forms.frontendconfiguration.FormScanner) ResourcesProvider(com.bakdata.conquery.resources.ResourcesProvider) JobManager(com.bakdata.conquery.models.jobs.JobManager) PermissionCleanupTask(com.bakdata.conquery.tasks.PermissionCleanupTask) ClearFilterSourceSearch(com.bakdata.conquery.tasks.ClearFilterSourceSearch) ReportConsistencyTask(com.bakdata.conquery.tasks.ReportConsistencyTask) JerseyClientBuilder(io.dropwizard.client.JerseyClientBuilder) QueryCleanupTask(com.bakdata.conquery.tasks.QueryCleanupTask) AuthorizationController(com.bakdata.conquery.models.auth.AuthorizationController) AdminServlet(com.bakdata.conquery.resources.admin.AdminServlet) MetaStorage(com.bakdata.conquery.io.storage.MetaStorage) ShutdownTask(com.bakdata.conquery.resources.admin.ShutdownTask) DatasetRegistry(com.bakdata.conquery.models.worker.DatasetRegistry)

Example 2 with JobManager

use of com.bakdata.conquery.models.jobs.JobManager in project conquery by bakdata.

the class ShardNode method run.

@Override
protected void run(Environment environment, Namespace namespace, ConqueryConfig config) throws Exception {
    this.environment = environment;
    connector = new NioSocketConnector();
    jobManager = new JobManager(getName(), config.isFailOnError());
    synchronized (environment) {
        environment.lifecycle().manage(this);
        validator = environment.getValidator();
        scheduler = environment.lifecycle().scheduledExecutorService("Scheduled Messages").build();
    }
    this.config = config;
    config.initialize(this);
    scheduler.scheduleAtFixedRate(this::reportJobManagerStatus, 30, 1, TimeUnit.SECONDS);
    final ObjectMapper binaryMapper = config.configureObjectMapper(Jackson.copyMapperAndInjectables(Jackson.BINARY_MAPPER));
    ((MutableInjectableValues) binaryMapper.getInjectableValues()).add(Validator.class, environment.getValidator());
    workers = new Workers(getConfig().getQueries().getExecutionPool(), binaryMapper, getConfig().getCluster().getEntityBucketSize());
    final Collection<WorkerStorage> workerStorages = config.getStorage().loadWorkerStorages();
    for (WorkerStorage workerStorage : workerStorages) {
        workers.createWorker(workerStorage, config.isFailOnError());
    }
    log.info("All Worker Storages loaded: {}", workers.getWorkers().size());
}
Also used : NioSocketConnector(org.apache.mina.transport.socket.nio.NioSocketConnector) Workers(com.bakdata.conquery.models.worker.Workers) MutableInjectableValues(com.bakdata.conquery.io.jackson.MutableInjectableValues) JobManager(com.bakdata.conquery.models.jobs.JobManager) WorkerStorage(com.bakdata.conquery.io.storage.WorkerStorage) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper)

Aggregations

JobManager (com.bakdata.conquery.models.jobs.JobManager)2 MutableInjectableValues (com.bakdata.conquery.io.jackson.MutableInjectableValues)1 MetaStorage (com.bakdata.conquery.io.storage.MetaStorage)1 WorkerStorage (com.bakdata.conquery.io.storage.WorkerStorage)1 AuthorizationController (com.bakdata.conquery.models.auth.AuthorizationController)1 FormScanner (com.bakdata.conquery.models.forms.frontendconfiguration.FormScanner)1 DatasetRegistry (com.bakdata.conquery.models.worker.DatasetRegistry)1 Workers (com.bakdata.conquery.models.worker.Workers)1 ResourcesProvider (com.bakdata.conquery.resources.ResourcesProvider)1 AdminServlet (com.bakdata.conquery.resources.admin.AdminServlet)1 ShutdownTask (com.bakdata.conquery.resources.admin.ShutdownTask)1 ClearFilterSourceSearch (com.bakdata.conquery.tasks.ClearFilterSourceSearch)1 PermissionCleanupTask (com.bakdata.conquery.tasks.PermissionCleanupTask)1 QueryCleanupTask (com.bakdata.conquery.tasks.QueryCleanupTask)1 ReportConsistencyTask (com.bakdata.conquery.tasks.ReportConsistencyTask)1 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 JerseyClientBuilder (io.dropwizard.client.JerseyClientBuilder)1 NioSocketConnector (org.apache.mina.transport.socket.nio.NioSocketConnector)1