Search in sources :

Example 1 with WorkerInformation

use of com.bakdata.conquery.models.worker.WorkerInformation in project conquery by bakdata.

the class UIContext method getWorkerStatuses.

public boolean[] getWorkerStatuses() {
    boolean[] result = new boolean[namespaces.getShardNodes().values().size()];
    int id = 0;
    for (WorkerInformation wi : namespaces.getWorkers().values()) {
        result[id++] = wi.isConnected();
    }
    return result;
}
Also used : WorkerInformation(com.bakdata.conquery.models.worker.WorkerInformation)

Example 2 with WorkerInformation

use of com.bakdata.conquery.models.worker.WorkerInformation in project conquery by bakdata.

the class ImportJob method distributeWorkerResponsibilities.

private void distributeWorkerResponsibilities(DictionaryMapping primaryMapping) {
    log.debug("Updating bucket assignments.");
    synchronized (namespace) {
        for (int entity : primaryMapping.target()) {
            int bucket = Entity.getBucket(entity, bucketSize);
            if (namespace.getResponsibleWorkerForBucket(bucket) != null) {
                continue;
            }
            namespace.addResponsibility(bucket);
        }
        // While we hold the lock on the namespace distribute the new, consistent state among the workers
        for (WorkerInformation w : namespace.getWorkers()) {
            w.send(new UpdateWorkerBucket(w));
        }
    }
}
Also used : WorkerInformation(com.bakdata.conquery.models.worker.WorkerInformation)

Example 3 with WorkerInformation

use of com.bakdata.conquery.models.worker.WorkerInformation in project conquery by bakdata.

the class ImportJob method sendBuckets.

/**
 * select, then send buckets.
 */
private Map<WorkerId, Set<BucketId>> sendBuckets(Map<Integer, Integer> starts, Map<Integer, Integer> lengths, DictionaryMapping primaryMapping, Import imp, Map<Integer, List<Integer>> buckets2LocalEntities, ColumnStore[] storesSorted) throws JsonProcessingException {
    Map<WorkerId, Set<BucketId>> newWorkerAssignments = new HashMap<>();
    final ProgressReporter subJob = getProgressReporter().subJob(buckets2LocalEntities.size());
    for (Map.Entry<Integer, List<Integer>> bucket2entities : buckets2LocalEntities.entrySet()) {
        WorkerInformation responsibleWorker = Objects.requireNonNull(namespace.getResponsibleWorkerForBucket(bucket2entities.getKey()), () -> "No responsible worker for Bucket#" + bucket2entities.getKey());
        awaitFreeJobQueue(responsibleWorker);
        final Bucket bucket = selectBucket(starts, lengths, storesSorted, primaryMapping, imp, bucket2entities.getKey(), bucket2entities.getValue());
        newWorkerAssignments.computeIfAbsent(responsibleWorker.getId(), (ignored) -> new HashSet<>()).add(bucket.getId());
        log.trace("Sending Bucket[{}] to {}", bucket.getId(), responsibleWorker.getId());
        responsibleWorker.send(ImportBucket.forBucket(bucket));
        subJob.report(1);
    }
    subJob.done();
    return newWorkerAssignments;
}
Also used : Dictionary(com.bakdata.conquery.models.dictionary.Dictionary) ColumnStore(com.bakdata.conquery.models.events.stores.root.ColumnStore) java.util(java.util) ConqueryConfig(com.bakdata.conquery.models.config.ConqueryConfig) Getter(lombok.Getter) PreprocessedHeader(com.bakdata.conquery.models.preproc.PreprocessedHeader) RequiredArgsConstructor(lombok.RequiredArgsConstructor) PreprocessedData(com.bakdata.conquery.models.preproc.PreprocessedData) com.bakdata.conquery.models.identifiable.ids.specific(com.bakdata.conquery.models.identifiable.ids.specific) NamespaceStorage(com.bakdata.conquery.io.storage.NamespaceStorage) IdMutex(com.bakdata.conquery.models.identifiable.IdMutex) com.bakdata.conquery.models.datasets(com.bakdata.conquery.models.datasets) IntegerStore(com.bakdata.conquery.models.events.stores.root.IntegerStore) WorkerInformation(com.bakdata.conquery.models.worker.WorkerInformation) BadRequestException(javax.ws.rs.BadRequestException) JSONException(com.bakdata.conquery.models.exceptions.JSONException) DictionaryMapping(com.bakdata.conquery.models.dictionary.DictionaryMapping) IntegerParser(com.bakdata.conquery.models.preproc.parser.specific.IntegerParser) PreprocessedReader(com.bakdata.conquery.models.preproc.PreprocessedReader) ResourceUtil(com.bakdata.conquery.util.ResourceUtil) Bucket(com.bakdata.conquery.models.events.Bucket) MajorTypeId(com.bakdata.conquery.models.events.MajorTypeId) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) IOException(java.io.IOException) StringStore(com.bakdata.conquery.models.events.stores.root.StringStore) com.bakdata.conquery.models.messages.namespaces.specific(com.bakdata.conquery.models.messages.namespaces.specific) Collectors(java.util.stream.Collectors) Entity(com.bakdata.conquery.models.query.entity.Entity) ProgressReporter(com.bakdata.conquery.util.progressreporter.ProgressReporter) Slf4j(lombok.extern.slf4j.Slf4j) IntList(it.unimi.dsi.fastutil.ints.IntList) Response(javax.ws.rs.core.Response) IntOpenHashSet(it.unimi.dsi.fastutil.ints.IntOpenHashSet) WebApplicationException(javax.ws.rs.WebApplicationException) IntSet(it.unimi.dsi.fastutil.ints.IntSet) PreprocessedDictionaries(com.bakdata.conquery.models.preproc.PreprocessedDictionaries) IntArrayList(it.unimi.dsi.fastutil.ints.IntArrayList) Namespace(com.bakdata.conquery.models.worker.Namespace) InputStream(java.io.InputStream) WorkerInformation(com.bakdata.conquery.models.worker.WorkerInformation) IntOpenHashSet(it.unimi.dsi.fastutil.ints.IntOpenHashSet) IntSet(it.unimi.dsi.fastutil.ints.IntSet) ProgressReporter(com.bakdata.conquery.util.progressreporter.ProgressReporter) Bucket(com.bakdata.conquery.models.events.Bucket) IntList(it.unimi.dsi.fastutil.ints.IntList) IntArrayList(it.unimi.dsi.fastutil.ints.IntArrayList) IntOpenHashSet(it.unimi.dsi.fastutil.ints.IntOpenHashSet)

Example 4 with WorkerInformation

use of com.bakdata.conquery.models.worker.WorkerInformation in project conquery by bakdata.

the class ShardNode method sessionOpened.

@Override
public void sessionOpened(IoSession session) throws Exception {
    setLocation(session);
    NetworkSession networkSession = new NetworkSession(session);
    context = new NetworkMessageContext.ShardNodeNetworkContext(jobManager, networkSession, workers, config, validator);
    log.info("Connected to ManagerNode @ {}", session.getRemoteAddress());
    // Authenticate with ManagerNode
    context.send(new AddShardNode());
    for (Worker w : workers.getWorkers().values()) {
        w.setSession(new NetworkSession(session));
        WorkerInformation info = w.getInfo();
        log.info("Sending worker identity '{}'", info.getName());
        networkSession.send(new RegisterWorker(info));
    }
}
Also used : WorkerInformation(com.bakdata.conquery.models.worker.WorkerInformation) NetworkSession(com.bakdata.conquery.io.mina.NetworkSession) RegisterWorker(com.bakdata.conquery.models.messages.network.specific.RegisterWorker) NetworkMessageContext(com.bakdata.conquery.models.messages.network.NetworkMessageContext) AddShardNode(com.bakdata.conquery.models.messages.network.specific.AddShardNode) Worker(com.bakdata.conquery.models.worker.Worker) RegisterWorker(com.bakdata.conquery.models.messages.network.specific.RegisterWorker) ShardNodeNetworkContext(com.bakdata.conquery.models.messages.network.NetworkMessageContext.ShardNodeNetworkContext)

Aggregations

WorkerInformation (com.bakdata.conquery.models.worker.WorkerInformation)4 NetworkSession (com.bakdata.conquery.io.mina.NetworkSession)1 NamespaceStorage (com.bakdata.conquery.io.storage.NamespaceStorage)1 ConqueryConfig (com.bakdata.conquery.models.config.ConqueryConfig)1 com.bakdata.conquery.models.datasets (com.bakdata.conquery.models.datasets)1 Dictionary (com.bakdata.conquery.models.dictionary.Dictionary)1 DictionaryMapping (com.bakdata.conquery.models.dictionary.DictionaryMapping)1 Bucket (com.bakdata.conquery.models.events.Bucket)1 MajorTypeId (com.bakdata.conquery.models.events.MajorTypeId)1 ColumnStore (com.bakdata.conquery.models.events.stores.root.ColumnStore)1 IntegerStore (com.bakdata.conquery.models.events.stores.root.IntegerStore)1 StringStore (com.bakdata.conquery.models.events.stores.root.StringStore)1 JSONException (com.bakdata.conquery.models.exceptions.JSONException)1 IdMutex (com.bakdata.conquery.models.identifiable.IdMutex)1 com.bakdata.conquery.models.identifiable.ids.specific (com.bakdata.conquery.models.identifiable.ids.specific)1 com.bakdata.conquery.models.messages.namespaces.specific (com.bakdata.conquery.models.messages.namespaces.specific)1 NetworkMessageContext (com.bakdata.conquery.models.messages.network.NetworkMessageContext)1 ShardNodeNetworkContext (com.bakdata.conquery.models.messages.network.NetworkMessageContext.ShardNodeNetworkContext)1 AddShardNode (com.bakdata.conquery.models.messages.network.specific.AddShardNode)1 RegisterWorker (com.bakdata.conquery.models.messages.network.specific.RegisterWorker)1