use of com.bakdata.conquery.models.worker.WorkerInformation in project conquery by bakdata.
the class UIContext method getWorkerStatuses.
public boolean[] getWorkerStatuses() {
boolean[] result = new boolean[namespaces.getShardNodes().values().size()];
int id = 0;
for (WorkerInformation wi : namespaces.getWorkers().values()) {
result[id++] = wi.isConnected();
}
return result;
}
use of com.bakdata.conquery.models.worker.WorkerInformation in project conquery by bakdata.
the class ImportJob method distributeWorkerResponsibilities.
private void distributeWorkerResponsibilities(DictionaryMapping primaryMapping) {
log.debug("Updating bucket assignments.");
synchronized (namespace) {
for (int entity : primaryMapping.target()) {
int bucket = Entity.getBucket(entity, bucketSize);
if (namespace.getResponsibleWorkerForBucket(bucket) != null) {
continue;
}
namespace.addResponsibility(bucket);
}
// While we hold the lock on the namespace distribute the new, consistent state among the workers
for (WorkerInformation w : namespace.getWorkers()) {
w.send(new UpdateWorkerBucket(w));
}
}
}
use of com.bakdata.conquery.models.worker.WorkerInformation in project conquery by bakdata.
the class ImportJob method sendBuckets.
/**
* select, then send buckets.
*/
private Map<WorkerId, Set<BucketId>> sendBuckets(Map<Integer, Integer> starts, Map<Integer, Integer> lengths, DictionaryMapping primaryMapping, Import imp, Map<Integer, List<Integer>> buckets2LocalEntities, ColumnStore[] storesSorted) throws JsonProcessingException {
Map<WorkerId, Set<BucketId>> newWorkerAssignments = new HashMap<>();
final ProgressReporter subJob = getProgressReporter().subJob(buckets2LocalEntities.size());
for (Map.Entry<Integer, List<Integer>> bucket2entities : buckets2LocalEntities.entrySet()) {
WorkerInformation responsibleWorker = Objects.requireNonNull(namespace.getResponsibleWorkerForBucket(bucket2entities.getKey()), () -> "No responsible worker for Bucket#" + bucket2entities.getKey());
awaitFreeJobQueue(responsibleWorker);
final Bucket bucket = selectBucket(starts, lengths, storesSorted, primaryMapping, imp, bucket2entities.getKey(), bucket2entities.getValue());
newWorkerAssignments.computeIfAbsent(responsibleWorker.getId(), (ignored) -> new HashSet<>()).add(bucket.getId());
log.trace("Sending Bucket[{}] to {}", bucket.getId(), responsibleWorker.getId());
responsibleWorker.send(ImportBucket.forBucket(bucket));
subJob.report(1);
}
subJob.done();
return newWorkerAssignments;
}
use of com.bakdata.conquery.models.worker.WorkerInformation in project conquery by bakdata.
the class ShardNode method sessionOpened.
@Override
public void sessionOpened(IoSession session) throws Exception {
setLocation(session);
NetworkSession networkSession = new NetworkSession(session);
context = new NetworkMessageContext.ShardNodeNetworkContext(jobManager, networkSession, workers, config, validator);
log.info("Connected to ManagerNode @ {}", session.getRemoteAddress());
// Authenticate with ManagerNode
context.send(new AddShardNode());
for (Worker w : workers.getWorkers().values()) {
w.setSession(new NetworkSession(session));
WorkerInformation info = w.getInfo();
log.info("Sending worker identity '{}'", info.getName());
networkSession.send(new RegisterWorker(info));
}
}
Aggregations