use of com.alibaba.maxgraph.common.ServerAssignment in project GraphScope by alibaba.
the class PartitionManager method getServerAssignment.
public ServerAssignment getServerAssignment(Integer serverId) throws Exception {
ServerAssignment serverAssignment = assignments.get(serverId);
if (serverAssignment == null) {
try (LockWrapper ignore = partitionLock.openWriteLock()) {
serverAssignment = assignments.get(serverId);
if (serverAssignment == null) {
LOG.info("assigner:{}, instance config:{}", assigner, instanceConfig);
List<Integer> assignment = assigner.getAssignment(instanceConfig.getPartitionNum(), instanceConfig.getResourceExecutorCount(), serverId, replicaCount);
serverAssignment = new ServerAssignment(serverId, assignment);
LOG.info("assign partition to server {}: {}", serverId, assignment);
Map<Integer, ServerAssignment> temp = Maps.newHashMap(assignments);
temp.put(serverId, serverAssignment);
serverDataManager.namingProxy.persistentServerAssignment(temp);
assignments.put(serverId, serverAssignment);
}
return serverAssignment;
}
}
return assignments.get(serverId);
}
use of com.alibaba.maxgraph.common.ServerAssignment in project GraphScope by alibaba.
the class RemoteGraph method refresh.
@Override
public void refresh() throws Exception {
InstanceStatus serverStatuses = clientManager.getServerDataApiClient().getServerStatuses();
List<RemoteProxy> proxys = new ArrayList<>(serverStatuses.server2WorkerInfo.size());
Set<WorkerInfo> workerInfoSet = new TreeSet<>(serverStatuses.getWorkInfo(RoleType.EXECUTOR));
for (WorkerInfo v : workerInfoSet) {
if (serverStatuses.assignments.containsKey(v.id)) {
ServerAssignment serverAssignment = serverStatuses.readServerAssignment(v.id);
serverAssignment.getPartitions().forEach(p -> partitionDistri.put(p, v.id));
proxys.add(new RemoteProxy(v.endpoint.getIp(), v.endpoint.getPort(), 120L, schemaFetcher, this));
}
}
List<Endpoint> endpointList = workerInfoSet.stream().map(workerInfo -> workerInfo.endpoint).collect(toList());
LOG.info("proxys: {}", JSON.toJson(endpointList));
List<RemoteProxy> prevProxyList = this.proxys;
this.proxys = proxys;
try {
for (RemoteProxy remoteProxy : prevProxyList) {
remoteProxy.close();
}
} finally {
LOG.info("Close all previous remote proxy");
}
// TODO : dynamic configuration of different Graph Partitioner;
this.partitioner = new DefaultGraphPartitioner(schemaFetcher.getPartitionNum());
}
use of com.alibaba.maxgraph.common.ServerAssignment in project GraphScope by alibaba.
the class DefaultPartitionGroupStrategy method getGroupInfo.
@Override
public List<List<Integer>> getGroupInfo(Map<Integer, ServerAssignment> assignments) {
int storeNodeCount = assignments.size();
Set<Set<Integer>> dedupPartitionsSet = new HashSet<>();
for (ServerAssignment assignment : assignments.values()) {
dedupPartitionsSet.add(assignment.getPartitions());
}
int nodesPerGroup = dedupPartitionsSet.size();
if (storeNodeCount % nodesPerGroup != 0) {
throw new IllegalArgumentException(String.format("Total node: %s, nodes per group: %s", storeNodeCount, nodesPerGroup));
}
// group -> nodes_list
List<List<Integer>> groupInfo = new ArrayList<>();
int serverID = 1;
for (int i = 0; i < storeNodeCount / nodesPerGroup; i++) {
List<Integer> group = new ArrayList<>();
for (int j = 0; j < nodesPerGroup; j++) {
group.add(serverID);
serverID++;
}
groupInfo.add(group);
}
return groupInfo;
}
use of com.alibaba.maxgraph.common.ServerAssignment in project GraphScope by alibaba.
the class InstanceInfo method toSimpleProto.
public InstanceInfoProto.Builder toSimpleProto() {
InstanceInfoProto.Builder builder = InstanceInfoProto.newBuilder();
PartitionProtos.Builder pBuilder = PartitionProtos.newBuilder();
for (Map.Entry<Integer, ServerAssignment> assignmentEntry : serverDataManager.partitionManager.assignments.entrySet()) {
pBuilder.clear();
pBuilder.addAllPartitionId(assignmentEntry.getValue().getPartitions());
builder.putAssignment(assignmentEntry.getKey(), pBuilder.build());
}
InstanceInfoProto.Status status = getInstanceServingStatus();
builder.setStatus(status);
try (LockWrapper ignore = lock.openReadLock()) {
WorkerInfoProtos.Builder wBuilder = WorkerInfoProtos.newBuilder();
for (Map.Entry<RoleType, Collection<WorkerInfo>> workerInfo : workerInfoMap.asMap().entrySet()) {
wBuilder.clear();
for (WorkerInfo info : workerInfo.getValue()) {
wBuilder.addInfos(info.toProto());
}
RoleType roleType = workerInfo.getKey();
builder.putWorkerInfos(roleType.getNumber(), wBuilder.build());
}
}
return builder;
}
use of com.alibaba.maxgraph.common.ServerAssignment in project GraphScope by alibaba.
the class InstanceInfo method onServerHeartBeat.
public void onServerHeartBeat(ServerHBResp.Builder builder, DataStatus serverStatus) throws Exception {
// update status
setServerStatus(serverStatus);
updateWorkerExistTimestamp(serverStatus.serverId);
updateWorkerRegisterTimestamp(serverStatus.serverId);
ServerAssignment serverAssignment = serverDataManager.partitionManager.getServerAssignment(serverStatus.serverId);
if (serverAssignment != null) {
builder.addAllPartitions(serverAssignment.getPartitions());
serverDataManager.runtimeManager.initRuntimeResp(builder, serverStatus);
LOG.debug("executor id: {}, command is : {}", serverStatus.serverId, builder);
}
}
Aggregations