Search in sources :

Example 1 with ServerAssignment

use of com.alibaba.maxgraph.common.ServerAssignment in project GraphScope by alibaba.

the class PartitionManager method getServerAssignment.

public ServerAssignment getServerAssignment(Integer serverId) throws Exception {
    ServerAssignment serverAssignment = assignments.get(serverId);
    if (serverAssignment == null) {
        try (LockWrapper ignore = partitionLock.openWriteLock()) {
            serverAssignment = assignments.get(serverId);
            if (serverAssignment == null) {
                LOG.info("assigner:{}, instance config:{}", assigner, instanceConfig);
                List<Integer> assignment = assigner.getAssignment(instanceConfig.getPartitionNum(), instanceConfig.getResourceExecutorCount(), serverId, replicaCount);
                serverAssignment = new ServerAssignment(serverId, assignment);
                LOG.info("assign partition to server {}: {}", serverId, assignment);
                Map<Integer, ServerAssignment> temp = Maps.newHashMap(assignments);
                temp.put(serverId, serverAssignment);
                serverDataManager.namingProxy.persistentServerAssignment(temp);
                assignments.put(serverId, serverAssignment);
            }
            return serverAssignment;
        }
    }
    return assignments.get(serverId);
}
Also used : LockWrapper(com.alibaba.maxgraph.common.lock.LockWrapper) ServerAssignment(com.alibaba.maxgraph.common.ServerAssignment)

Example 2 with ServerAssignment

use of com.alibaba.maxgraph.common.ServerAssignment in project GraphScope by alibaba.

the class RemoteGraph method refresh.

@Override
public void refresh() throws Exception {
    InstanceStatus serverStatuses = clientManager.getServerDataApiClient().getServerStatuses();
    List<RemoteProxy> proxys = new ArrayList<>(serverStatuses.server2WorkerInfo.size());
    Set<WorkerInfo> workerInfoSet = new TreeSet<>(serverStatuses.getWorkInfo(RoleType.EXECUTOR));
    for (WorkerInfo v : workerInfoSet) {
        if (serverStatuses.assignments.containsKey(v.id)) {
            ServerAssignment serverAssignment = serverStatuses.readServerAssignment(v.id);
            serverAssignment.getPartitions().forEach(p -> partitionDistri.put(p, v.id));
            proxys.add(new RemoteProxy(v.endpoint.getIp(), v.endpoint.getPort(), 120L, schemaFetcher, this));
        }
    }
    List<Endpoint> endpointList = workerInfoSet.stream().map(workerInfo -> workerInfo.endpoint).collect(toList());
    LOG.info("proxys: {}", JSON.toJson(endpointList));
    List<RemoteProxy> prevProxyList = this.proxys;
    this.proxys = proxys;
    try {
        for (RemoteProxy remoteProxy : prevProxyList) {
            remoteProxy.close();
        }
    } finally {
        LOG.info("Close all previous remote proxy");
    }
    // TODO : dynamic configuration of different Graph Partitioner;
    this.partitioner = new DefaultGraphPartitioner(schemaFetcher.getPartitionNum());
}
Also used : GraphSchema(com.alibaba.maxgraph.compiler.api.schema.GraphSchema) java.util(java.util) ElementId(com.alibaba.maxgraph.sdkcommon.graph.ElementId) CompositeId(com.alibaba.maxgraph.sdkcommon.graph.CompositeId) LoggerFactory(org.slf4j.LoggerFactory) DefaultGraphPartitioner(com.alibaba.maxgraph.structure.DefaultGraphPartitioner) GraphPartitioner(com.alibaba.maxgraph.structure.GraphPartitioner) Edge(com.alibaba.maxgraph.structure.Edge) ServerAssignment(com.alibaba.maxgraph.common.ServerAssignment) Vertex(com.alibaba.maxgraph.structure.Vertex) Lists(com.google.common.collect.Lists) Pair(org.apache.commons.lang3.tuple.Pair) IteratorList(com.alibaba.maxgraph.iterator.IteratorList) WorkerInfo(com.alibaba.maxgraph.common.client.WorkerInfo) IteratorUtils(org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils) JSON(com.alibaba.maxgraph.sdkcommon.util.JSON) Triple(org.apache.commons.lang3.tuple.Triple) IdManager(com.alibaba.maxgraph.structure.IdManager) Logger(org.slf4j.Logger) SchemaFetcher(com.alibaba.maxgraph.compiler.api.schema.SchemaFetcher) InstanceStatus(com.alibaba.maxgraph.common.InstanceStatus) InstanceConfig(com.alibaba.maxgraph.common.cluster.InstanceConfig) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) RoleType(com.alibaba.maxgraph.proto.RoleType) MaxGraph(com.alibaba.maxgraph.structure.graph.MaxGraph) Maps(com.google.common.collect.Maps) Sets(com.google.common.collect.Sets) Direction(org.apache.tinkerpop.gremlin.structure.Direction) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Endpoint(com.alibaba.maxgraph.sdkcommon.client.Endpoint) Endpoint(com.alibaba.maxgraph.sdkcommon.client.Endpoint) InstanceStatus(com.alibaba.maxgraph.common.InstanceStatus) WorkerInfo(com.alibaba.maxgraph.common.client.WorkerInfo) DefaultGraphPartitioner(com.alibaba.maxgraph.structure.DefaultGraphPartitioner) ServerAssignment(com.alibaba.maxgraph.common.ServerAssignment)

Example 3 with ServerAssignment

use of com.alibaba.maxgraph.common.ServerAssignment in project GraphScope by alibaba.

the class DefaultPartitionGroupStrategy method getGroupInfo.

@Override
public List<List<Integer>> getGroupInfo(Map<Integer, ServerAssignment> assignments) {
    int storeNodeCount = assignments.size();
    Set<Set<Integer>> dedupPartitionsSet = new HashSet<>();
    for (ServerAssignment assignment : assignments.values()) {
        dedupPartitionsSet.add(assignment.getPartitions());
    }
    int nodesPerGroup = dedupPartitionsSet.size();
    if (storeNodeCount % nodesPerGroup != 0) {
        throw new IllegalArgumentException(String.format("Total node: %s, nodes per group: %s", storeNodeCount, nodesPerGroup));
    }
    // group -> nodes_list
    List<List<Integer>> groupInfo = new ArrayList<>();
    int serverID = 1;
    for (int i = 0; i < storeNodeCount / nodesPerGroup; i++) {
        List<Integer> group = new ArrayList<>();
        for (int j = 0; j < nodesPerGroup; j++) {
            group.add(serverID);
            serverID++;
        }
        groupInfo.add(group);
    }
    return groupInfo;
}
Also used : ServerAssignment(com.alibaba.maxgraph.common.ServerAssignment)

Example 4 with ServerAssignment

use of com.alibaba.maxgraph.common.ServerAssignment in project GraphScope by alibaba.

the class InstanceInfo method toSimpleProto.

public InstanceInfoProto.Builder toSimpleProto() {
    InstanceInfoProto.Builder builder = InstanceInfoProto.newBuilder();
    PartitionProtos.Builder pBuilder = PartitionProtos.newBuilder();
    for (Map.Entry<Integer, ServerAssignment> assignmentEntry : serverDataManager.partitionManager.assignments.entrySet()) {
        pBuilder.clear();
        pBuilder.addAllPartitionId(assignmentEntry.getValue().getPartitions());
        builder.putAssignment(assignmentEntry.getKey(), pBuilder.build());
    }
    InstanceInfoProto.Status status = getInstanceServingStatus();
    builder.setStatus(status);
    try (LockWrapper ignore = lock.openReadLock()) {
        WorkerInfoProtos.Builder wBuilder = WorkerInfoProtos.newBuilder();
        for (Map.Entry<RoleType, Collection<WorkerInfo>> workerInfo : workerInfoMap.asMap().entrySet()) {
            wBuilder.clear();
            for (WorkerInfo info : workerInfo.getValue()) {
                wBuilder.addInfos(info.toProto());
            }
            RoleType roleType = workerInfo.getKey();
            builder.putWorkerInfos(roleType.getNumber(), wBuilder.build());
        }
    }
    return builder;
}
Also used : WorkerInfo(com.alibaba.maxgraph.common.client.WorkerInfo) LockWrapper(com.alibaba.maxgraph.common.lock.LockWrapper) ConcurrentMap(java.util.concurrent.ConcurrentMap) ServerAssignment(com.alibaba.maxgraph.common.ServerAssignment)

Example 5 with ServerAssignment

use of com.alibaba.maxgraph.common.ServerAssignment in project GraphScope by alibaba.

the class InstanceInfo method onServerHeartBeat.

public void onServerHeartBeat(ServerHBResp.Builder builder, DataStatus serverStatus) throws Exception {
    // update status
    setServerStatus(serverStatus);
    updateWorkerExistTimestamp(serverStatus.serverId);
    updateWorkerRegisterTimestamp(serverStatus.serverId);
    ServerAssignment serverAssignment = serverDataManager.partitionManager.getServerAssignment(serverStatus.serverId);
    if (serverAssignment != null) {
        builder.addAllPartitions(serverAssignment.getPartitions());
        serverDataManager.runtimeManager.initRuntimeResp(builder, serverStatus);
        LOG.debug("executor id: {}, command is : {}", serverStatus.serverId, builder);
    }
}
Also used : ServerAssignment(com.alibaba.maxgraph.common.ServerAssignment)

Aggregations

ServerAssignment (com.alibaba.maxgraph.common.ServerAssignment)5 WorkerInfo (com.alibaba.maxgraph.common.client.WorkerInfo)2 LockWrapper (com.alibaba.maxgraph.common.lock.LockWrapper)2 InstanceStatus (com.alibaba.maxgraph.common.InstanceStatus)1 InstanceConfig (com.alibaba.maxgraph.common.cluster.InstanceConfig)1 GraphSchema (com.alibaba.maxgraph.compiler.api.schema.GraphSchema)1 SchemaFetcher (com.alibaba.maxgraph.compiler.api.schema.SchemaFetcher)1 IteratorList (com.alibaba.maxgraph.iterator.IteratorList)1 RoleType (com.alibaba.maxgraph.proto.RoleType)1 Endpoint (com.alibaba.maxgraph.sdkcommon.client.Endpoint)1 CompositeId (com.alibaba.maxgraph.sdkcommon.graph.CompositeId)1 ElementId (com.alibaba.maxgraph.sdkcommon.graph.ElementId)1 JSON (com.alibaba.maxgraph.sdkcommon.util.JSON)1 DefaultGraphPartitioner (com.alibaba.maxgraph.structure.DefaultGraphPartitioner)1 Edge (com.alibaba.maxgraph.structure.Edge)1 GraphPartitioner (com.alibaba.maxgraph.structure.GraphPartitioner)1 IdManager (com.alibaba.maxgraph.structure.IdManager)1 Vertex (com.alibaba.maxgraph.structure.Vertex)1 MaxGraph (com.alibaba.maxgraph.structure.graph.MaxGraph)1 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1