Examples with DuplicateNode com.bonree.brfs.duplication.filenode.duplicates.DuplicateNode used on opensource projects

Example 1 with DuplicateNode

use of com.bonree.brfs.duplication.filenode.duplicates.DuplicateNode in project BRFS by zhangnianli.

the class ResourceDuplicateNodeSelector method getRandom.

/**
 * 概述：随机返回数据节点
 * @param storageId
 * @param nums
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
private DuplicateNode[] getRandom(int storageId, int nums) {
    List<Service> serviceList = serviceManager.getServiceListByGroup(Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_DATA_SERVICE_GROUP_NAME));
    if (serviceList.isEmpty()) {
        LOG.info("[DUP] seviceList is empty");
        return new DuplicateNode[0];
    }
    List<DuplicateNode> nodes = new ArrayList<DuplicateNode>();
    while (!serviceList.isEmpty() && nodes.size() < nums) {
        Service service = serviceList.remove(rand.nextInt(serviceList.size()));
        DuplicateNode node = new DuplicateNode(service.getServiceGroup(), service.getServiceId());
        DiskNodeConnection conn = connectionPool.getConnection(node.getGroup(), node.getId());
        if (conn == null || conn.getClient() == null) {
            continue;
        }
        nodes.add(node);
    }
    DuplicateNode[] result = new DuplicateNode[nodes.size()];
    return nodes.toArray(result);
}

Example 2 with DuplicateNode

use of com.bonree.brfs.duplication.filenode.duplicates.DuplicateNode in project BRFS by zhangnianli.

the class DefaultFileObjectSyncProcessor method process.

@Override
public boolean process(FileObjectSyncTask task) {
    FileObject file = task.file();
    LOG.info("start to synchronize file[{}]", file.node().getName());
    DuplicateNode[] nodeList = file.node().getDuplicateNodes();
    boolean syncAccomplished = true;
    List<FileObjectSyncState> fileStateList = getFileStateList(file.node());
    if (fileStateList.isEmpty()) {
        // 文件所在的所有磁盘节点都处于异常状态
        LOG.error("No available duplicate node is found to sync file[{}]", file.node().getName());
        if (task.isExpired()) {
            task.callback().timeout(file);
            return true;
        }
        return false;
    }
    if (fileStateList.size() != nodeList.length) {
        // 文件所在的所有磁盘节点中有部分不可用，这种情况先同步可用的磁盘节点信息
        LOG.warn("Not all duplicate nodes are available to sync file[{}]", file.node().getName());
        syncAccomplished = false;
    }
    long maxLength = -1;
    for (FileObjectSyncState state : fileStateList) {
        maxLength = Math.max(maxLength, state.getFileLength());
    }
    List<FileObjectSyncState> lack = new ArrayList<FileObjectSyncState>();
    List<FileObjectSyncState> full = new ArrayList<FileObjectSyncState>();
    for (FileObjectSyncState state : fileStateList) {
        if (state.getFileLength() != maxLength) {
            lack.add(state);
        } else {
            full.add(state);
        }
    }
    if (lack.isEmpty()) {
        if (syncAccomplished) {
            LOG.info("file[{}] is ok!", file.node().getName());
            task.callback().complete(file, maxLength);
            return true;
        } else {
            LOG.info("file[{}] is lack of some duplicate node!", file.node().getName());
            if (task.isExpired()) {
                LOG.info("file[{}] sync is expired!", file.node().getName());
                task.callback().timeout(file);
                return true;
            }
            return false;
        }
    } else {
        syncAccomplished &= doSynchronize(file.node(), maxLength, lack, full);
        if (syncAccomplished) {
            LOG.info("file[{}] sync is completed!", file.node().getName());
            task.callback().complete(file, maxLength);
            return true;
        } else {
            LOG.info("file[{}] sync is failed!", file.node().getName());
            if (task.isExpired()) {
                LOG.info("file[{}] sync is expired!", file.node().getName());
                task.callback().timeout(file);
                return true;
            }
            return false;
        }
    }
}

Example 3 with DuplicateNode

use of com.bonree.brfs.duplication.filenode.duplicates.DuplicateNode in project BRFS by zhangnianli.

the class ResourceDuplicateNodeSelector method getResource.

private DuplicateNode[] getResource(int storageId, int nums, List<Pair<String, Integer>> servers) {
    List<Service> serviceList = serviceManager.getServiceListByGroup(Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_DATA_SERVICE_GROUP_NAME));
    if (serviceList.isEmpty()) {
        LOG.error("[{}] resource select serviceList is empty !!!", Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_DATA_SERVICE_GROUP_NAME));
        return new DuplicateNode[0];
    }
    Map<String, Service> map = new HashMap<String, Service>();
    for (Service server : serviceList) {
        map.put(server.getServiceId(), server);
    }
    String group = serviceList.get(0).getServiceGroup();
    List<DuplicateNode> nodes = new ArrayList<DuplicateNode>();
    List<String> uNeeds = new ArrayList<String>();
    long start = System.currentTimeMillis();
    while (!serviceList.isEmpty() && nodes.size() < nums) {
        String serverId = WeightRandomPattern.getWeightRandom(servers, rand, uNeeds);
        Service service = null;
        if (BrStringUtils.isEmpty(serverId) && !serviceList.isEmpty()) {
            service = serviceList.remove(rand.nextInt(serviceList.size()));
        } else {
            uNeeds.add(serverId);
            service = map.get(serverId);
        }
        if (service == null) {
            continue;
        }
        DuplicateNode node = new DuplicateNode(service.getServiceGroup(), service.getServiceId());
        DiskNodeConnection conn = connectionPool.getConnection(node.getGroup(), node.getId());
        if (conn == null || conn.getClient() == null) {
            continue;
        }
        serviceList.remove(service);
        nodes.add(node);
    }
    DuplicateNode[] result = new DuplicateNode[nodes.size()];
    long end = System.currentTimeMillis();
    LOG.info("select time {} ms, select services :{}", (end - start), uNeeds);
    return nodes.toArray(result);
}

Example 4 with DuplicateNode

use of com.bonree.brfs.duplication.filenode.duplicates.DuplicateNode in project BRFS by zhangnianli.

the class MinimalDuplicateNodeSelector method getDuplicationNodes.

@Override
public DuplicateNode[] getDuplicationNodes(int storageId, int nums) {
    List<Service> serviceList = serviceManager.getServiceListByGroup(Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_DATA_SERVICE_GROUP_NAME));
    if (serviceList.isEmpty()) {
        return new DuplicateNode[0];
    }
    List<DuplicateNode> nodes = new ArrayList<DuplicateNode>();
    while (!serviceList.isEmpty() && nodes.size() < nums) {
        Service service = serviceList.remove(rand.nextInt(serviceList.size()));
        DuplicateNode node = new DuplicateNode(service.getServiceGroup(), service.getServiceId());
        DiskNodeConnection conn = connectionPool.getConnection(node.getGroup(), node.getId());
        if (conn == null || conn.getClient() == null || !conn.getClient().ping()) {
            continue;
        }
        nodes.add(node);
    }
    DuplicateNode[] result = new DuplicateNode[nodes.size()];
    return nodes.toArray(result);
}

Example 5 with DuplicateNode

use of com.bonree.brfs.duplication.filenode.duplicates.DuplicateNode in project BRFS by zhangnianli.

the class ResourceWriteSelector method getDuplicationNodes.

@Override
public DuplicateNode[] getDuplicationNodes(int storageId, int nums) {
    long start = System.currentTimeMillis();
    DuplicateNode[] duplicateNodes;
    try {
        StorageRegion sr = this.storageRegionManager.findStorageRegionById(storageId);
        String srName = sr.getName();
        if (sr == null) {
            LOG.error("srid : {} is not exist !!!", storageId);
            return new DuplicateNode[0];
        }
        // 获取资源信息
        Collection<ResourceModel> resources = daemon.getClusterResources();
        // 采集资源未上传则使用备用选择器
        if (resources == null || resources.isEmpty()) {
            LOG.warn("[{}] select resource list is empty !!!! use bak selector", groupName);
            return this.bakSelector.getDuplicationNodes(storageId, nums);
        }
        // 过滤资源异常服务
        Collection<ResourceModel> selectors = this.resourceSelector.filterService(resources, srName);
        if (selectors == null || selectors.isEmpty()) {
            LOG.error("[{}] none avaible server to selector !!!", groupName);
            return new DuplicateNode[0];
        }
        // 按策略获取服务
        Collection<ResourceModel> wins = this.resourceSelector.selector(selectors, srName, nums);
        if (wins == null || wins.isEmpty()) {
            LOG.error("[{}] no service can write !!!", groupName);
            return new DuplicateNode[0];
        }
        if (wins.size() != nums) {
            LOG.warn("[{}] service can write !!!need {} but give {} ", groupName, nums, wins.size());
        }
        duplicateNodes = new DuplicateNode[wins.size()];
        Iterator<ResourceModel> iterator = wins.iterator();
        int i = 0;
        StringBuilder sBuild = new StringBuilder();
        sBuild.append("select service -> ");
        ResourceModel next;
        while (iterator.hasNext()) {
            next = iterator.next();
            duplicateNodes[i] = new DuplicateNode(groupName, next.getServerId());
            i++;
            sBuild.append(i).append(":").append(next.getServerId()).append("(").append(next.getHost()).append(", remainSize").append(next.getLocalRemainSizeValue(srName)).append("b ), ");
        }
        LOG.info("{}", sBuild.toString());
        return duplicateNodes;
    } finally {
        long stop = System.currentTimeMillis();
        LOG.info("resource select node time: {} ms", (stop - start));
    }
}