Examples with DiskNodeConnection com.bonree.brfs.duplication.datastream.connection.DiskNodeConnection used on opensource projects

Example 1 with DiskNodeConnection

use of com.bonree.brfs.duplication.datastream.connection.DiskNodeConnection in project BRFS by zhangnianli.

the class ResourceDuplicateNodeSelector method getRandom.

/**
 * 概述：随机返回数据节点
 * @param storageId
 * @param nums
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
private DuplicateNode[] getRandom(int storageId, int nums) {
    List<Service> serviceList = serviceManager.getServiceListByGroup(Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_DATA_SERVICE_GROUP_NAME));
    if (serviceList.isEmpty()) {
        LOG.info("[DUP] seviceList is empty");
        return new DuplicateNode[0];
    }
    List<DuplicateNode> nodes = new ArrayList<DuplicateNode>();
    while (!serviceList.isEmpty() && nodes.size() < nums) {
        Service service = serviceList.remove(rand.nextInt(serviceList.size()));
        DuplicateNode node = new DuplicateNode(service.getServiceGroup(), service.getServiceId());
        DiskNodeConnection conn = connectionPool.getConnection(node.getGroup(), node.getId());
        if (conn == null || conn.getClient() == null) {
            continue;
        }
        nodes.add(node);
    }
    DuplicateNode[] result = new DuplicateNode[nodes.size()];
    return nodes.toArray(result);
}

Example 2 with DiskNodeConnection

use of com.bonree.brfs.duplication.datastream.connection.DiskNodeConnection in project BRFS by zhangnianli.

the class MachineResourceWriterSelector method selectNode.

/**
 * 服务选择
 * @param intValues
 * @param num
 * @return
 */
public List<ResourceModel> selectNode(DiskNodeConnectionPool pool, Map<String, ResourceModel> map, List<Pair<String, Integer>> intValues, String groupName, int num) {
    List<ResourceModel> resourceModels = new ArrayList<>();
    String key;
    String ip;
    ResourceModel tmp;
    DiskNodeConnection conn;
    // ip选中优先选择
    Set<String> ips = new HashSet<>();
    List<String> uneedServices = new ArrayList<>();
    int tSize = map.size();
    // 按资源选择
    while (resourceModels.size() != num && resourceModels.size() != tSize && uneedServices.size() != tSize) {
        key = WeightRandomPattern.getWeightRandom(intValues, new Random(), uneedServices);
        tmp = map.get(key);
        ip = tmp.getHost();
        if (pool != null) {
            conn = pool.getConnection(groupName, key);
            if (conn == null || !conn.isValid()) {
                LOG.warn("{} :[{}({})]is unused !!", groupName, key, ip);
                uneedServices.add(key);
                continue;
            }
        }
        // 不同ip的添加
        if (ips.add(ip)) {
            resourceModels.add(tmp);
        } else {
            LOG.info("{} is selectd !! get next", ip);
        }
        uneedServices.add(tmp.getServerId());
    }
    return resourceModels;
}

Example 3 with DiskNodeConnection

use of com.bonree.brfs.duplication.datastream.connection.DiskNodeConnection in project BRFS by zhangnianli.

the class MachineResourceWriterSelector method selectRandom.

/**
 * 随机选择
 * @param pool
 * @param map
 * @param sids
 * @param intValues
 * @param groupName
 * @param sn
 * @param num
 * @return
 */
public Collection<ResourceModel> selectRandom(DiskNodeConnectionPool pool, Map<String, ResourceModel> map, Set<String> sids, List<Pair<String, Integer>> intValues, String groupName, String sn, int num) {
    List<ResourceModel> resourceModels = new ArrayList<>();
    String key;
    String ip;
    ResourceModel tmp;
    DiskNodeConnection conn;
    // ip选中优先选择
    int tSize = map.size();
    // 按资源选择
    Random random = new Random();
    boolean sendFlag = System.currentTimeMillis() - repeatTime > INVERTTIME;
    repeatTime = sendFlag ? System.currentTimeMillis() : repeatTime;
    while (resourceModels.size() != num && resourceModels.size() != tSize && sids.size() != tSize) {
        key = WeightRandomPattern.getWeightRandom(intValues, random, sids);
        tmp = map.get(key);
        ip = tmp.getHost();
        if (pool != null) {
            conn = pool.getConnection(groupName, key);
            if (conn == null || !conn.isValid()) {
                LOG.warn("{} :[{}({})]is unused !!", groupName, key, ip);
                sids.add(key);
                continue;
            }
        }
        if (sendFlag) {
            EmailPool emailPool = EmailPool.getInstance();
            MailWorker.Builder builder = MailWorker.newBuilder(emailPool.getProgramInfo()).setModel(this.getClass().getSimpleName() + "服务选择").setMessage("sr [" + sn + "]即将 在 " + key + "(" + ip + ") 服务 写入重复数据");
            emailPool.sendEmail(builder);
        }
        resourceModels.add(tmp);
        sids.add(tmp.getServerId());
    }
    return resourceModels;
}

Example 4 with DiskNodeConnection

use of com.bonree.brfs.duplication.datastream.connection.DiskNodeConnection in project BRFS by zhangnianli.

the class ResourceDuplicateNodeSelector method getResource.

private DuplicateNode[] getResource(int storageId, int nums, List<Pair<String, Integer>> servers) {
    List<Service> serviceList = serviceManager.getServiceListByGroup(Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_DATA_SERVICE_GROUP_NAME));
    if (serviceList.isEmpty()) {
        LOG.error("[{}] resource select serviceList is empty !!!", Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_DATA_SERVICE_GROUP_NAME));
        return new DuplicateNode[0];
    }
    Map<String, Service> map = new HashMap<String, Service>();
    for (Service server : serviceList) {
        map.put(server.getServiceId(), server);
    }
    String group = serviceList.get(0).getServiceGroup();
    List<DuplicateNode> nodes = new ArrayList<DuplicateNode>();
    List<String> uNeeds = new ArrayList<String>();
    long start = System.currentTimeMillis();
    while (!serviceList.isEmpty() && nodes.size() < nums) {
        String serverId = WeightRandomPattern.getWeightRandom(servers, rand, uNeeds);
        Service service = null;
        if (BrStringUtils.isEmpty(serverId) && !serviceList.isEmpty()) {
            service = serviceList.remove(rand.nextInt(serviceList.size()));
        } else {
            uNeeds.add(serverId);
            service = map.get(serverId);
        }
        if (service == null) {
            continue;
        }
        DuplicateNode node = new DuplicateNode(service.getServiceGroup(), service.getServiceId());
        DiskNodeConnection conn = connectionPool.getConnection(node.getGroup(), node.getId());
        if (conn == null || conn.getClient() == null) {
            continue;
        }
        serviceList.remove(service);
        nodes.add(node);
    }
    DuplicateNode[] result = new DuplicateNode[nodes.size()];
    long end = System.currentTimeMillis();
    LOG.info("select time {} ms, select services :{}", (end - start), uNeeds);
    return nodes.toArray(result);
}

Example 5 with DiskNodeConnection

use of com.bonree.brfs.duplication.datastream.connection.DiskNodeConnection in project BRFS by zhangnianli.

the class DefaultFileObjectFactory method createFile.

@Override
public FileObject createFile(StorageRegion storageRegion) {
    DuplicateNode[] nodes = duplicationNodeSelector.getDuplicationNodes(storageRegion.getId(), storageRegion.getReplicateNum());
    if (nodes.length == 0) {
        LOG.error("No available duplication node to build FileNode");
        // 没有磁盘节点可用
        return null;
    }
    FileNode.Builder fileNodeBuilder = FileNode.newBuilder().setStorageName(storageRegion.getName()).setStorageId(storageRegion.getId()).setServiceId(service.getServiceId()).setServiceGroup(service.getServiceGroup()).setName(FileNameBuilder.createFile(idManager, storageRegion, nodes)).setDuplicateNodes(nodes).setTimeDuration(Duration.parse(storageRegion.getFilePartitionDuration()).toMillis());
    long capacity = -1;
    for (DuplicateNode node : nodes) {
        LOG.info("start init node[{}] for file[{}]", node, fileNodeBuilder.build().getName());
        DiskNodeConnection connection = connectionPool.getConnection(node.getGroup(), node.getId());
        if (connection == null || connection.getClient() == null) {
            LOG.info("can not write header for file[{}] because [{}] is disconnected", fileNodeBuilder.build().getName(), node);
            continue;
        }
        String serverId = idManager.getOtherSecondID(node.getId(), storageRegion.getId());
        String filePath = FilePathBuilder.buildFilePath(fileNodeBuilder.build(), serverId);
        LOG.info("client opening file [{}]", filePath);
        long result = connection.getClient().openFile(filePath, storageRegion.getFileCapacity());
        if (result < 0) {
            continue;
        }
        LOG.info("open file[{}] from datanode[{}] with capacity[{}]", filePath, node, result);
        if (capacity < 0) {
            capacity = result;
            continue;
        }
        if (capacity != result) {
            LOG.error("different capacity be received from different dupcate nodes");
            capacity = -1;
            break;
        }
    }
    // 如果没有一个磁盘节点写入头数据成功，则放弃使用此文件节点
    if (capacity < 0) {
        LOG.error("can not open file at any duplicate node for file[{}]", fileNodeBuilder.build().getName());
        return null;
    }
    FileNode fileNode = fileNodeBuilder.setCapacity(capacity).build();
    try {
        fileNodeStorer.save(fileNode);
        return new FileObject(fileNode);
    } catch (Exception e) {
        LOG.error("store file node[{}] error!", fileNode.getName(), e);
    }
    return null;
}