Examples with Service com.bonree.brfs.common.service.Service used on opensource projects

Example 11 with Service

use of com.bonree.brfs.common.service.Service in project BRFS by zhangnianli.

the class DefaultStorageNameStick method deleteData.

@Override
public boolean deleteData(DateTime startTime, DateTime endTime) {
    try {
        Service[] serviceList = regionNodeSelector.select(regionNodeSelector.serviceNum());
        if (serviceList.length == 0) {
            throw new BRFSException("none disknode!!!");
        }
        for (Service service : serviceList) {
            StringBuilder pathBuilder = new StringBuilder();
            pathBuilder.append(config.getDuplicateUrlRoot()).append("/").append(storageId).append("/").append(startTime.toString()).append("_").append(endTime.toString());
            URI uri = new URIBuilder().setScheme(config.getUrlSchema()).setHost(service.getHost()).setPort(service.getPort()).setPath(pathBuilder.toString()).build();
            HttpResponse response = null;
            try {
                response = client.executeDelete(uri, defaultHeaders);
            } catch (Exception e) {
                LOG.warn("delete data http request failed", e);
                continue;
            }
            if (response == null) {
                throw new Exception("can not get response for createStorageName!");
            }
            if (response.isReponseOK()) {
                return true;
            }
            String code = new String(response.getResponseBody(), StandardCharsets.UTF_8);
            ReturnCode returnCode = ReturnCode.checkCode(storageName, code);
            LOG.info("returnCode:" + returnCode);
        }
    } catch (Exception e) {
        LOG.error("delete data error", e);
    }
    return false;
}

Example 12 with Service

use of com.bonree.brfs.common.service.Service in project BRFS by zhangnianli.

the class CopyCheckJob method operation.

@Override
public void operation(JobExecutionContext context) throws Exception {
    LOG.info("createCheck Copy Job working");
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    MetaTaskManagerInterface release = mcf.getTm();
    StorageRegionManager snm = mcf.getSnm();
    ServiceManager sm = mcf.getSm();
    List<String> srs = TaskStateLifeContral.getSRs(snm);
    TaskStateLifeContral.watchSR(release, srs, TaskType.SYSTEM_COPY_CHECK.name());
    // 判断是否有恢复任务，有恢复任务则不进行创建
    if (WatchSomeThingJob.getState(WatchSomeThingJob.RECOVERY_STATUSE)) {
        LOG.warn("rebalance task is running !! skip check copy task");
        return;
    }
    String taskType = TaskType.SYSTEM_COPY_CHECK.name();
    List<Service> services = sm.getServiceListByGroup(Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_DATA_SERVICE_GROUP_NAME));
    if (services == null || services.isEmpty()) {
        LOG.warn("skip create {} task, because service is empty", taskType);
        return;
    }
    List<StorageRegion> snList = snm.getStorageRegionList();
    if (snList == null || snList.isEmpty()) {
        LOG.warn("storagename list is null");
        return;
    }
    // 1.获取sn创建任务的实际那
    TaskTypeModel tmodel = release.getTaskTypeInfo(taskType);
    if (tmodel == null) {
        tmodel = new TaskTypeModel();
        tmodel.setSwitchFlag(true);
        release.setTaskTypeModel(taskType, tmodel);
    }
    Map<String, Long> sourceTimes = tmodel.getSnTimes();
    LOG.debug("update init sn time :{}", sourceTimes);
    // 2.过滤不符合副本校验的sn信息
    List<StorageRegion> needSns = CopyCountCheck.filterSn(snList, services.size());
    // 3.针对第一次出现的sn补充时间
    sourceTimes = CopyCountCheck.repairTime(sourceTimes, needSns);
    Map<String, List<String>> losers = CopyCountCheck.collectLossFile(needSns, services, sourceTimes);
    LOG.info("update before sn time :{}", sourceTimes);
    Pair<TaskModel, Map<String, Long>> pair = CreateSystemTask.creatTaskWithFiles(sourceTimes, losers, needSns, TaskType.SYSTEM_COPY_CHECK, RECOVERY_NUM, 0);
    if (pair == null) {
        LOG.warn("create pair is empty !!!!");
        return;
    }
    TaskModel task = pair.getFirst();
    String taskName = null;
    if (task != null) {
        List<String> servers = CreateSystemTask.getServerIds(services);
        taskName = CreateSystemTask.updateTask(release, task, servers, TaskType.SYSTEM_COPY_CHECK);
    }
    if (!BrStringUtils.isEmpty(taskName)) {
        LOG.info("create {} {} task successfull !!!", taskType, taskName);
    }
    sourceTimes = pair.getSecond();
    // 更新sn临界信息
    tmodel = release.getTaskTypeInfo(taskType);
    if (tmodel == null) {
        tmodel = new TaskTypeModel();
        tmodel.setSwitchFlag(true);
        LOG.warn("taskType {} metadata loss create against !!", taskType);
    }
    tmodel.putAllSnTimes(sourceTimes);
    release.setTaskTypeModel(taskType, tmodel);
    LOG.debug("update sn time {}", sourceTimes);
    createTransferTasks(release);
}

Example 13 with Service

use of com.bonree.brfs.common.service.Service in project BRFS by zhangnianli.

the class ServerMain method main.

public static void main(String[] args) {
    ProcessFinalizer finalizer = new ProcessFinalizer();
    try {
        ResourceTaskConfig resourceConfig = ResourceTaskConfig.parse();
        // 初始化email发送配置
        EmailPool.getInstance();
        String zkAddresses = Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_ZOOKEEPER_ADDRESSES);
        CuratorClient leaderClient = CuratorClient.getClientInstance(zkAddresses, 1000, 1000);
        CuratorClient client = CuratorClient.getClientInstance(zkAddresses);
        CuratorCacheFactory.init(zkAddresses);
        ZookeeperPaths zookeeperPaths = ZookeeperPaths.create(Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_CLUSTER_NAME), zkAddresses);
        SimpleAuthentication authentication = SimpleAuthentication.getAuthInstance(zookeeperPaths.getBaseUserPath(), zookeeperPaths.getBaseLocksPath(), client.getInnerClient());
        UserModel model = authentication.getUser("root");
        if (model == null) {
            LOG.error("please init server!!!");
            System.exit(1);
        }
        ServerIDManager idManager = new ServerIDManager(client.getInnerClient(), zookeeperPaths);
        idManager.getFirstServerID();
        StorageRegionManager snManager = new DefaultStorageRegionManager(client.getInnerClient().usingNamespace(zookeeperPaths.getBaseClusterName().substring(1)), null);
        snManager.addStorageRegionStateListener(new StorageRegionStateListener() {

            @Override
            public void storageRegionAdded(StorageRegion node) {
                LOG.info("-----------StorageNameAdded--[{}]", node);
                idManager.getSecondServerID(node.getId());
            }

            @Override
            public void storageRegionUpdated(StorageRegion node) {
            }

            @Override
            public void storageRegionRemoved(StorageRegion node) {
                LOG.info("-----------StorageNameRemove--[{}]", node);
                idManager.deleteSecondServerID(node.getId());
            }
        });
        snManager.start();
        finalizer.add(snManager);
        ServiceManager sm = new DefaultServiceManager(client.getInnerClient().usingNamespace(zookeeperPaths.getBaseClusterName().substring(1)));
        sm.start();
        finalizer.add(sm);
        // 磁盘管理模块
        // EmptyMain diskMain = new EmptyMain(sm);
        // diskMain.start();
        // 
        // finalizer.add(diskMain);
        DataNodeBootStrap dataNodeBootStrap = new DataNodeBootStrap(sm);
        dataNodeBootStrap.start();
        finalizer.add(dataNodeBootStrap);
        // 副本平衡模块
        sm.addServiceStateListener(Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_DATA_SERVICE_GROUP_NAME), new ServerChangeTaskGenetor(leaderClient, client, sm, idManager, zookeeperPaths.getBaseRebalancePath(), 3000, snManager));
        @SuppressWarnings("resource") RebalanceManager rebalanceServer = new RebalanceManager(zookeeperPaths, idManager, snManager, sm);
        rebalanceServer.start();
        String host = Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_HOST);
        int port = Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_PORT);
        int readPort = Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_FILE_PORT);
        Service selfService = new Service();
        selfService.setHost(host);
        selfService.setPort(port);
        selfService.setExtraPort(readPort);
        selfService.setServiceGroup(Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_DATA_SERVICE_GROUP_NAME));
        String serviceId = idManager.getFirstServerID();
        selfService.setServiceId(serviceId);
        Service checkService = sm.getServiceById(Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_DATA_SERVICE_GROUP_NAME), serviceId);
        if (checkService == null) {
            sm.registerService(selfService);
            System.out.println(selfService);
        } else {
            LOG.error("serviceId : {} is exists, system will exit!!!", serviceId);
            System.exit(1);
        }
        finalizer.add(new Closeable() {

            @Override
            public void close() throws IOException {
                try {
                    sm.unregisterService(selfService);
                } catch (Exception e) {
                    LOG.error("unregister service[{}] error", selfService, e);
                }
            }
        });
        // 资源管理模块
        InitTaskManager.initManager(resourceConfig, zookeeperPaths, sm, snManager, idManager, client);
    } catch (Exception e) {
        LOG.error("launch server error!!!", e);
        System.exit(1);
    } finally {
        Runtime.getRuntime().addShutdownHook(finalizer);
    }
}

Example 14 with Service

use of com.bonree.brfs.common.service.Service in project BRFS by zhangnianli.

the class CopyCountCheck method collectionSnFiles.

/**
 * 概述：获取集群对应目录的文件
 * @param services
 * @param snList
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static Map<StorageRegion, List<String>> collectionSnFiles(List<Service> services, List<StorageRegion> snList, final Map<String, Long> snTimes) throws Exception {
    Map<StorageRegion, List<String>> snMap = new HashMap<>();
    DiskNodeClient client = null;
    int reCount;
    String snName = null;
    String path;
    List<String> strs;
    long time;
    String dirName;
    String sid;
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    ServerIDManager sim = mcf.getSim();
    CuratorClient zkClient = mcf.getClient();
    SecondIDParser parser;
    String basePath = mcf.getZkPath().getBaseRoutePath();
    int timeout = 10000;
    for (Service service : services) {
        try {
            client = TcpClientUtils.getClient(service.getHost(), service.getPort(), service.getExtraPort(), timeout);
            long granule;
            for (StorageRegion sn : snList) {
                parser = new SecondIDParser(zkClient, sn.getId(), basePath);
                parser.updateRoute();
                sid = sim.getOtherSecondID(service.getServiceId(), sn.getId());
                granule = Duration.parse(sn.getFilePartitionDuration()).toMillis();
                reCount = sn.getReplicateNum();
                snName = sn.getName();
                if (!snTimes.containsKey(snName)) {
                    LOG.debug("sntime don't contain {}", snName);
                    continue;
                }
                time = snTimes.get(snName);
                dirName = TimeUtils.timeInterval(time, granule);
                for (int i = 1; i <= reCount; i++) {
                    path = "/" + snName + "/" + i + "/" + dirName;
                    LOG.debug("path :{}", path);
                    strs = getFileList(parser, client, path, sid);
                    if (strs == null || strs.isEmpty()) {
                        LOG.debug("files is empty {}", path);
                        continue;
                    }
                    LOG.debug("Collection dirName :{},{} size :{}", dirName, path, strs.size());
                    if (!snMap.containsKey(sn)) {
                        snMap.put(sn, new ArrayList<>());
                    }
                    snMap.get(sn).addAll(strs);
                }
            }
        } catch (Exception e) {
            EmailPool emailPool = EmailPool.getInstance();
            MailWorker.Builder builder = MailWorker.newBuilder(emailPool.getProgramInfo());
            builder.setModel("collect file execute 模块服务发生问题");
            builder.setException(e);
            builder.setMessage(mcf.getGroupName() + "(" + mcf.getServerId() + ")服务 执行任务时发生问题");
            Map<String, String> map = new HashedMap();
            map.put("remote ", service.getHost());
            map.put("connectTimeout", String.valueOf(timeout));
            map.put("sn", StringUtils.isEmpty(snName) ? "" : snName);
            if (snTimes != null && !snTimes.isEmpty()) {
                map.put("sntime", JSON.toJSONString(snTimes));
            }
            builder.setVariable(map);
            emailPool.sendEmail(builder);
            throw e;
        } finally {
            if (client != null) {
                try {
                    client.close();
                } catch (IOException e) {
                    LOG.error("{}", e);
                }
            }
        }
    }
    return clearUnLawFiles(snMap);
}

Example 15 with Service

use of com.bonree.brfs.common.service.Service in project BRFS by zhangnianli.

the class DefaultServiceManager method buildFrom.

private static Service buildFrom(ServiceInstance<Service> instance) {
    Service service = new Service();
    service.setServiceId(instance.getId());
    service.setServiceGroup(instance.getName());
    service.setHost(instance.getAddress());
    service.setPort(instance.getPort());
    return service;
}