Examples with DockerJob com.chinaunicom.rundocker.bean.DockerJob used on opensource projects

Example 6 with DockerJob

use of com.chinaunicom.rundocker.bean.DockerJob in project mesosFramework by zhizuqiu.

the class RestHandler method getExpose.

private Map getExpose() {
    Map<String, String> exposes = new HashMap<>();
    // 遍历所有的job
    for (String key : AppDataStore.jobsKeySet()) {
        DockerJob dockerJob = AppDataStore.jobsGet(key);
        Map<String, String> ex = dockerJob.getExposeRulesReal();
        if (ex != null) {
            // 把不相同的put进去
            for (String exKey : ex.keySet()) {
                String realValue = ex.get(exKey);
                if (!realValue.contains(OPERATOR_JOBS) && !realValue.contains(OPERATOR_STATUS)) {
                    exposes.put(exKey, realValue);
                }
            }
        }
    }
    return exposes;
}

Example 7 with DockerJob

use of com.chinaunicom.rundocker.bean.DockerJob in project mesosFramework by zhizuqiu.

the class DockerScheduler method resourceOffers.

@Override
public void resourceOffers(SchedulerDriver schedulerDriver, List<Offer> offers) {
    // logger.info("[" + logTime + "]"+"----------------------------------");
    // logger.info("[" + logTime + "]"+"offers.size(): " + offers.size());
    // 时间戳
    Long logTime = System.currentTimeMillis();
    // 真正需要下发的任务列表
    List<DockerJob> pendingJobs = new ArrayList<>();
    for (String key : AppDataStore.jobsKeySet()) {
        DockerJob job = AppDataStore.jobsGet(key);
        DockerStatus jobStatus = AppDataStore.statusGet(key);
        if (jobStatus != null) {
            // 下发条件：1.没有提交，2.依赖都运行，3.满足重试时间间隔
            if (!jobStatus.getSubmitted() && SchedulerService.hasRunDependencies(job) && SchedulerService.hasSatisfyRetryInterval(job, jobStatus)) {
                pendingJobs.add(job);
            }
        }
    }
    // 遍历offers
    for (Offer offer : offers) {
        logger.info("[" + logTime + "]" + "----------------------");
        logger.info("[" + logTime + "]" + "offer hostname: " + offer.getHostname());
        logger.info("[" + logTime + "]" + "----------------------");
        logger.info("[" + logTime + "]" + offer);
        // 如果待下发的任务列表不为空
        if (!pendingJobs.isEmpty()) {
            // 存储并处理offer资源的工具类
            OfferTemp offerTemp = new OfferTemp(offer);
            DockerJob jobOld = pendingJobs.get(0);
            DockerJob dockerJob = SchedulerService.getRealDockerJob(jobOld);
            DockerStatus jobStatus = AppDataStore.statusGet(dockerJob.getKey());
            // 匹配约束
            List<List<String>> constraints = dockerJob.getConstraints();
            Boolean matchConstraints = SchedulerService.matchConstraints(offer, constraints);
            if (!matchConstraints) {
                logger.info("[" + logTime + "]" + "------------");
                logger.info("[" + logTime + "]" + "not match constraints");
                schedulerDriver.declineOffer(offer.getId());
                continue;
            }
            // 匹配usePreHost
            Boolean matchPreHost = SchedulerService.matchUsePreHost(offer, dockerJob, jobStatus);
            if (!matchPreHost) {
                logger.info("[" + logTime + "]" + "------------");
                logger.info("[" + logTime + "]" + "not match preHost");
                schedulerDriver.declineOffer(offer.getId());
                continue;
            }
            // 匹配资源
            if (!offerTemp.acceptTask(dockerJob)) {
                logger.info("[" + logTime + "]" + "------------");
                logger.info("[" + logTime + "]" + "not accept task with resource");
                schedulerDriver.declineOffer(offer.getId());
                continue;
            }
            // 上面是下发前的检查工作
            // 要接受的offer id列表
            List<OfferID> offerIds = new ArrayList<>();
            // 要进行操作的列表
            List<Offer.Operation> operations = new ArrayList<>();
            logger.info("[" + logTime + "]" + "------------");
            logger.info("[" + logTime + "]" + "persistenceStatus is " + jobStatus.get_persistenceInfo().getPersistenceStatus());
            switch(jobStatus.get_persistenceInfo().getPersistenceStatus()) {
                case NONE:
                    break;
                case NEW:
                    // 预留资源
                    Resource resourceReserve = SchedulerService.buildReserveResource(dockerJob);
                    if (resourceReserve == null) {
                        logger.error("[" + logTime + "]" + "buildReserveResource() error");
                        schedulerDriver.declineOffer(offer.getId());
                        continue;
                    } else {
                        // 添加到operations中
                        operations.add(Offer.Operation.newBuilder().setType(Offer.Operation.Type.RESERVE).setReserve(Offer.Operation.Reserve.newBuilder().addResources(resourceReserve).build()).build());
                    }
                    // 持久卷
                    Resource resource = SchedulerService.buildCreateResource(dockerJob);
                    if (resource == null) {
                        logger.error("[" + logTime + "]" + "buildCreateResource() error");
                        schedulerDriver.declineOffer(offer.getId());
                        continue;
                    } else {
                        // 添加到operations中
                        operations.add(Offer.Operation.newBuilder().setType(Offer.Operation.Type.CREATE).setCreate(Offer.Operation.Create.newBuilder().addVolumes(resource).build()).build());
                    }
                    // 持久卷重复创建并不会导致任务启动失败，但是会导致多预留一块资源
                    jobStatus.get_persistenceInfo().setPersistenceStatus(DockerStatus.PersistenceStatus.TRY_TO_RESERVE_AND_CREATE);
                    jobStatus.get_persistenceInfo().setPersistenceAgentid(offer.getSlaveId().getValue());
                    // 更新etcd
                    etcdService.setStatusAtomic(logTime, jobStatus);
                    break;
                case TRY_TO_RESERVE_AND_CREATE:
                case END:
                    // todo 检查持久卷
                    logger.info("检查持久卷");
                    Boolean matchPersistentVolumes = SchedulerService.matchPersistentVolumes(offer, dockerJob);
                    if (!matchPersistentVolumes) {
                        logger.info("[" + logTime + "]" + "------------");
                        logger.info("[" + logTime + "]" + "not match persistent volumes");
                        schedulerDriver.declineOffer(offer.getId());
                        continue;
                    }
                    break;
                default:
            }
            // 准备要Launch的任务
            TaskInfo task;
            try {
                task = SchedulerService.buildTaskInfo(offer, dockerJob);
            } catch (Exception e) {
                logger.error("[" + logTime + "]" + "job id " + dockerJob.getId() + " buildTaskInfo Exception:" + e.getMessage());
                schedulerDriver.declineOffer(offer.getId());
                etcdService.recordException(e);
                continue;
            }
            logger.info("[" + logTime + "]" + "------------");
            logger.info("[" + logTime + "]" + "taskInfo: " + task.getName());
            // 添加到operations中
            operations.add(Offer.Operation.newBuilder().setType(Offer.Operation.Type.LAUNCH).setLaunch(Offer.Operation.Launch.newBuilder().addTaskInfos(TaskInfo.newBuilder(task))).build());
            // 修改任务状态
            jobStatus.setHostname(offer.getHostname());
            jobStatus.launch();
            jobStatus.setCurrentTime();
            jobStatus.setSlaveId(offer.getSlaveId().getValue());
            // 设置confMap
            jobStatus.set_confMap(jobOld.getEnvConfMap());
            // 更新etcd中的任务状态
            EtcdService.SetResult setResult = etcdService.setStatusAtomic(logTime, jobStatus);
            switch(setResult) {
                case HAS_EXCEPTION:
                    // todo 触发同步状态方法
                    break;
                default:
                    break;
            }
            logger.info("[" + logTime + "]" + "resourceOffers->setStatusAtomic:" + setResult);
            // 把将要下发的任务从待下发任务列表中删除
            pendingJobs.remove(0);
            // 接受offer
            offerIds.add(offer.getId());
            // declineOffer时的过滤器
            Filters filters = Filters.newBuilder().setRefuseSeconds(1).build();
            Status status = schedulerDriver.acceptOffers(offerIds, operations, filters);
            logger.info("[" + logTime + "]" + "acceptOffers:" + status);
        } else {
            // 如果待下发的任务列表为空，拒绝offer
            schedulerDriver.declineOffer(offer.getId());
        // logger.info("[" + logTime + "]"+"------------");
        // logger.info("[" + logTime + "]"+"declineOffer :" + offer.getId());
        }
    }
    if (pendingJobs.isEmpty()) {
        // 判断是否所有任务已下发，是不需要接受offer
        if (SchedulerService.checkAllAppsIsAccept()) {
            schedulerDriver.suppressOffers();
            logger.info("suppressOffers");
        }
    }
}

Example 8 with DockerJob

use of com.chinaunicom.rundocker.bean.DockerJob in project mesosFramework by zhizuqiu.

the class Tools method getObjFromJobsUseJsonPath.

/**
 * 使用jsonPath工具，获取job的json里面的值
 *
 * @param id       appid
 * @param operator jsonPath参数
 * @return 值
 */
public static Object getObjFromJobsUseJsonPath(String id, String operator) {
    if (id == null || id.trim().isEmpty()) {
        return null;
    }
    String key = Tools.getStringByJsonpathRecursion(id.trim().replaceAll("/", "."));
    DockerJob job = AppDataStore.jobsGet(key);
    if (job == null) {
        return null;
    }
    if (operator == null || operator.trim().isEmpty()) {
        return job;
    }
    Object obj = null;
    try {
        String json = new Gson().toJson(job);
        obj = JsonPath.read(json, operator);
    } catch (Exception e) {
        logger.error(e.getMessage());
    }
    return obj;
}

Example 9 with DockerJob

use of com.chinaunicom.rundocker.bean.DockerJob in project mesosFramework by zhizuqiu.

the class AppDataStore method getJobAndStatusTemp.

// ------------
public static JobAndStatusTemp getJobAndStatusTemp(String key) {
    DockerJob dockerJob = jobsGet(key);
    if (dockerJob == null) {
        return null;
    }
    DockerStatus dockerStatus = statusGet(key);
    if (dockerStatus == null) {
        return null;
    }
    return new JobAndStatusTemp(key, dockerJob, dockerStatus);
}

Example 10 with DockerJob

use of com.chinaunicom.rundocker.bean.DockerJob in project mesosFramework by zhizuqiu.

the class EtcdServiceTest method putJobsAndStatusTest.

@Test
public void putJobsAndStatusTest() {
    String id = "/2048/1";
    DockerJob job = new DockerJob();
    job.setId(id);
    DockerStatus dockerStatus = new DockerStatus();
    dockerStatus.setId(id);
    String id2 = "/2048/2";
    DockerJob job2 = new DockerJob();
    job2.setId(id2);
    DockerStatus dockerStatus2 = new DockerStatus();
    dockerStatus2.setId(id2);
    AppDataStore.JobAndStatusTemp jobAndStatusTemp = new AppDataStore.JobAndStatusTemp(id, job, dockerStatus);
    AppDataStore.JobAndStatusTemp jobAndStatusTemp2 = new AppDataStore.JobAndStatusTemp(id2, job2, dockerStatus2);
    // put两个值
    AppDataStore.JobAndStatusTemp jobAndStatusTempResult = AppDataStore.postJobsAndStatus(jobAndStatusTemp);
    AppDataStore.JobAndStatusTemp jobAndStatusTempResult2 = AppDataStore.postJobsAndStatus(jobAndStatusTemp2);
    System.out.println(AppDataStore.jobsGet());
    System.out.println(AppDataStore.statusGet());
    System.out.println("------------");
    // 回滚1值
    AppDataStore.postJobsAndStatus(jobAndStatusTempResult);
    System.out.println(AppDataStore.jobsGet());
    System.out.println(AppDataStore.statusGet());
    System.out.println("------------");
    // 回滚2值
    AppDataStore.postJobsAndStatus(jobAndStatusTempResult2);
    System.out.println(AppDataStore.jobsGet());
    System.out.println(AppDataStore.statusGet());
}