Examples with ReplicasSpec com.qlangtech.tis.config.k8s.ReplicasSpec used on opensource projects

Example 1 with ReplicasSpec

use of com.qlangtech.tis.config.k8s.ReplicasSpec in project plugins by qlangtech.

the class K8SDataXJobWorker method launchService.

@Override
public void launchService() {
    if (inService()) {
        throw new IllegalStateException("k8s instance of:" + KEY_FIELD_NAME + " is running can not relaunch");
    }
    try {
        // 启动服务
        // ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3);
        // CuratorFrameworkFactory.Builder curatorBuilder = CuratorFrameworkFactory.builder();
        // curatorBuilder.retryPolicy(retryPolicy);
        // this.client = curatorBuilder.connectString(this.zkAddress).build();
        K8sImage k8sImage = this.getK8SImage();
        // this.k8sClient = k8SImage.createApiClient();
        ReplicasSpec replicasSpec = this.getReplicasSpec();
        Objects.requireNonNull(replicasSpec, "replicasSpec can not be null");
        EnvVarsBuilder varsBuilder = new EnvVarsBuilder("tis-datax-executor") {

            @Override
            public String getAppOptions() {
                // return "-D" + DataxUtils.DATAX_QUEUE_ZK_PATH + "=" + getZkQueuePath() + " -D" + DataxUtils.DATAX_ZK_ADDRESS + "=" + getZookeeperAddress();
                return getZookeeperAddress() + " " + getZkQueuePath();
            }

            @Override
            public String getExtraSysProps() {
                return "-D" + CuratorDataXTaskMessage.SYSTEM_KEY_LOGBACK_PATH_KEY + "=" + CuratorDataXTaskMessage.SYSTEM_KEY_LOGBACK_PATH_VALUE;
            }

            @Override
            protected String processHost(String address) {
                return processDefaultHost(address);
            }
        };
        // K8sImage config, CoreV1Api api, String name, ReplicasSpec incrSpec, List< V1EnvVar > envs
        // CoreV1Api k8sV1Api = new CoreV1Api(k8sClient);
        // K8sImage k8sImage = this.getK8SImage();
        this.getK8SController().createReplicationController(K8S_DATAX_INSTANCE_NAME, replicasSpec, varsBuilder.build());
        if (supportHPA()) {
            HorizontalpodAutoscaler hap = this.getHpa();
            createHorizontalpodAutoscaler(k8sImage, hap);
        }
        writeLaunchToken();
    } catch (ApiException e) {
        logger.error(e.getResponseBody(), e);
        throw K8sExceptionUtils.convert(e);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 2 with ReplicasSpec

use of com.qlangtech.tis.config.k8s.ReplicasSpec in project tis by qlangtech.

the class TISK8sDelegate method main.

public static void main(String[] args) throws Exception {
    ReplicasSpec incrSpec = new ReplicasSpec();
    incrSpec.setReplicaCount(1);
    incrSpec.setCpuLimit(Specification.parse("2"));
    incrSpec.setCpuRequest(Specification.parse("500m"));
    incrSpec.setMemoryLimit(Specification.parse("2G"));
    incrSpec.setMemoryRequest(Specification.parse("500M"));
    TISK8sDelegate incrK8s = new TISK8sDelegate("search4totalpay");
    incrK8s.isRCDeployment(true);
}

Example 3 with ReplicasSpec

use of com.qlangtech.tis.config.k8s.ReplicasSpec in project tis by qlangtech.

the class IncrUtils method main.

public static void main(String[] args) throws Exception {
    Specification s = Specification.parse("300");
    System.out.println(s.getUnit() + "," + s.getVal());
    // System.out.println(isNumber("123000"));
    // 
    // System.out.println(isNumber("0123000"));
    // BuildStatus status = readLastBuildRecordStatus("search4totalpay");
    // System.out.println(status.getBuildName());
    // rollbackIncrDeploy("search4totalpay");
    ReplicasSpec spec = new ReplicasSpec();
    // spec.setGitAddress("git@git.2dfire-inc.com.qlangtech.searcher/tis-mars.git");
    // spec.setGitRef("develop");
    // 
    // Specification s = new Specification();
    // s.setVal(100);
    // s.setUnit("c");
    // spec.setCpuLimit(s);
    // 
    // s = new Specification();
    // s.setVal(200);
    // s.setUnit("cores");
    // spec.setCpuRequest(s);
    // 
    // s = new Specification();
    // s.setVal(2);
    // s.setUnit("G");
    // spec.setMemoryLimit(s);
    // 
    // s = new Specification();
    // s.setVal(200);
    // s.setUnit("M");
    // spec.setMemoryRequest(s);
    // 
    // saveIncrSpec("search4totalpay", spec);
    spec = readIncrSpec("search4totalpay");
// System.out.println(spec.getGitAddress());
}

Example 4 with ReplicasSpec

use of com.qlangtech.tis.config.k8s.ReplicasSpec in project tis by qlangtech.

the class IncrUtils method parseIncrSpec.

/**
 * @param context
 * @param form
 * @param msg
 * @return
 */
public static IncrSpecResult parseIncrSpec(Context context, final JSONObject form, IControlMsgHandler msg) {
    ReplicasSpec spec = new ReplicasSpec();
    IncrSpecResult result = new IncrSpecResult(spec, context, msg);
    result.success = false;
    String fieldPods = "pods";
    int replicCount = form.getIntValue(fieldPods);
    if (replicCount < 1) {
        // msg.addErrorMessage(context, "请设置Pods，不能小于1");
        msg.addFieldError(context, fieldPods, "请设置Pods，不能小于1");
    // return result;
    } else if (replicCount > 20) {
        msg.addFieldError(context, fieldPods, "不能大于20");
    // return result;
    }
    spec.setReplicaCount(replicCount);
    Specification s = null;
    String fieldCuprequest = "cuprequest";
    String cpurequest = StringUtils.trimToEmpty(form.getString(fieldCuprequest));
    String cpuRequestUnit = form.getString("cuprequestunit");
    if (StringUtils.isEmpty(cpuRequestUnit)) {
        msg.addFieldError(context, fieldCuprequest, "请填写CPU请求单位");
    }
    cpuRequestUnit = "cores".equals(cpuRequestUnit) ? StringUtils.EMPTY : cpuRequestUnit;
    if (StringUtils.isEmpty(cpurequest)) {
        msg.addFieldError(context, fieldCuprequest, "请填写");
    } else if (!IncrUtils.isNumber(cpurequest)) {
        // msg.addErrorMessage(context, "cpurequest must be " + IncrUtils.PATTERN_NUMBER);
        msg.addFieldError(context, fieldCuprequest, "必须是非负整数");
    } else {
        s = new Specification();
        s.setVal(Integer.parseInt(cpurequest));
        s.setUnit(cpuRequestUnit);
        int maxCpuCoresRequest = 4;
        if (s.normalizeCPU() > (maxCpuCoresRequest * 1024)) {
            msg.addFieldError(context, fieldCuprequest, "请检查CPU请求资源，不能大于" + maxCpuCoresRequest + "cores");
        // return result;
        }
        spec.setCpuRequest(s);
    }
    String filedCpuLimit = "cuplimit";
    String cupLimit = StringUtils.trimToEmpty(form.getString(filedCpuLimit));
    String cupLimitUnit = form.getString("cuplimitunit");
    if (StringUtils.isEmpty(cupLimitUnit)) {
        msg.addFieldError(context, filedCpuLimit, "请填写CPU最大请求单位");
    }
    cupLimitUnit = "cores".equals(cupLimitUnit) ? StringUtils.EMPTY : cupLimitUnit;
    if (StringUtils.isEmpty(cupLimit)) {
        msg.addFieldError(context, filedCpuLimit, "请填写");
    } else if (!IncrUtils.isNumber(cupLimit)) {
        // msg.addErrorMessage(context, "CPU limit must be " + IncrUtils.PATTERN_NUMBER);
        msg.addFieldError(context, filedCpuLimit, "必须是非负整数");
    // return result;
    } else {
        s = new Specification();
        s.setVal(Integer.parseInt(cupLimit));
        s.setUnit(cupLimitUnit);
        int maxCpuCoresLimit = 8;
        if (s.normalizeCPU() > maxCpuCoresLimit * 1024) {
            msg.addFieldError(context, filedCpuLimit, "请检查CPU最大申请资源，不能大于" + maxCpuCoresLimit + "cores");
        // return result;
        }
        spec.setCpuLimit(s);
    }
    if (spec.getCpuRequest() != null && spec.getCpuLimit() != null && spec.getCpuRequest().cpuBigThan(spec.getCpuLimit())) {
        msg.addFieldError(context, fieldCuprequest, "请检查CPU`申请资源`，不能大于`最大申请资源`");
        msg.addFieldError(context, filedCpuLimit, "请检查CPU`最大申请资源`，不能小于`申请资源`");
    }
    // cupLimit = cupLimit + cupLimitUnit;
    String fieldMemoryRequest = "memoryrequest";
    String memoryRequest = form.getString(fieldMemoryRequest);
    String memoryRequestUnit = form.getString("memoryrequestunit");
    if (StringUtils.isEmpty(memoryRequest)) {
        msg.addFieldError(context, fieldMemoryRequest, "请填写");
    } else if (!IncrUtils.isNumber(memoryRequest)) {
        // msg.addErrorMessage(context, "内存格式" + IncrUtils.PATTERN_NUMBER);
        msg.addFieldError(context, fieldMemoryRequest, "必须是非负整数");
    // return result;
    } else {
        s = new Specification();
        s.setVal(Integer.parseInt(memoryRequest));
        s.setUnit(memoryRequestUnit);
        int maxMemoryRequest = 4;
        if (s.normalizeMemory() > (maxMemoryRequest * 1024)) {
            msg.addFieldError(context, fieldMemoryRequest, "请检查内存申请资源，不能大于" + maxMemoryRequest + "G");
        // return result;
        }
        spec.setMemoryRequest(s);
    }
    if (StringUtils.isEmpty(memoryRequestUnit)) {
        // msg.addErrorMessage(context, "请填写内存请求单位");
        msg.addFieldError(context, fieldMemoryRequest, "请填写内存请求单位");
    // return result;
    }
    String fieldMemorylimit = "memorylimit";
    String memoryLimit = (form.getString(fieldMemorylimit));
    String memoryLimitUnit = form.getString("memorylimitunit");
    if (StringUtils.isEmpty(memoryLimit)) {
        msg.addFieldError(context, fieldMemorylimit, "请填写");
    } else if (!IncrUtils.isNumber(memoryLimit)) {
        // msg.addErrorMessage(context, "内存上限" + IncrUtils.PATTERN_NUMBER);
        msg.addFieldError(context, fieldMemorylimit, "必须为非负整数");
    // return result;
    } else {
        int maxMemorylimit = 8;
        s = new Specification();
        s.setVal(Integer.parseInt(memoryLimit));
        s.setUnit(memoryLimitUnit);
        if (s.normalizeMemory() > (maxMemorylimit * 1024)) {
            msg.addFieldError(context, fieldMemorylimit, "请检查内存最大申请资源，不能大于" + maxMemorylimit + "G");
        // return result;
        }
        spec.setMemoryLimit(s);
    }
    if (StringUtils.isEmpty(memoryLimitUnit)) {
        msg.addFieldError(context, fieldMemorylimit, "请填写内存上限单位");
    // return result;
    }
    if (spec.getMemoryRequest() != null && spec.getMemoryLimit() != null && spec.getMemoryRequest().memoryBigThan(spec.getMemoryLimit())) {
        msg.addFieldError(context, fieldMemoryRequest, "请检查内存`申请资源`，不能大于`最大申请资源`");
        msg.addFieldError(context, fieldMemorylimit, "请检查内存`最大申请资源`，不能小于`申请资源`");
    // return result;
    }
    final String KEY_MIN_HPA_POD = "minHpaPod";
    final String KEY_MAX_HPA_POD = "maxHpaPod";
    final String KEY_CPU_AVERAGE_UTILIZATION = "cpuAverageUtilization";
    boolean supportHpa = form.getBooleanValue("supportHpa");
    if (supportHpa) {
        // 支持弹性扩容
        Integer maxHpaPod = form.getInteger(KEY_MAX_HPA_POD);
        Integer minHpaPod = form.getInteger(KEY_MIN_HPA_POD);
        if (minHpaPod == null) {
            msg.addFieldError(context, KEY_MIN_HPA_POD, "请填写'最小Pods'");
        }
        if (maxHpaPod == null) {
            msg.addFieldError(context, KEY_MAX_HPA_POD, "请填写'最大Pods'");
        }
        Integer cpuAverageUtilization = form.getInteger(KEY_CPU_AVERAGE_UTILIZATION);
        if (cpuAverageUtilization == null) {
            msg.addFieldError(context, KEY_CPU_AVERAGE_UTILIZATION, "请填写'CPU平均利用率'");
        }
        if (context.hasErrors()) {
            result.success = false;
            return result;
        }
        boolean hasErr = false;
        if (cpuAverageUtilization < 1 || cpuAverageUtilization >= 100) {
            msg.addFieldError(context, KEY_CPU_AVERAGE_UTILIZATION, "必须大于0且小于100");
            hasErr = true;
        }
        if (minHpaPod < 1) {
            msg.addFieldError(context, KEY_MIN_HPA_POD, "最小弹性Pod数必须大于0");
            hasErr = true;
        }
        if (maxHpaPod <= minHpaPod) {
            msg.addFieldError(context, KEY_MAX_HPA_POD, "必须大于最小弹性Pod数");
            msg.addFieldError(context, KEY_MIN_HPA_POD, "必须小于最大弹性Pod数");
            hasErr = true;
        }
        if (hasErr) {
            return result;
        }
        HorizontalpodAutoscaler hpa = new HorizontalpodAutoscaler();
        hpa.setMaxPod(maxHpaPod);
        hpa.setMinPod(minHpaPod);
        hpa.setCpuAverageUtilization(cpuAverageUtilization);
        if (hpa.getMinPod() > spec.getReplicaCount()) {
            // msg.addFieldError(context, KEY_MAX_HPA_POD, "必须大于最小弹性Pod数");
            // msg.addFieldError(context, KEY_MIN_HPA_POD, "必须小于最大弹性Pod数");
            msg.addFieldError(context, KEY_MIN_HPA_POD, "不能大于'Pods'数");
            msg.addFieldError(context, fieldPods, "不能小于'弹性扩容最小Pods数'");
            // msgHandler.(context, resourceLiteria + " Request Is inValid " + request);
            return result;
        }
        result.hpa = hpa;
    }
    result.success = !context.hasErrors();
    return result;
}

Example 5 with ReplicasSpec

use of com.qlangtech.tis.config.k8s.ReplicasSpec in project tis by qlangtech.

the class CollectionAction method createIncrSyncChannel.

/**
 * 创建增量同步通道
 *
 * @param incrCfg
 */
private // , WorkFlow df
boolean createIncrSyncChannel(// , WorkFlow df
Context context, JSONObject incrCfg) throws Exception {
    // 生成DAO脚本
    HeteroEnum pluginType = HeteroEnum.MQ;
    UploadPluginMeta pluginMeta = UploadPluginMeta.parse(pluginType.identity + ":" + UploadPluginMeta.KEY_REQUIRE);
    PluginItems incrPluginItems = getPluginItems(incrCfg, pluginType, pluginMeta);
    if (incrPluginItems.items.size() < 1) {
        throw new IllegalStateException("incr plugin item size can not small than 1");
    }
    for (AttrValMap vals : incrPluginItems.items) {
        if (!vals.validate(context, false).isValid()) {
            // return columnMeta.invalid();
            return false;
        }
        // MQListenerFactory mqListenerFactory = (MQListenerFactory) vals.createDescribable().instance;
        break;
    }
    incrPluginItems.save(context);
    /**
     *=======================================
     *开始生成脚本并且编译打包
     *=======================================
     */
    // SqlTaskNodeMeta.SqlDataFlowTopology wfTopology = SqlTaskNodeMeta.getSqlDataFlowTopology(df.getName());
    IndexIncrStatus incrStatus = CoreAction.generateDAOAndIncrScript(this, context, true, true);
    if (context.hasErrors()) {
        return false;
    }
    ReplicasSpec incrPodSpec = new ReplicasSpec();
    // FIXME 目前先写死
    incrPodSpec.setReplicaCount(1);
    incrPodSpec.setMemoryRequest(Specification.parse("1G"));
    incrPodSpec.setMemoryLimit(Specification.parse("2G"));
    incrPodSpec.setCpuRequest(Specification.parse("500m"));
    incrPodSpec.setCpuLimit(Specification.parse("1"));
    // IncrUtils.IncrSpecResult applySpec = IncrUtils.parseIncrSpec(context, this.parseJsonPost(), this);
    // if (!applySpec.isSuccess()) {
    // return;
    // }
    // 将打包好的构建，发布到k8s集群中去
    // https://github.com/kubernetes-client/java
    TISK8sDelegate k8sClient = TISK8sDelegate.getK8SDelegate(this.getCollectionName());
    // 通过k8s发布
    k8sClient.deploy(incrPodSpec, incrStatus.getIncrScriptTimestamp());
    return true;
}