Examples with FinishCheck com.vip.saturn.it.base.FinishCheck used on opensource projects

Example 16 with FinishCheck

use of com.vip.saturn.it.base.FinishCheck in project Saturn by vipshop.

the class ScriptJobIT method test_B_ForceStop.

/**
 * 作业启用状态，关闭Executor，将强停作业
 */
@Test
public void test_B_ForceStop() throws Exception {
    // bacause ScriptPidUtils.isPidRunning don't support mac
    if (!OS.isFamilyUnix() || OS.isFamilyMac()) {
        return;
    }
    final int shardCount = 3;
    // 避免多个IT同时跑该作业
    final String jobName = "test_B_ForceStop_" + new Random().nextInt(100);
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setCron("9 9 9 9 9 ? 2099");
    jobConfig.setJobType(JobType.SHELL_JOB.toString());
    jobConfig.setShardingTotalCount(shardCount);
    jobConfig.setShardingItemParameters("0=sh " + LONG_TIME_SH_PATH + ",1=sh " + LONG_TIME_SH_PATH + ",2=sh " + LONG_TIME_SH_PATH);
    addJob(jobConfig);
    Thread.sleep(1000);
    // 将会删除该作业的一些pid垃圾数据
    startOneNewExecutorList();
    Thread.sleep(1000);
    final String executorName = saturnExecutorList.get(0).getExecutorName();
    enableJob(jobName);
    Thread.sleep(1000);
    runAtOnce(jobName);
    Thread.sleep(2000);
    // 不优雅退出，直接关闭
    stopExecutor(0);
    try {
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                for (int j = 0; j < shardCount; j++) {
                    long pid = ScriptPidUtils.getFirstPidFromFile(executorName, jobName, "" + j);
                    if (pid > 0 && ScriptPidUtils.isPidRunning(pid)) {
                        return false;
                    }
                }
                return true;
            }
        }, 10);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
    disableJob(jobName);
    Thread.sleep(1000);
    removeJob(jobName);
}

Example 17 with FinishCheck

use of com.vip.saturn.it.base.FinishCheck in project Saturn by vipshop.

the class ScriptJobIT method test_C_ReuseItem.

/**
 * 作业禁用状态，关闭Executor，分片进程不强杀。 下次启动Executor，将其正在运行分片，重新持久化分片状态（running节点），并监听其状态（运行完，删除running节点，持久化completed节点）
 */
@Test
public void test_C_ReuseItem() throws Exception {
    // because ScriptPidUtils.isPidRunning don't supoort mac
    if (!OS.isFamilyUnix() || OS.isFamilyMac()) {
        return;
    }
    final int shardCount = 3;
    // 避免多个IT同时跑该作业
    final String jobName = "test_C_ReuseItem" + new Random().nextInt(100);
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setCron("9 9 9 9 9 ? 2099");
    jobConfig.setJobType(JobType.SHELL_JOB.toString());
    jobConfig.setShardingTotalCount(shardCount);
    jobConfig.setShardingItemParameters("0=sh " + LONG_TIME_SH_PATH + ",1=sh " + LONG_TIME_SH_PATH + ",2=sh " + LONG_TIME_SH_PATH);
    addJob(jobConfig);
    Thread.sleep(1000);
    // 将会删除该作业的一些pid垃圾数据
    startOneNewExecutorList();
    Thread.sleep(1000);
    final String executorName = saturnExecutorList.get(0).getExecutorName();
    enableJob(jobName);
    Thread.sleep(1000);
    runAtOnce(jobName);
    Thread.sleep(1000);
    disableJob(jobName);
    Thread.sleep(1000);
    // 不优雅退出，直接关闭
    stopExecutor(0);
    try {
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return !isOnline(executorName);
            }
        }, 10);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
    for (int j = 0; j < shardCount; j++) {
        long pid = ScriptPidUtils.getFirstPidFromFile(executorName, jobName, "" + j);
        if (pid < 0 || !ScriptPidUtils.isPidRunning(pid)) {
            fail("item " + j + ", pid " + pid + " should running");
        }
    }
    startOneNewExecutorList();
    Thread.sleep(2000);
    try {
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                for (int j = 0; j < shardCount; j++) {
                    if (!regCenter.isExisted(JobNodePath.getNodeFullPath(jobName, ExecutionNode.getRunningNode(j)))) {
                        return false;
                    }
                }
                return true;
            }
        }, 10);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
    forceStopJob(jobName);
    Thread.sleep(1000);
    try {
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                for (int j = 0; j < shardCount; j++) {
                    if (!regCenter.isExisted(JobNodePath.getNodeFullPath(jobName, ExecutionNode.getCompletedNode(j)))) {
                        return false;
                    }
                }
                return true;
            }
        }, 10);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
    removeJob(jobName);
}

Example 18 with FinishCheck

use of com.vip.saturn.it.base.FinishCheck in project Saturn by vipshop.

the class ShardingWithChangingLeaderIT method test_A_StopNamespaceShardingManagerLeader.

@Test
public void test_A_StopNamespaceShardingManagerLeader() throws Exception {
    startSaturnConsoleList(2);
    Thread.sleep(1000);
    final String jobName = "test_A_StopNamespaceShardingManagerLeader";
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setCron("9 9 9 9 9 ? 2099");
    jobConfig.setJobType(JobType.JAVA_JOB.toString());
    jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
    jobConfig.setShardingTotalCount(1);
    jobConfig.setShardingItemParameters("0=0");
    addJob(jobConfig);
    Thread.sleep(1000);
    Main executor = startOneNewExecutorList();
    Thread.sleep(1000);
    String hostValue = regCenter.get(SaturnExecutorsNode.LEADER_HOSTNODE_PATH);
    assertThat(hostValue).isNotNull();
    stopSaturnConsole(0);
    Thread.sleep(1000);
    String hostValue2 = regCenter.get(SaturnExecutorsNode.LEADER_HOSTNODE_PATH);
    assertThat(hostValue2).isNotNull().isNotEqualTo(hostValue);
    enableJob(jobName);
    Thread.sleep(1000);
    runAtOnce(jobName);
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            if (isNeedSharding(jobName)) {
                return false;
            }
            return true;
        }
    }, 10);
    List<Integer> items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor.getExecutorName()))));
    assertThat(items).contains(0);
    disableJob(jobName);
    Thread.sleep(1000);
    removeJob(jobName);
    Thread.sleep(1000);
    stopExecutorListGracefully();
    stopSaturnConsoleList();
}

Example 19 with FinishCheck

use of com.vip.saturn.it.base.FinishCheck in project Saturn by vipshop.

the class ShardingWithTrafficIT method test_B_NoTrafficNotTakeOverFailoverShard.

@Test
public void test_B_NoTrafficNotTakeOverFailoverShard() throws Exception {
    final String jobName = "test_B_NoTrafficNotTakeOverFailoverShard";
    LongtimeJavaJob.JobStatus status = new LongtimeJavaJob.JobStatus();
    status.runningCount = 0;
    status.sleepSeconds = 10;
    status.finished = false;
    status.timeout = false;
    LongtimeJavaJob.statusMap.put(jobName + "_0", status);
    final JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setCron("9 9 9 9 9 ? 2099");
    jobConfig.setJobType(JobType.JAVA_JOB.toString());
    jobConfig.setJobClass(LongtimeJavaJob.class.getCanonicalName());
    jobConfig.setShardingTotalCount(1);
    jobConfig.setShardingItemParameters("0=0");
    addJob(jobConfig);
    Thread.sleep(1000L);
    enableJob(jobName);
    Thread.sleep(1000L);
    startOneNewExecutorList();
    Main executor2 = startOneNewExecutorList();
    String executorName2 = executor2.getExecutorName();
    extractTraffic(executorName2);
    Thread.sleep(1000L);
    runAtOnceAndWaitShardingCompleted(jobName);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return regCenter.isExisted(JobNodePath.getNodeFullPath(jobName, ExecutionNode.getRunningNode(0)));
        }
    }, 4);
    stopExecutor(0);
    Thread.sleep(100L);
    List<String> items = regCenter.getChildrenKeys(JobNodePath.getNodeFullPath(jobName, "leader/failover/items"));
    assertThat(items).isNotNull().containsOnly("0");
    assertThat(!isFailoverAssigned(jobName, 0));
    LongtimeJavaJob.statusMap.clear();
    disableJob(jobName);
    Thread.sleep(1000L);
    removeJob(jobName);
    Thread.sleep(1000L);
    stopExecutorListGracefully();
}

Example 20 with FinishCheck

use of com.vip.saturn.it.base.FinishCheck in project Saturn by vipshop.

the class ShardingIT method test_I_ContainerWithLocalModeAndUseDispreferList.

/**
 * preferList配置了容器资源，并且是本地模式，useDispreferList为true。当该容器有executor在线，则得到分片，并且分片数为1；当该容器全部executor下线，则其他executor也得不到分片，因为useDispreferList对本地模式无效
 */
@Test
public void test_I_ContainerWithLocalModeAndUseDispreferList() throws Exception {
    // 启动一个非容器executor
    Main executor1 = startOneNewExecutorList();
    boolean cleanOld = SystemEnvProperties.VIP_SATURN_EXECUTOR_CLEAN;
    String taskOld = SystemEnvProperties.VIP_SATURN_CONTAINER_DEPLOYMENT_ID;
    try {
        String taskId = "test1";
        SystemEnvProperties.VIP_SATURN_EXECUTOR_CLEAN = true;
        SystemEnvProperties.VIP_SATURN_CONTAINER_DEPLOYMENT_ID = taskId;
        // 启动一个容器executor
        Main executor2 = startOneNewExecutorList();
        int shardCount = 2;
        final String jobName = "test_I_ContainerWithLocalModeAndUseDispreferList";
        for (int i = 0; i < shardCount; i++) {
            String key = jobName + "_" + i;
            SimpleJavaJob.statusMap.put(key, 0);
        }
        JobConfig jobConfig = new JobConfig();
        jobConfig.setJobName(jobName);
        jobConfig.setCron("9 9 9 9 9 ? 2099");
        jobConfig.setJobType(JobType.JAVA_JOB.toString());
        jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
        jobConfig.setShardingTotalCount(shardCount);
        jobConfig.setShardingItemParameters("*=a");
        // 设置localMode为true
        jobConfig.setLocalMode(true);
        // 设置preferList为@taskId
        jobConfig.setPreferList("@" + taskId);
        // 设置useDispreferList为true
        jobConfig.setUseDispreferList(true);
        addJob(jobConfig);
        Thread.sleep(1000);
        enableJob(jobName);
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return isNeedSharding(jobName);
            }
        }, 10);
        runAtOnce(jobName);
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return !isNeedSharding(jobName);
            }
        }, 10);
        // executor2获取到0分片，executor1获取不到分片
        List<Integer> items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor2.getExecutorName()))));
        assertThat(items).hasSize(1).contains(0);
        items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor1.getExecutorName()))));
        assertThat(items).isEmpty();
        // wait running completed
        Thread.sleep(1000);
        // executor2下线
        stopExecutorGracefully(1);
        Thread.sleep(1000L);
        // 等待sharding分片完成
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return isNeedSharding(jobName);
            }
        }, 10);
        runAtOnce(jobName);
        // 等待拿走分片
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return !isNeedSharding(jobName);
            }
        }, 10);
        // executor1仍然拿不到分片
        items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor1.getExecutorName()))));
        assertThat(items).isEmpty();
        disableJob(jobName);
        Thread.sleep(1000);
        removeJob(jobName);
        stopExecutorListGracefully();
    } finally {
        SystemEnvProperties.VIP_SATURN_EXECUTOR_CLEAN = cleanOld;
        SystemEnvProperties.VIP_SATURN_CONTAINER_DEPLOYMENT_ID = taskOld;
    }
}