Examples with TisZkClient com.qlangtech.tis.TisZkClient used on opensource projects

Example 1 with TisZkClient

use of com.qlangtech.tis.TisZkClient in project tis by qlangtech.

the class MockZKUtils method createZkMock.

public static ITISCoordinator createZkMock() throws Exception {
    TisZkClient zkCoordinator = EasyMockUtil.mock("zkCoordinator", TisZkClient.class);
    EasyMock.expect(zkCoordinator.shallConnect2RemoteIncrStatusServer()).andReturn(false).anyTimes();
    EasyMock.expect(zkCoordinator.unwrap()).andReturn(zkCoordinator).anyTimes();
    createAssembleLogCollectPathMock(zkCoordinator);
    try (InputStream input = MockZKUtils.class.getResourceAsStream("overseer_elect_leader.json")) {
        Assert.assertNotNull(input);
        IExpectationSetters<byte[]> expect = EasyMock.expect(zkCoordinator.getData(ZkUtils.ZK_PATH_OVERSEER_ELECT_LEADER, null, new Stat(), true));
        expect.andReturn(IOUtils.toByteArray(input)).anyTimes();
    }
    return zkCoordinator;
}

Example 2 with TisZkClient

use of com.qlangtech.tis.TisZkClient in project plugins by qlangtech.

the class LocalTableDumpFactory method triggerTask.

public static IRemoteJobTrigger triggerTask(TaskContext context, ILocalTask task) {
    TisZkClient zk = context.getCoordinator().unwrap();
    Objects.requireNonNull(zk, "zk(TisZkClient) can not be null");
    AtomicReference<Throwable> errRef = new AtomicReference<>();
    CountDownLatch countDown = new CountDownLatch(1);
    final ExecutorService executor = Executors.newSingleThreadExecutor((r) -> {
        Thread t = new Thread(r);
        t.setUncaughtExceptionHandler((thread, e) -> {
            errRef.set(e);
            logger.error(e.getMessage(), e);
            countDown.countDown();
        });
        return t;
    });
    return new IRemoteJobTrigger() {

        @Override
        public void submitJob() {
            executor.execute(() -> {
                RpcServiceReference statusRpc = null;
                try {
                    statusRpc = StatusRpcClient.getService(zk);
                    task.process(statusRpc);
                    countDown.countDown();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                } finally {
                    try {
                        statusRpc.get().close();
                    } catch (Throwable e) {
                    }
                }
            });
        }

        @Override
        public RunningStatus getRunningStatus() {
            RunningStatus runningStatus = null;
            // 反馈执行状态
            if (countDown.getCount() > 0) {
                runningStatus = new RunningStatus(0, false, false);
            } else {
                executor.shutdown();
                runningStatus = new RunningStatus(1, true, errRef.get() == null);
            }
            return runningStatus;
        }
    };
}

Example 3 with TisZkClient

use of com.qlangtech.tis.TisZkClient in project plugins by qlangtech.

the class TestLocalDataXJobSubmit method testCreateDataXJob.

public void testCreateDataXJob() throws Exception {
    Optional<DataXJobSubmit> dataXJobSubmit = DataXJobSubmit.getDataXJobSubmit(DataXJobSubmit.InstanceType.LOCAL);
    Assert.assertTrue("dataXJobSubmit shall present", dataXJobSubmit.isPresent());
    LocalDataXJobSubmit jobSubmit = (LocalDataXJobSubmit) dataXJobSubmit.get();
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypoint.class.getName());
    jobSubmit.setWorkingDirectory(new File("."));
    jobSubmit.setClasspath("target/classes:target/test-classes");
    AtomicReference<ITISRpcService> ref = new AtomicReference<>();
    ref.set(StatusRpcClient.AssembleSvcCompsite.MOCK_PRC);
    RpcServiceReference statusRpc = new RpcServiceReference(ref);
    DataXJobSubmit.IDataXJobContext dataXJobContext = EasyMock.createMock("dataXJobContext", DataXJobSubmit.IDataXJobContext.class);
    IExecChainContext taskContext = EasyMock.createMock("taskContext", IExecChainContext.class);
    EasyMock.expect(dataXJobContext.getTaskContext()).andReturn(taskContext).anyTimes();
    IDataxProcessor dataxProcessor = EasyMock.createMock("dataxProcessor", IDataxProcessor.class);
    EasyMock.expect(taskContext.getIndexName()).andReturn(dataXName).anyTimes();
    EasyMock.expect(taskContext.getTaskId()).andReturn(TaskId).anyTimes();
    int preSuccessTaskId = 99;
    PhaseStatusCollection preSuccessTask = new PhaseStatusCollection(preSuccessTaskId, new ExecutePhaseRange(FullbuildPhase.FullDump, FullbuildPhase.FullDump));
    DumpPhaseStatus preDumpStatus = new DumpPhaseStatus(preSuccessTaskId);
    DumpPhaseStatus.TableDumpStatus tableDumpStatus = preDumpStatus.getTable(dataXfileName);
    tableDumpStatus.setAllRows(LocalDataXJobMainEntrypoint.testAllRows);
    preSuccessTask.setDumpPhase(preDumpStatus);
    EasyMock.expect(taskContext.loadPhaseStatusFromLatest(dataXName)).andReturn(preSuccessTask).times(3);
    TisZkClient zkClient = EasyMock.createMock("TisZkClient", TisZkClient.class);
    String zkSubPath = "nodes0000000020";
    EasyMock.expect(zkClient.getChildren(ZkUtils.ZK_ASSEMBLE_LOG_COLLECT_PATH, null, true)).andReturn(Collections.singletonList(zkSubPath)).times(3);
    EasyMock.expect(zkClient.getData(EasyMock.eq(ZkUtils.ZK_ASSEMBLE_LOG_COLLECT_PATH + "/" + zkSubPath), EasyMock.isNull(), EasyMock.anyObject(Stat.class), EasyMock.eq(true))).andReturn(statusCollectorHost.getBytes(TisUTF8.get())).times(3);
    EasyMock.expect(taskContext.getZkClient()).andReturn(zkClient).anyTimes();
    EasyMock.replay(taskContext, dataxProcessor, zkClient, dataXJobContext);
    IRemoteTaskTrigger dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    RunningStatus running = getRunningStatus(dataXJob);
    assertTrue("running.isSuccess", running.isSuccess());
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypointThrowException.class.getName());
    dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    running = getRunningStatus(dataXJob);
    assertFalse("shall faild", running.isSuccess());
    assertTrue("shall complete", running.isComplete());
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypointCancellable.class.getName());
    dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    running = getRunningStatus(dataXJob, false);
    Thread.sleep(2000);
    dataXJob.cancel();
    int i = 0;
    while (i++ < 3 && !(running = dataXJob.getRunningStatus()).isComplete()) {
        Thread.sleep(1000);
    }
    assertFalse("shall faild", running.isSuccess());
    assertTrue("shall complete", running.isComplete());
    EasyMock.verify(taskContext, dataxProcessor, zkClient);
}

Example 4 with TisZkClient

use of com.qlangtech.tis.TisZkClient in project tis by qlangtech.

the class TriggerJobManage method hasGrantCollectLock.

/**
 * @param indexName
 *            索引名称
 * @return
 * @throws SessionExpiredException
 */
private boolean hasGrantCollectLock(String indexName) throws SessionExpiredException {
    // 睡一个随机数
    try {
        Thread.sleep((long) (Math.random() * 1000));
    } catch (InterruptedException e1) {
        e1.printStackTrace();
    }
    try {
        String COLLECT_STATE_PATH = "/terminator-lock/jst_full_dump_trigger_lock/" + indexName;
        TisZkClient zokeeper = this.getZookeeper();
        // 判断是否要执行收集流程
        final Date now = new Date();
        if (!zokeeper.exists(COLLECT_STATE_PATH, false)) {
            // 当前节点为空，创建节点立即返回
            zokeeper.create(COLLECT_STATE_PATH, parseCurrnetTimeStamp(now), CreateMode.EPHEMERAL, true);
            log.info("create new lock path:" + COLLECT_STATE_PATH);
            return true;
        }
        final Stat stat = new Stat();
        final byte[] content = zokeeper.getData(COLLECT_STATE_PATH, null, stat, true);
        final long lastExecuteTimeStamp = parseLatestExecuteTimeStamp(content);
        if ((lastExecuteTimeStamp + (COLLECT_STATE_INTERVAL * 30 * 1000)) <= now.getTime()) {
            // 取得锁，将现在的时间写回锁
            zokeeper.setData(COLLECT_STATE_PATH, parseCurrnetTimeStamp(now), stat.getVersion(), true);
            log.info("update the lock path:" + COLLECT_STATE_PATH);
            return true;
        }
        return false;
    } catch (SessionExpiredException e) {
        // zookeeper客户端会话超时
        throw e;
    } catch (KeeperException e) {
        log.warn("zookeeper error", e);
        return false;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 5 with TisZkClient

use of com.qlangtech.tis.TisZkClient in project tis by qlangtech.

the class ClusterStateReader method createSerivce.

@Override
protected TISZkStateReader createSerivce(RunEnvironment runtime) {
    try {
        TisZkClient zkClinet = (TisZkClient) zooKeeperGetter.getInstance(runtime);
        final TISZkStateReader zkStateReader = new TISZkStateReader(zkClinet.getZK());
        zkClinet.addOnReconnect(() -> {
            try {
                zkStateReader.createClusterStateWatchersAndUpdate();
            } catch (KeeperException e) {
                LOG.error("A ZK error has occurred", e);
                throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "A ZK error has occurred", e);
            } catch (InterruptedException e) {
                // Restore the interrupted status
                Thread.currentThread().interrupt();
                LOG.error("Interrupted", e);
                throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "Interrupted", e);
            }
        });
        zkStateReader.createClusterStateWatchersAndUpdate();
        return zkStateReader;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}