Examples with DataXJobWorker com.qlangtech.tis.datax.job.DataXJobWorker used on opensource projects

Example 1 with DataXJobWorker

use of com.qlangtech.tis.datax.job.DataXJobWorker in project plugins by qlangtech.

the class DistributedOverseerDataXJobSubmit method getCuratorDistributedQueue.

private DistributedQueue<CuratorDataXTaskMessage> getCuratorDistributedQueue() {
    synchronized (this) {
        if (curatorClient != null && !curatorClient.getZookeeperClient().isConnected()) {
            curatorClient.close();
            curatorClient = null;
            curatorDistributedQueue = null;
        }
        if (curatorDistributedQueue == null) {
            DataXJobWorker dataxJobWorker = DataXJobWorker.getJobWorker(DataXJobWorker.K8S_DATAX_INSTANCE_NAME);
            if (curatorClient == null) {
                this.curatorClient = DataXJobConsumer.getCuratorFramework(dataxJobWorker.getZookeeperAddress());
            }
            this.curatorDistributedQueue = DataXJobConsumer.createQueue(curatorClient, dataxJobWorker.getZkQueuePath(), null);
        }
        return this.curatorDistributedQueue;
    }
}

Example 2 with DataXJobWorker

use of com.qlangtech.tis.datax.job.DataXJobWorker in project plugins by qlangtech.

the class TestDistributedOverseerDataXJobSubmit method testPushMsgToDistributeQueue.

public void testPushMsgToDistributeQueue() {
    DataXJobWorker dataxJobWorker = DataXJobWorker.getJobWorker(DataXJobWorker.K8S_DATAX_INSTANCE_NAME);
    assertEquals("/datax/jobs", dataxJobWorker.getZkQueuePath());
    assertEquals("192.168.28.200:2181/tis/cloud", dataxJobWorker.getZookeeperAddress());
    DataxProcessor dataxProcessor = IAppSource.load(DATAX_NAME);
    assertNotNull(dataxProcessor);
    // IDataxProcessor dataxProcessor = EasyMock.createMock("dataxProcessor", IDataxProcessor.class);
    // EasyMock.expect(dataxProcessor.getDataxCfgDir()).andReturn();
    IJoinTaskContext taskContext = EasyMock.createMock("joinTaskContext", IJoinTaskContext.class);
    EasyMock.expect(taskContext.getIndexName()).andReturn(DATAX_NAME);
    EasyMock.expect(taskContext.getTaskId()).andReturn(DATAX_TASK_ID);
    AtomicReference<ITISRpcService> ref = new AtomicReference<>();
    ref.set(StatusRpcClient.AssembleSvcCompsite.MOCK_PRC);
    RpcServiceReference svcRef = new RpcServiceReference(ref);
    Optional<DataXJobSubmit> jobSubmit = DataXJobSubmit.getDataXJobSubmit(DataXJobSubmit.InstanceType.DISTRIBUTE);
    assertTrue(jobSubmit.isPresent());
    DataXJobSubmit submit = jobSubmit.get();
    DataXJobSubmit.IDataXJobContext jobContext = submit.createJobContext(taskContext);
    EasyMock.replay(taskContext);
    // IJoinTaskContext taskContext
    // , RpcServiceReference statusRpc, IDataxProcessor dataxProcessor, String dataXfileName
    IRemoteTaskTrigger dataXJob = submit.createDataXJob(jobContext, svcRef, dataxProcessor, DATAX_JOB_FILE_NAME);
    dataXJob.run();
    EasyMock.verify(taskContext);
}

Example 3 with DataXJobWorker

use of com.qlangtech.tis.datax.job.DataXJobWorker in project tis by qlangtech.

the class DataXJobSubmit method getDataXTriggerType.

public static DataXJobSubmit.InstanceType getDataXTriggerType() {
    DataXJobWorker jobWorker = DataXJobWorker.getJobWorker(DataXJobWorker.K8S_DATAX_INSTANCE_NAME);
    // .isDataXWorkerServiceOnDuty();
    boolean dataXWorkerServiceOnDuty = jobWorker != null && jobWorker.inService();
    return dataXWorkerServiceOnDuty ? DataXJobSubmit.InstanceType.DISTRIBUTE : DataXJobSubmit.InstanceType.LOCAL;
}

Example 4 with DataXJobWorker

use of com.qlangtech.tis.datax.job.DataXJobWorker in project tis by qlangtech.

the class DataxAction method doGetDataxWorkerHpa.

@Func(value = PermissionConstant.DATAX_MANAGE, sideEffect = false)
public void doGetDataxWorkerHpa(Context context) {
    DataXJobWorker jobWorker = DataXJobWorker.getJobWorker(this.getK8SJobWorkerTargetName());
    if (jobWorker.getHpa() != null) {
        RcHpaStatus hpaStatus = jobWorker.getHpaStatus();
        this.setBizResult(context, hpaStatus);
    }
}

Example 5 with DataXJobWorker

use of com.qlangtech.tis.datax.job.DataXJobWorker in project tis by qlangtech.

the class DataxAction method getJobWoker.

private void getJobWoker(Context context, TargetResName targetName) {
    Optional<DataXJobWorker> firstWorker = // dataxJobWorkerStore.getPlugins().stream().filter((p) -> isJobWorkerMatch(targetName, p.getDescriptor())).findFirst();
    Optional.ofNullable(DataXJobWorker.getJobWorker((targetName)));
    DataXJobWorkerStatus jobWorkerStatus = new DataXJobWorkerStatus();
    if (!firstWorker.isPresent()) {
        jobWorkerStatus.setK8sReplicationControllerCreated(false);
        this.setBizResult(context, jobWorkerStatus);
        return;
    }
    DataXJobWorker jobWorker = firstWorker.get();
    boolean disableRcdeployment = this.getBoolean("disableRcdeployment");
    jobWorkerStatus.setK8sReplicationControllerCreated(jobWorker.inService());
    if (jobWorkerStatus.isK8sReplicationControllerCreated() && !disableRcdeployment) {
        jobWorkerStatus.setRcDeployment(jobWorker.getRCDeployment());
    }
    this.setBizResult(context, jobWorkerStatus);
}