Examples with ITISRpcService com.tis.hadoop.rpc.ITISRpcService used on opensource projects

Example 1 with ITISRpcService

use of com.tis.hadoop.rpc.ITISRpcService in project plugins by qlangtech.

the class TestDistributedOverseerDataXJobSubmit method testPushMsgToDistributeQueue.

public void testPushMsgToDistributeQueue() {
    DataXJobWorker dataxJobWorker = DataXJobWorker.getJobWorker(DataXJobWorker.K8S_DATAX_INSTANCE_NAME);
    assertEquals("/datax/jobs", dataxJobWorker.getZkQueuePath());
    assertEquals("192.168.28.200:2181/tis/cloud", dataxJobWorker.getZookeeperAddress());
    DataxProcessor dataxProcessor = IAppSource.load(DATAX_NAME);
    assertNotNull(dataxProcessor);
    // IDataxProcessor dataxProcessor = EasyMock.createMock("dataxProcessor", IDataxProcessor.class);
    // EasyMock.expect(dataxProcessor.getDataxCfgDir()).andReturn();
    IJoinTaskContext taskContext = EasyMock.createMock("joinTaskContext", IJoinTaskContext.class);
    EasyMock.expect(taskContext.getIndexName()).andReturn(DATAX_NAME);
    EasyMock.expect(taskContext.getTaskId()).andReturn(DATAX_TASK_ID);
    AtomicReference<ITISRpcService> ref = new AtomicReference<>();
    ref.set(StatusRpcClient.AssembleSvcCompsite.MOCK_PRC);
    RpcServiceReference svcRef = new RpcServiceReference(ref);
    Optional<DataXJobSubmit> jobSubmit = DataXJobSubmit.getDataXJobSubmit(DataXJobSubmit.InstanceType.DISTRIBUTE);
    assertTrue(jobSubmit.isPresent());
    DataXJobSubmit submit = jobSubmit.get();
    DataXJobSubmit.IDataXJobContext jobContext = submit.createJobContext(taskContext);
    EasyMock.replay(taskContext);
    // IJoinTaskContext taskContext
    // , RpcServiceReference statusRpc, IDataxProcessor dataxProcessor, String dataXfileName
    IRemoteTaskTrigger dataXJob = submit.createDataXJob(jobContext, svcRef, dataxProcessor, DATAX_JOB_FILE_NAME);
    dataXJob.run();
    EasyMock.verify(taskContext);
}

Example 2 with ITISRpcService

use of com.tis.hadoop.rpc.ITISRpcService in project tis by qlangtech.

the class DataXExecuteInterceptor method getDataXExecReporter.

protected RpcServiceReference getDataXExecReporter() {
    IncrStatusUmbilicalProtocolImpl statusServer = IncrStatusUmbilicalProtocolImpl.getInstance();
    IncrStatusUmbilicalProtocol statReceiveSvc = new AdapterStatusUmbilicalProtocol() {

        @Override
        public void reportDumpTableStatus(DumpPhaseStatus.TableDumpStatus tableDumpStatus) {
            statusServer.reportDumpTableStatus(tableDumpStatus.getTaskid(), tableDumpStatus.isComplete(), tableDumpStatus.isWaiting(), tableDumpStatus.isFaild(), tableDumpStatus.getName());
        }
    };
    AtomicReference<ITISRpcService> ref = new AtomicReference<>();
    ref.set(new DataXAssembleSvcCompsite(statReceiveSvc));
    return new RpcServiceReference(ref);
}

Example 3 with ITISRpcService

use of com.tis.hadoop.rpc.ITISRpcService in project tis by qlangtech.

the class BasicDataXExecutorTestCase method setUp.

@Override
protected void setUp() throws Exception {
    super.setUp();
    System.clearProperty(Config.DEFAULT_DATA_DIR);
    setDataDir();
    PluginStubUtils.setTISField();
    AtomicReference<ITISRpcService> ref = new AtomicReference<>();
    ref.set(StatusRpcClient.AssembleSvcCompsite.MOCK_PRC);
    statusRpc = new RpcServiceReference(ref);
    executor = createExecutor();
}

Example 4 with ITISRpcService

use of com.tis.hadoop.rpc.ITISRpcService in project plugins by qlangtech.

the class TestLocalDataXJobSubmit method testCreateDataXJob.

public void testCreateDataXJob() throws Exception {
    Optional<DataXJobSubmit> dataXJobSubmit = DataXJobSubmit.getDataXJobSubmit(DataXJobSubmit.InstanceType.LOCAL);
    Assert.assertTrue("dataXJobSubmit shall present", dataXJobSubmit.isPresent());
    LocalDataXJobSubmit jobSubmit = (LocalDataXJobSubmit) dataXJobSubmit.get();
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypoint.class.getName());
    jobSubmit.setWorkingDirectory(new File("."));
    jobSubmit.setClasspath("target/classes:target/test-classes");
    AtomicReference<ITISRpcService> ref = new AtomicReference<>();
    ref.set(StatusRpcClient.AssembleSvcCompsite.MOCK_PRC);
    RpcServiceReference statusRpc = new RpcServiceReference(ref);
    DataXJobSubmit.IDataXJobContext dataXJobContext = EasyMock.createMock("dataXJobContext", DataXJobSubmit.IDataXJobContext.class);
    IExecChainContext taskContext = EasyMock.createMock("taskContext", IExecChainContext.class);
    EasyMock.expect(dataXJobContext.getTaskContext()).andReturn(taskContext).anyTimes();
    IDataxProcessor dataxProcessor = EasyMock.createMock("dataxProcessor", IDataxProcessor.class);
    EasyMock.expect(taskContext.getIndexName()).andReturn(dataXName).anyTimes();
    EasyMock.expect(taskContext.getTaskId()).andReturn(TaskId).anyTimes();
    int preSuccessTaskId = 99;
    PhaseStatusCollection preSuccessTask = new PhaseStatusCollection(preSuccessTaskId, new ExecutePhaseRange(FullbuildPhase.FullDump, FullbuildPhase.FullDump));
    DumpPhaseStatus preDumpStatus = new DumpPhaseStatus(preSuccessTaskId);
    DumpPhaseStatus.TableDumpStatus tableDumpStatus = preDumpStatus.getTable(dataXfileName);
    tableDumpStatus.setAllRows(LocalDataXJobMainEntrypoint.testAllRows);
    preSuccessTask.setDumpPhase(preDumpStatus);
    EasyMock.expect(taskContext.loadPhaseStatusFromLatest(dataXName)).andReturn(preSuccessTask).times(3);
    TisZkClient zkClient = EasyMock.createMock("TisZkClient", TisZkClient.class);
    String zkSubPath = "nodes0000000020";
    EasyMock.expect(zkClient.getChildren(ZkUtils.ZK_ASSEMBLE_LOG_COLLECT_PATH, null, true)).andReturn(Collections.singletonList(zkSubPath)).times(3);
    EasyMock.expect(zkClient.getData(EasyMock.eq(ZkUtils.ZK_ASSEMBLE_LOG_COLLECT_PATH + "/" + zkSubPath), EasyMock.isNull(), EasyMock.anyObject(Stat.class), EasyMock.eq(true))).andReturn(statusCollectorHost.getBytes(TisUTF8.get())).times(3);
    EasyMock.expect(taskContext.getZkClient()).andReturn(zkClient).anyTimes();
    EasyMock.replay(taskContext, dataxProcessor, zkClient, dataXJobContext);
    IRemoteTaskTrigger dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    RunningStatus running = getRunningStatus(dataXJob);
    assertTrue("running.isSuccess", running.isSuccess());
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypointThrowException.class.getName());
    dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    running = getRunningStatus(dataXJob);
    assertFalse("shall faild", running.isSuccess());
    assertTrue("shall complete", running.isComplete());
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypointCancellable.class.getName());
    dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    running = getRunningStatus(dataXJob, false);
    Thread.sleep(2000);
    dataXJob.cancel();
    int i = 0;
    while (i++ < 3 && !(running = dataXJob.getRunningStatus()).isComplete()) {
        Thread.sleep(1000);
    }
    assertFalse("shall faild", running.isSuccess());
    assertTrue("shall complete", running.isComplete());
    EasyMock.verify(taskContext, dataxProcessor, zkClient);
}