Examples with RpcServiceReference com.tis.hadoop.rpc.RpcServiceReference used on opensource projects

Example 1 with RpcServiceReference

use of com.tis.hadoop.rpc.RpcServiceReference in project plugins by qlangtech.

the class TestDistributedOverseerDataXJobSubmit method testPushMsgToDistributeQueue.

public void testPushMsgToDistributeQueue() {
    DataXJobWorker dataxJobWorker = DataXJobWorker.getJobWorker(DataXJobWorker.K8S_DATAX_INSTANCE_NAME);
    assertEquals("/datax/jobs", dataxJobWorker.getZkQueuePath());
    assertEquals("192.168.28.200:2181/tis/cloud", dataxJobWorker.getZookeeperAddress());
    DataxProcessor dataxProcessor = IAppSource.load(DATAX_NAME);
    assertNotNull(dataxProcessor);
    // IDataxProcessor dataxProcessor = EasyMock.createMock("dataxProcessor", IDataxProcessor.class);
    // EasyMock.expect(dataxProcessor.getDataxCfgDir()).andReturn();
    IJoinTaskContext taskContext = EasyMock.createMock("joinTaskContext", IJoinTaskContext.class);
    EasyMock.expect(taskContext.getIndexName()).andReturn(DATAX_NAME);
    EasyMock.expect(taskContext.getTaskId()).andReturn(DATAX_TASK_ID);
    AtomicReference<ITISRpcService> ref = new AtomicReference<>();
    ref.set(StatusRpcClient.AssembleSvcCompsite.MOCK_PRC);
    RpcServiceReference svcRef = new RpcServiceReference(ref);
    Optional<DataXJobSubmit> jobSubmit = DataXJobSubmit.getDataXJobSubmit(DataXJobSubmit.InstanceType.DISTRIBUTE);
    assertTrue(jobSubmit.isPresent());
    DataXJobSubmit submit = jobSubmit.get();
    DataXJobSubmit.IDataXJobContext jobContext = submit.createJobContext(taskContext);
    EasyMock.replay(taskContext);
    // IJoinTaskContext taskContext
    // , RpcServiceReference statusRpc, IDataxProcessor dataxProcessor, String dataXfileName
    IRemoteTaskTrigger dataXJob = submit.createDataXJob(jobContext, svcRef, dataxProcessor, DATAX_JOB_FILE_NAME);
    dataXJob.run();
    EasyMock.verify(taskContext);
}

Example 2 with RpcServiceReference

use of com.tis.hadoop.rpc.RpcServiceReference in project tis by qlangtech.

the class DataXExecuteInterceptor method getDataXExecReporter.

protected RpcServiceReference getDataXExecReporter() {
    IncrStatusUmbilicalProtocolImpl statusServer = IncrStatusUmbilicalProtocolImpl.getInstance();
    IncrStatusUmbilicalProtocol statReceiveSvc = new AdapterStatusUmbilicalProtocol() {

        @Override
        public void reportDumpTableStatus(DumpPhaseStatus.TableDumpStatus tableDumpStatus) {
            statusServer.reportDumpTableStatus(tableDumpStatus.getTaskid(), tableDumpStatus.isComplete(), tableDumpStatus.isWaiting(), tableDumpStatus.isFaild(), tableDumpStatus.getName());
        }
    };
    AtomicReference<ITISRpcService> ref = new AtomicReference<>();
    ref.set(new DataXAssembleSvcCompsite(statReceiveSvc));
    return new RpcServiceReference(ref);
}

Example 3 with RpcServiceReference

use of com.tis.hadoop.rpc.RpcServiceReference in project tis by qlangtech.

the class DataxExecutor method main.

/**
 * 入口开始执行
 *
 * @param args
 */
public static void main(String[] args) throws Exception {
    if (args.length != 7) {
        throw new IllegalArgumentException("args length must be 7,but now is " + args.length);
    }
    Integer jobId = Integer.parseInt(args[0]);
    String jobName = args[1];
    String dataXName = args[2];
    String incrStateCollectAddress = args[3];
    DataXJobSubmit.InstanceType execMode = DataXJobSubmit.InstanceType.parse(args[4]);
    final int allRows = Integer.parseInt(args[5]);
    // 任务每次执行会生成一个时间戳
    final String execTimeStamp = args[6];
    MDC.put(IParamContext.KEY_TASK_ID, String.valueOf(jobId));
    MDC.put(TISCollectionUtils.KEY_COLLECTION, dataXName);
    if (StringUtils.isEmpty(jobName)) {
        throw new IllegalArgumentException("arg 'jobName' can not be null");
    }
    if (StringUtils.isEmpty(dataXName)) {
        throw new IllegalArgumentException("arg 'dataXName' can not be null");
    }
    if (StringUtils.isEmpty(incrStateCollectAddress)) {
        throw new IllegalArgumentException("arg 'incrStateCollectAddress' can not be null");
    }
    StatusRpcClient.AssembleSvcCompsite statusRpc = StatusRpcClient.connect2RemoteIncrStatusServer(incrStateCollectAddress);
    Runtime.getRuntime().addShutdownHook(new Thread("dataX ShutdownHook") {

        @Override
        public void run() {
            statusRpc.close();
            // if (flumeAppendEnable) {
            com.qlangtech.tis.datax.log.TisFlumeLogstashV1Appender.instance.stop();
        // }
        }
    });
    DataxExecutor dataxExecutor = new DataxExecutor(new RpcServiceReference(new AtomicReference<>(statusRpc)), execMode, allRows);
    if (execMode == DataXJobSubmit.InstanceType.DISTRIBUTE) {
        // 如果是分布式执行状态，需要通过RPC的方式来监听监工是否执行了客户端终止操作
        Object thread = monitorDistributeCommand(jobId, jobName, dataXName, statusRpc, dataxExecutor);
        Objects.requireNonNull(thread);
    }
    try {
        dataxExecutor.reportDataXJobStatus(false, false, false, jobId, jobName);
        dataxExecutor.exec(jobId, jobName, dataXName, execTimeStamp);
        dataxExecutor.reportDataXJobStatus(false, jobId, jobName);
    } catch (Throwable e) {
        dataxExecutor.reportDataXJobStatus(true, jobId, jobName);
        logger.error(e.getMessage(), e);
        try {
            // 确保日志向远端写入了
            Thread.sleep(3000);
        } catch (InterruptedException ex) {
        }
        System.exit(1);
    }
    logger.info("dataX:" + dataXName + ",taskid:" + jobId + " finished");
    System.exit(0);
}

Example 4 with RpcServiceReference

use of com.tis.hadoop.rpc.RpcServiceReference in project tis by qlangtech.

the class BasicDataXExecutorTestCase method setUp.

@Override
protected void setUp() throws Exception {
    super.setUp();
    System.clearProperty(Config.DEFAULT_DATA_DIR);
    setDataDir();
    PluginStubUtils.setTISField();
    AtomicReference<ITISRpcService> ref = new AtomicReference<>();
    ref.set(StatusRpcClient.AssembleSvcCompsite.MOCK_PRC);
    statusRpc = new RpcServiceReference(ref);
    executor = createExecutor();
}

Example 5 with RpcServiceReference

use of com.tis.hadoop.rpc.RpcServiceReference in project plugins by qlangtech.

the class LocalTableDumpFactory method triggerTask.

public static IRemoteJobTrigger triggerTask(TaskContext context, ILocalTask task) {
    TisZkClient zk = context.getCoordinator().unwrap();
    Objects.requireNonNull(zk, "zk(TisZkClient) can not be null");
    AtomicReference<Throwable> errRef = new AtomicReference<>();
    CountDownLatch countDown = new CountDownLatch(1);
    final ExecutorService executor = Executors.newSingleThreadExecutor((r) -> {
        Thread t = new Thread(r);
        t.setUncaughtExceptionHandler((thread, e) -> {
            errRef.set(e);
            logger.error(e.getMessage(), e);
            countDown.countDown();
        });
        return t;
    });
    return new IRemoteJobTrigger() {

        @Override
        public void submitJob() {
            executor.execute(() -> {
                RpcServiceReference statusRpc = null;
                try {
                    statusRpc = StatusRpcClient.getService(zk);
                    task.process(statusRpc);
                    countDown.countDown();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                } finally {
                    try {
                        statusRpc.get().close();
                    } catch (Throwable e) {
                    }
                }
            });
        }

        @Override
        public RunningStatus getRunningStatus() {
            RunningStatus runningStatus = null;
            // 反馈执行状态
            if (countDown.getCount() > 0) {
                runningStatus = new RunningStatus(0, false, false);
            } else {
                executor.shutdown();
                runningStatus = new RunningStatus(1, true, errRef.get() == null);
            }
            return runningStatus;
        }
    };
}