Examples with ITISCoordinator com.qlangtech.tis.cloud.ITISCoordinator used on opensource projects

Example 6 with ITISCoordinator

use of com.qlangtech.tis.cloud.ITISCoordinator in project tis by qlangtech.

the class DataXJobConsumer method createQueue.

public static DistributedQueue<CuratorDataXTaskMessage> createQueue(CuratorFramework curatorClient, String zkQueuePath, QueueConsumer<CuratorDataXTaskMessage> consumer) {
    try {
        if (StringUtils.isEmpty(zkQueuePath)) {
            throw new IllegalArgumentException("param zkQueuePath can not be null");
        }
        // TaskConfig taskConfig = TaskConfig.getInstance();
        int count = 0;
        while (!curatorClient.getZookeeperClient().isConnected()) {
            if (count++ > 4) {
                throw new IllegalStateException(" zookeeper server can not be established");
            }
            logger.info("waiting connect to zookeeper server");
            Thread.sleep(5000);
        }
        ITISCoordinator coordinator = getCoordinator(null, curatorClient);
        ZkUtils.guaranteeExist(coordinator, zkQueuePath);
        QueueBuilder<CuratorDataXTaskMessage> builder = QueueBuilder.builder(curatorClient, consumer, new MessageSerializer(), zkQueuePath);
        // .maxItems(4);
        DistributedQueue<CuratorDataXTaskMessage> queue = builder.buildQueue();
        queue.start();
        return queue;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 7 with ITISCoordinator

use of com.qlangtech.tis.cloud.ITISCoordinator in project tis by qlangtech.

the class SysInitializeAction method getCoordinator.

private static ITISCoordinator getCoordinator(ZooKeeper zooKeeper) throws Exception {
    ITISCoordinator coordinator = null;
    coordinator = new AdapterTisCoordinator() {

        @Override
        public List<String> getChildren(String zkPath, Watcher watcher, boolean b) {
            try {
                return zooKeeper.getChildren(zkPath, watcher);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }

        @Override
        public boolean exists(String path, boolean watch) {
            try {
                return zooKeeper.exists(path, watch) != null;
            } catch (Exception e) {
                throw new RuntimeException(path, e);
            }
        }

        @Override
        public void create(String path, byte[] data, boolean persistent, boolean sequential) {
            CreateMode createMode = null;
            if (persistent) {
                createMode = sequential ? CreateMode.PERSISTENT_SEQUENTIAL : CreateMode.PERSISTENT;
            } else {
                createMode = sequential ? CreateMode.EPHEMERAL_SEQUENTIAL : CreateMode.EPHEMERAL;
            }
            try {
                zooKeeper.create(path, data, ZooDefs.Ids.OPEN_ACL_UNSAFE, createMode);
            } catch (Exception e) {
                throw new RuntimeException("path:" + path, e);
            }
        }

        @Override
        public byte[] getData(String zkPath, Watcher o, Stat stat, boolean b) {
            try {
                return zooKeeper.getData(zkPath, o, stat);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    };
    return coordinator;
}

Example 8 with ITISCoordinator

use of com.qlangtech.tis.cloud.ITISCoordinator in project plugins by qlangtech.

the class TestLocalTableDumpAndIndex method testSingleTableDump.

public void testSingleTableDump() throws Exception {
    LocalTableDumpFactory tableDumpFactory = new LocalTableDumpFactory();
    File dumpRoot = LocalTableDumpFactory.getLocalOfflineRootDir();
    tableDumpFactory.name = "test";
    DataSourceFactory mockEmployeesDataSource = MockDataSourceFactory.getMockEmployeesDataSource();
    tableDumpFactory.setDataSourceFactoryGetter((tab) -> {
        return mockEmployeesDataSource;
    });
    ITISCoordinator zkCoordinator = MockZKUtils.createZkMock();
    // search4(.+?)_shard(\d+?)_replica_n(\d+?)
    String mockSolrCore = INDEX_COLLECTION + "_shard1_replica_n1";
    IJoinTaskContext execContext = this.mock("execContext", IJoinTaskContext.class);
    // EntityName targetTableName = EntityName.parse(DB_EMPLOYEES+"."); ctx.getAttribute(IParamContext.KEY_BUILD_TARGET_TABLE_NAME);
    EasyMock.expect(execContext.getAttribute(IParamContext.KEY_BUILD_TARGET_TABLE_NAME)).andReturn(getEmployeeTab()).anyTimes();
    replay();
    int round = 0;
    ArrayDeque<Date> createDates = Queues.newArrayDeque();
    // 一共测试5轮
    Date timestamp = null;
    while (round++ < 5) {
        timestamp = new Date();
        createDates.addLast(timestamp);
        TaskContext taskContext = MockTaskContextUtils.create(timestamp);
        taskContext.setCoordinator(zkCoordinator);
        /**
         * -----------------------------------------------------------
         * 开始执行数据导入流程
         *             -----------------------------------------------------------
         */
        startDump(tableDumpFactory, taskContext);
        /**
         * -----------------------------------------------------------
         * 开始执行索引构建流程
         *             -----------------------------------------------------------
         */
        startIndexBuild(mockSolrCore, execContext, zkCoordinator, MockTaskContextUtils.timeFormatYyyyMMddHHmmss.get().format(timestamp));
        Thread.sleep(1000);
    }
    int index = 0;
    File tableRoot = new File(dumpRoot, DB_EMPLOYEES + "/" + TABLE_EMPLOYEES + "/all");
    assertTrue(tableRoot.exists());
    String[] subTimeStampFiles = tableRoot.list();
    Set<String> timestamps = Sets.newHashSet();
    int maxHistorySave = ITableDumpConstant.MAX_PARTITION_SAVE + 1;
    while (index++ < maxHistorySave) {
        timestamps.add(MockTaskContextUtils.timeFormatYyyyMMddHHmmss.get().format(createDates.pollLast()));
    }
    assertEquals("maxHistorySave", maxHistorySave, subTimeStampFiles.length);
    for (String subFile : subTimeStampFiles) {
        assertTrue("shall contain file:" + new File(tableRoot, subFile), timestamps.contains(subFile));
    // TODO 继续校验文件夹中的内容是否正确
    }
    File indexBuildRoot = new File(dumpRoot, INDEX_COLLECTION + "/all/0/output");
    for (String indexBuildRootSub : indexBuildRoot.list()) {
        assertTrue("shall contain file:" + new File(indexBuildRoot, indexBuildRootSub), timestamps.contains(indexBuildRootSub));
    }
    verifyAll();
}

Example 9 with ITISCoordinator

use of com.qlangtech.tis.cloud.ITISCoordinator in project tis by qlangtech.

the class SysInitializeAction method initializeZkPath.

// 初始化ZK内容
public boolean initializeZkPath(String zkHost) {
    Matcher matcher = PATTERN_ZK_ADDRESS.matcher(zkHost);
    if (!matcher.matches()) {
        throw new IllegalStateException("zk address " + zkHost + " is not match " + PATTERN_ZK_ADDRESS);
    }
    final String zkServer = matcher.group(1);
    String zkSubDir = StringUtils.trimToEmpty(matcher.group(2));
    logger.info("zkServer:{},zkSubDir:{}", zkServer, zkSubDir);
    if (StringUtils.endsWith(zkSubDir, "/")) {
        zkSubDir = StringUtils.substring(zkSubDir, 0, zkSubDir.length() - 1);
    }
    ZooKeeper zk = null;
    StringBuffer buildLog = new StringBuffer();
    String createPath = null;
    List<String> createPaths = Lists.newArrayList();
    try {
        // final Watcher watcher = new Watcher() {
        // @Override
        // public void process(WatchedEvent event) {
        // logger.info(event.getType() + "," + event.getState() + "," + event.getPath());
        // }
        // };
        // new ZooKeeper(zkServer, 50000, watcher);
        zk = this.createZK(zkServer);
        zk.getChildren("/", false);
        buildLog.append("create zkServer ").append(zkServer);
        createPath = zkSubDir + "/tis";
        ITISCoordinator coordinator = getCoordinator(zk);
        logger.info("guaranteeExist:{}", createPath);
        createPaths.add(createPath);
        ZkUtils.guaranteeExist(coordinator, createPath);
        buildLog.append(",path1:").append(createPath);
        createPath = zkSubDir + "/tis-lock/dumpindex";
        createPaths.add(createPath);
        ZkUtils.guaranteeExist(coordinator, createPath);
        buildLog.append(",path2:").append(createPath);
        // createPath = zkSubDir + "/configs/" + CoreAction.DEFAULT_SOLR_CONFIG;
        // createPaths.add(createPath);
        // ZkUtils.guaranteeExist(coordinator, createPath);
        // buildLog.append(",path3:").append(createPath);
        logger.info(buildLog.toString());
    } catch (Throwable e) {
        throw new IllegalStateException("zk address:" + zkServer + " can not connect Zookeeper server", e);
    } finally {
        try {
            zk.close();
        } catch (Throwable e) {
        }
    }
    try {
        Thread.sleep(10000);
    } catch (InterruptedException e) {
    }
    try {
        zk = this.createZK(zkServer);
        for (String p : createPaths) {
            if (zk.exists(p, false) == null) {
                throw new TisException("create path:" + p + " must be exist");
            }
        }
    } catch (TisException e) {
        throw e;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        try {
            zk.close();
        } catch (InterruptedException e) {
        }
    }
    // }
    return true;
}

Example 10 with ITISCoordinator

use of com.qlangtech.tis.cloud.ITISCoordinator in project tis by qlangtech.

the class TestCollectionAction method testDoFullbuild.

public void testDoFullbuild() throws Exception {
    ITISCoordinator zkCoordinator = MockZKUtils.createZkMock();
    // createAssembleLogCollectPathMock(zkCoordinator);
    MockZooKeeperGetter.mockCoordinator = zkCoordinator;
    request.setParameter("emethod", "fullbuild");
    request.setParameter("action", "collection_action");
    JSONObject content = new JSONObject();
    content.put(CollectionAction.KEY_INDEX_NAME, TEST_TABLE_EMPLOYEES_NAME);
    request.setContent(content.toJSONString().getBytes(TisUTF8.get()));
    ActionProxy proxy = getActionProxy();
    this.replay();
    String result = proxy.execute();
    assertEquals("CollectionAction_ajax", result);
    AjaxValve.ActionExecResult aResult = showBizResult();
    assertNotNull(aResult);
    assertTrue(aResult.isSuccess());
    CoreAction.TriggerBuildResult triggerResult = (CoreAction.TriggerBuildResult) aResult.getBizResult();
    assertNotNull(triggerResult);
    assertEquals(1234, triggerResult.getTaskid());
    this.verifyAll();
}