Examples with ITISCoordinator com.qlangtech.tis.cloud.ITISCoordinator used on opensource projects

Example 11 with ITISCoordinator

use of com.qlangtech.tis.cloud.ITISCoordinator in project tis by qlangtech.

the class TestCollectionAction method testDoCreate.

public void testDoCreate() throws Exception {
    this.clearUpDB();
    ITISCoordinator zkCoordinator = MockZKUtils.createZkMock();
    MockZooKeeperGetter.mockCoordinator = zkCoordinator;
    // IExpectationSetters<byte[]> iExpectationSetters = createCoordinatorMock((coord) -> {
    // //      EasyMock.expect(coord.getChildren(ZkUtils.ZK_ASSEMBLE_LOG_COLLECT_PATH, null, true))
    // //        .andReturn();
    // createAssembleLogCollectPathMock(coord);
    // });
    // iExpectationSetters.times(2);
    TISZkStateReader tisZkStateReader = buildTisZkStateReaderMock();
    SelectableServer.CoreNode coreNode = new SelectableServer.CoreNode();
    coreNode.setHostName("hostname");
    coreNode.setNodeName("nodename");
    EasyMock.expect(tisZkStateReader.getSelectTableNodes()).andReturn(Collections.singletonList(coreNode));
    request.setParameter("emethod", "create");
    request.setParameter("action", "collection_action");
    JSONObject content = getPostJSONContent(TEST_TABLE_EMPLOYEES_NAME);
    request.setContent(content.toJSONString().getBytes(TisUTF8.get()));
    ActionProxy proxy = getActionProxy();
    AtomicReference<AppKey> appKeyRef = new AtomicReference<>();
    AddAppAction.appKeyProcess = (key) -> {
        appKeyRef.set(key);
    };
    AtomicBoolean schemaParseResultProcessed = new AtomicBoolean(false);
    SchemaAction.parseResultCallback4test = (cols, schemaParseResult) -> {
        List<PSchemaField> schemaFields = ((ParseResult) schemaParseResult).getSchemaFields();
        assertNotNull(schemaFields);
        assertEquals(8, schemaFields.size());
        Map<String, ISchemaField> fields = schemaFields.stream().collect(Collectors.toMap((c) -> c.getName(), (c) -> c));
        String emp_no = "emp_no";
        ISchemaField pk = fields.get(emp_no);
        assertNotNull(pk);
        assertTrue(StringUtils.isEmpty(pk.getTokenizerType()));
        assertEquals(ReflectSchemaFieldType.STRING.literia, pk.getTisFieldTypeName());
        assertEquals(emp_no, schemaParseResult.getUniqueKey());
        assertEquals(emp_no, schemaParseResult.getSharedKey());
        String birth_date = "birth_date";
        ISchemaField field = fields.get(birth_date);
        assertNotNull(field);
        assertEquals(ReflectSchemaFieldType.DATE.literia, field.getTisFieldTypeName());
        assertTrue(StringUtils.isEmpty(field.getTokenizerType()));
        // String first_name = "first_name";
        field = fields.get(FIELD_EMPLOYEES_FIRST_NAME);
        assertNotNull(field);
        assertEquals(ReflectSchemaFieldType.STRING.literia, field.getTisFieldTypeName());
        assertEquals(ReflectSchemaFieldType.LIKE.literia, field.getTokenizerType());
        // String last_name = "last_name";
        field = fields.get(FIELD_EMPLOYEES_LAST_NAME);
        assertNotNull(field);
        assertEquals(ReflectSchemaFieldType.STRING.literia, field.getTisFieldTypeName());
        assertEquals(ReflectSchemaFieldType.LIKE.literia, field.getTokenizerType());
        String gender = "gender";
        field = fields.get(gender);
        assertNotNull(field);
        assertEquals(ReflectSchemaFieldType.STRING.literia, field.getTisFieldTypeName());
        assertTrue(StringUtils.isEmpty(field.getTokenizerType()));
        String hire_date = "hire_date";
        field = fields.get(hire_date);
        assertNotNull(field);
        assertEquals(ReflectSchemaFieldType.DATE.literia, field.getTisFieldTypeName());
        assertTrue(StringUtils.isEmpty(field.getTokenizerType()));
        schemaParseResultProcessed.set(true);
    };
    this.replay();
    // 执行
    String result = proxy.execute();
    // assertEquals(Action.NONE, result);
    AjaxValve.ActionExecResult actionExecResult = showBizResult();
    CoreAction.TriggerBuildResult triggerResult = (CoreAction.TriggerBuildResult) actionExecResult.getBizResult();
    assertNotNull("triggerResult can not be null", triggerResult);
    assertTrue(triggerResult.success);
    assertEquals("taskId must large than 0", 1234, triggerResult.getTaskid());
    // SnapshotDomain snapshotDomain = HttpConfigFileReader.getResource(COLLECTION_NAME, targetSnapshotid, RunEnvironment.getSysRuntime(), ConfigFileReader.getAry);
    // 判断缓存中应该已经有snapshotDomain了
    assertNotNull("appKeyRef can not be null", appKeyRef.get());
    SnapshotDomain snapshotDomain = LoadSolrCoreConfigByAppNameServlet.getSnapshotDomain(ConfigFileReader.getConfigList(), appKeyRef.get().setFromCache(true), null);
    assertNotNull("snapshotDomain can not null", snapshotDomain);
    assertTrue(actionExecResult.isSuccess());
    assertTrue("schemaParseResultProcessed must be processd", schemaParseResultProcessed.get());
    this.verifyAll();
    AtomicBoolean executed = new AtomicBoolean(false);
    SolrFieldsParser.fieldTypeVisitor = (nodes) -> {
        NamedNodeMap tokenizerAttrs = null;
        outter: for (int i = 0; i < nodes.getLength(); i++) {
            Node node = nodes.item(i);
            NamedNodeMap attrs = node.getAttributes();
            String typeName = DOMUtil.getAttr(attrs, "name");
            if ("like".equals(typeName)) {
                NodeList childNodes = node.getChildNodes();
                for (int ii = 0; ii < childNodes.getLength(); ii++) {
                    Node item = childNodes.item(ii);
                    if ("analyzer".equals(item.getNodeName())) {
                        Node tokenizerNode = null;
                        NodeList analyzerChildNodes = item.getChildNodes();
                        for (int jj = 0; jj < analyzerChildNodes.getLength(); jj++) {
                            tokenizerNode = analyzerChildNodes.item(jj);
                            if ("tokenizer".equals(tokenizerNode.getNodeName())) {
                                tokenizerAttrs = tokenizerNode.getAttributes();
                                assertEquals(ISnapshotViewDAO.KEY_MIN_GRAM_SIZE, minGramSize, Integer.parseInt(DOMUtil.getAttr(tokenizerAttrs, ISnapshotViewDAO.KEY_MIN_GRAM_SIZE)));
                                assertEquals(ISnapshotViewDAO.KEY_MAX_GRAM_SIZE, maxGramSize, Integer.parseInt(DOMUtil.getAttr(tokenizerAttrs, ISnapshotViewDAO.KEY_MAX_GRAM_SIZE)));
                                break outter;
                            }
                        }
                        assertNotNull("tokenizerNode can not be null", tokenizerNode);
                        // =childNodes.item(0).getChildNodes().item(0);
                        break;
                    }
                }
            }
        }
        assertNotNull("tokenizerAttrs can not be null", tokenizerAttrs);
        executed.set(true);
    };
    SolrFieldsParser.parse(() -> {
        return snapshotDomain.getSolrSchema().getContent();
    });
    assertTrue("must have execute", executed.get());
}

Example 12 with ITISCoordinator

use of com.qlangtech.tis.cloud.ITISCoordinator in project tis by qlangtech.

the class TestCoreAction method testTriggerFullbuildTaskByWithoutDefinePrimaryTableShareKey.

/**
 * 执行索引全量构建过程中，测试ERRule没有定义主表的<b>shareKey</b>，会导致final表的分区函数无法正常创建，需要主动抛出一个异常
 */
public void testTriggerFullbuildTaskByWithoutDefinePrimaryTableShareKey() throws Exception {
    HttpUtils.addMockApply(-1, "tis-assemble/trigger", "assemble.trigger.result.faild.json", TestCoreAction.class);
    ITISCoordinator zkCoordinator = MockZKUtils.createZkMock();
    MockZooKeeperGetter.mockCoordinator = zkCoordinator;
    // this.createCoordinatorMock(false, (zk) -> {
    // });
    triggerFullbuildTask(null, (aResult) -> {
        assertFalse(aResult.isSuccess());
        List<String> errorMsgs = aResult.getErrorMsgs();
        assertNotNull(errorMsgs);
        assertTrue(errorMsgs.size() > 0);
    });
}

Example 13 with ITISCoordinator

use of com.qlangtech.tis.cloud.ITISCoordinator in project tis by qlangtech.

the class DataXJobConsumer method getDataXJobConsumer.

public static DataXJobConsumer getDataXJobConsumer(String zkQueuePath, String zkAddress) throws Exception {
    CuratorFramework curatorClient = getCuratorFramework(zkAddress);
    ITISCoordinator coordinator = getCoordinator(zkAddress, curatorClient);
    // String dataxName, Integer jobId, String jobName, String jobPath
    DataXJobConsumer dataXJobConsume = new DataXJobConsumer(curatorClient, coordinator);
    dataXJobConsume.createQueue(zkQueuePath);
    return dataXJobConsume;
}

Example 14 with ITISCoordinator

use of com.qlangtech.tis.cloud.ITISCoordinator in project tis by qlangtech.

the class DataXJobConsumer method getCoordinator.

private static ITISCoordinator getCoordinator(String zkAddress, CuratorFramework curatorClient) throws Exception {
    ITISCoordinator coordinator = null;
    final ZooKeeper zooKeeper = curatorClient.getZookeeperClient().getZooKeeper();
    coordinator = new AdapterTisCoordinator() {

        @Override
        public List<String> getChildren(String zkPath, Watcher watcher, boolean b) {
            try {
                return zooKeeper.getChildren(zkPath, watcher);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }

        @Override
        public boolean exists(String path, boolean watch) {
            try {
                Stat exists = zooKeeper.exists(path, false);
                return exists != null;
            } catch (Exception e) {
                throw new RuntimeException("path:" + path, e);
            }
        }

        @Override
        public void create(String path, byte[] data, boolean persistent, boolean sequential) {
            CreateMode createMode = null;
            if (persistent) {
                createMode = sequential ? CreateMode.PERSISTENT_SEQUENTIAL : CreateMode.PERSISTENT;
            } else {
                createMode = sequential ? CreateMode.EPHEMERAL_SEQUENTIAL : CreateMode.EPHEMERAL;
            }
            try {
                zooKeeper.create(path, data, ZooDefs.Ids.OPEN_ACL_UNSAFE, createMode);
            } catch (Exception e) {
                throw new RuntimeException("path:" + path, e);
            }
        }

        @Override
        public byte[] getData(String zkPath, Watcher o, Stat stat, boolean b) {
            try {
                return zooKeeper.getData(zkPath, o, stat);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    };
    logger.info("create TIS new zookeeper instance with ,system zkHost:{}", Config.getZKHost());
    // }
    return coordinator;
}

Example 15 with ITISCoordinator

use of com.qlangtech.tis.cloud.ITISCoordinator in project tis by qlangtech.

the class TestIncrTagHeatBeatMonitor method testBuild.

// public void test(){
// System.out.println(  String.format("hell%d",1) );
// }
public void testBuild() throws Exception {
    String resName = "collection_TopicTags_status_%d.json";
    AtomicInteger resFetchCount = new AtomicInteger();
    HttpUtils.mockConnMaker = new HttpUtils.DefaultMockConnectionMaker() {

        @Override
        protected MockHttpURLConnection createConnection(URL url, List<ConfigFileContext.Header> heads, ConfigFileContext.HTTPMethod method, byte[] content, HttpUtils.IClasspathRes cpRes) {
            String res = String.format(resName, resFetchCount.incrementAndGet());
            try {
                return new MockHttpURLConnection(TestIncrTagHeatBeatMonitor.class.getResourceAsStream(res));
            } catch (Throwable e) {
                throw new RuntimeException(res, e);
            }
        }
    };
    HttpUtils.addMockApply(0, "incr-control", StringUtils.EMPTY, TestIncrTagHeatBeatMonitor.class);
    ZooKeeperGetter zookeeper = EasyMock.createMock("zooKeeperGetter", ZooKeeperGetter.class);
    // EasyMock.createMock("coordinator", ITISCoordinator.class);
    ITISCoordinator coordinator = MockZKUtils.createZkMock();
    int times = 5;
    // TestCollectionAction.createAssembleLogCollectPathMock(coordinator,times);
    // final String childNodeName = "000001";
    // EasyMock.expect(coordinator.getChildren("/tis/incr-transfer-group/incr-state-collect", null, true))
    // .andReturn(Collections.singletonList(childNodeName)).times(times);
    // EasyMock.expect(coordinator.getData("/tis/incr-transfer-group/incr-state-collect/" + childNodeName, null, new Stat(), true))
    // .andReturn("127.0.0.1".getBytes(TisUTF8.get())).times(times);
    EasyMock.expect(zookeeper.getInstance()).andReturn(coordinator).times(times);
    // 增量节点处理
    final Map<String, TopicTagStatus> /* tag */
    transferTagStatus = new HashMap<>();
    final Map<String, TopicTagStatus> /* tag */
    binlogTopicTagStatus = new HashMap<>();
    Collection<TopicTagIncrStatus.FocusTags> focusTags = Lists.newArrayList();
    // String topic, Collection<String> tags
    ArrayList<String> tags = Lists.newArrayList(tag_servicebillinfo, tag_talpayinfo, "orderdetail", "specialfee", "ent_expense", "payinfo", "order_bill", "instancedetail", "ent_expense_order");
    focusTags.add(new TopicTagIncrStatus.FocusTags("test-topic", tags));
    TopicTagIncrStatus topicTagIncrStatus = new TopicTagIncrStatus(focusTags);
    MockWebSocketMessagePush wsMessagePush = new MockWebSocketMessagePush();
    MockMQConsumerStatus mqConsumerStatus = new MockMQConsumerStatus();
    // IncrTagHeatBeatMonitor incrTagHeatBeatMonitor = new IncrTagHeatBeatMonitor(
    // this.collectionName, wsMessagePush, transferTagStatus, binlogTopicTagStatus, topicTagIncrStatus, mqConsumerStatus, zookeeper);
    EasyMock.replay(zookeeper, coordinator);
    // incrTagHeatBeatMonitor.build();
    assertEquals(6, wsMessagePush.count);
    EasyMock.verify(zookeeper, coordinator);
}