Examples with AppKey com.qlangtech.tis.openapi.impl.AppKey used on opensource projects

Example 1 with AppKey

use of com.qlangtech.tis.openapi.impl.AppKey in project tis by qlangtech.

the class ResSynManager method createSynManagerOnlineFromDaily.

/**
 * 创建从日常更新配置同步到线上管理器(在日常执行)
 *
 * @param appName
 * @param runContext
 * @return
 * @throws Exception
 */
public static ResSynManager createSynManagerOnlineFromDaily(String appName, RunContext runContext) throws Exception {
    final AppKey appKey = new AppKey(appName, (short) 0, RunEnvironment.DAILY, true);
    // 取得内容从数据库中拿
    appKey.setFromCache(false);
    // 日常向线上推送的文件
    SnapshotDomain dailyResDomain = LoadSolrCoreConfigByAppNameServlet.getSnapshotDomain(ConfigFileReader.getConfigList(), appKey, runContext);
    if (dailyResDomain == null) {
        throw new IllegalStateException("appName:" + appName + " is not exist");
    }
    // 线上的配置
    SnapshotDomain onlineRes = ResSynManager.getOnlineResourceConfig(appName);
    return create(appName, dailyResDomain, onlineRes, runContext);
}

Example 2 with AppKey

use of com.qlangtech.tis.openapi.impl.AppKey in project tis by qlangtech.

the class AddAppAction method createCollection.

protected Optional<Application> createCollection(Context context, CreateIndexConfirmModel confiemModel, SchemaResult schemaResult, ICreateNewApp appCreator) throws Exception {
    ExtendApp extApp = confiemModel.getAppform();
    appendPrefix(extApp);
    String workflow = confiemModel.getAppform().getWorkflow();
    if (StringUtils.isBlank(workflow)) {
        this.addErrorMessage(context, "缺少全量数据流信息");
        return Optional.empty();
    }
    final String[] candidateNodeIps = confiemModel.getCoreNodeCandidate();
    if (candidateNodeIps == null || candidateNodeIps.length < 1) {
        this.addErrorMessage(context, "请选择引擎节点");
        return Optional.empty();
    }
    if (!schemaResult.isSuccess()) {
        return Optional.empty();
    }
    Application app = new Application();
    app.setAppId(confiemModel.getTplAppId());
    Integer publishSnapshotId = getPublishSnapshotId(this.getServerGroupDAO(), app);
    byte[] content = schemaResult.content;
    SelectableServer.ServerNodeTopology coreNode = confiemModel.getCoreNode();
    final int gourpCount = coreNode.getShardCount();
    int repliation = coreNode.getReplicaCount();
    // 由于是在日常环境中，默认就是设置为 1*1
    FCoreRequest request = new FCoreRequest(CoreAction.createIps(context, extApp.getProjectName(), candidateNodeIps), gourpCount);
    for (String ip : candidateNodeIps) {
        request.addNodeIps(gourpCount - 1, ip);
    }
    request.setValid(true);
    CreateAppResult createResult = appCreator.createNewApp(context, extApp, publishSnapshotId, content);
    if (!createResult.isSuccess()) {
        return Optional.of(app);
    }
    IAppSource.save(this, extApp.getProjectName(), extApp.createAppSource(this));
    /**
     * *************************************************************************************
     * 因为这里数据库的事务还没有提交，需要先将schema配置信息保存到缓存中去以便solrcore节点能获取到
     * 设置缓存
     * **************************************************************************************
     */
    final AppKey appKey = AppKey.create(extApp.getProjectName());
    if (!(createResult instanceof CreateSnapshotResult)) {
        throw new IllegalStateException("instance of createResult must be CreateSnapshotResult");
    }
    CreateSnapshotResult snapshotResult = (CreateSnapshotResult) createResult;
    appKey.setTargetSnapshotId((long) snapshotResult.getNewId());
    appKey.setFromCache(false);
    appKeyProcess.process(appKey);
    LoadSolrCoreConfigByAppNameServlet.getSnapshotDomain(ConfigFileReader.getConfigList(), appKey, this);
    CoreAction.createCollection(this, context, gourpCount, repliation, request, snapshotResult.getNewId());
    return Optional.of(app);
}

Example 3 with AppKey

use of com.qlangtech.tis.openapi.impl.AppKey in project tis by qlangtech.

the class LoadSolrCoreConfigByAppNameServlet method doGet.

@Override
protected void doGet(HttpServletRequest request, HttpServletResponse resp) throws ServletException, IOException {
    Matcher matcher = getMatcher(request);
    final AppKey appKey = new AppKey(matcher.group(1), /* appName ========== */
    Short.parseShort(matcher.group(2)), /* groupIndex */
    RunEnvironment.getEnum(matcher.group(3)), "true".equals(request.getParameter(HttpConfigFileReader.unmergeglobalparams)));
    setSnapshotId(request, appKey);
    log.warn("getres " + appKey.appName + ",unmerge:" + appKey.unmergeglobalparams + ",client ip:" + request.getRemoteAddr() + "=>" + request.getRequestURI());
    List<PropteryGetter> needRes = new ArrayList<PropteryGetter>();
    matcher = resourcePattern.matcher(getResources(request));
    while ((matcher).find()) {
        needRes.add(ConfigFileReader.createPropertyGetter(matcher.group()));
    }
    appKey.setFromCache(true);
    SnapshotDomain colon = getSnapshotDomain(needRes, appKey, this.getContext());
    this.wirteXml2Client(resp, colon);
}

Example 4 with AppKey

use of com.qlangtech.tis.openapi.impl.AppKey in project tis by qlangtech.

the class TestCollectionAction method testDoCreate.

public void testDoCreate() throws Exception {
    this.clearUpDB();
    ITISCoordinator zkCoordinator = MockZKUtils.createZkMock();
    MockZooKeeperGetter.mockCoordinator = zkCoordinator;
    // IExpectationSetters<byte[]> iExpectationSetters = createCoordinatorMock((coord) -> {
    // //      EasyMock.expect(coord.getChildren(ZkUtils.ZK_ASSEMBLE_LOG_COLLECT_PATH, null, true))
    // //        .andReturn();
    // createAssembleLogCollectPathMock(coord);
    // });
    // iExpectationSetters.times(2);
    TISZkStateReader tisZkStateReader = buildTisZkStateReaderMock();
    SelectableServer.CoreNode coreNode = new SelectableServer.CoreNode();
    coreNode.setHostName("hostname");
    coreNode.setNodeName("nodename");
    EasyMock.expect(tisZkStateReader.getSelectTableNodes()).andReturn(Collections.singletonList(coreNode));
    request.setParameter("emethod", "create");
    request.setParameter("action", "collection_action");
    JSONObject content = getPostJSONContent(TEST_TABLE_EMPLOYEES_NAME);
    request.setContent(content.toJSONString().getBytes(TisUTF8.get()));
    ActionProxy proxy = getActionProxy();
    AtomicReference<AppKey> appKeyRef = new AtomicReference<>();
    AddAppAction.appKeyProcess = (key) -> {
        appKeyRef.set(key);
    };
    AtomicBoolean schemaParseResultProcessed = new AtomicBoolean(false);
    SchemaAction.parseResultCallback4test = (cols, schemaParseResult) -> {
        List<PSchemaField> schemaFields = ((ParseResult) schemaParseResult).getSchemaFields();
        assertNotNull(schemaFields);
        assertEquals(8, schemaFields.size());
        Map<String, ISchemaField> fields = schemaFields.stream().collect(Collectors.toMap((c) -> c.getName(), (c) -> c));
        String emp_no = "emp_no";
        ISchemaField pk = fields.get(emp_no);
        assertNotNull(pk);
        assertTrue(StringUtils.isEmpty(pk.getTokenizerType()));
        assertEquals(ReflectSchemaFieldType.STRING.literia, pk.getTisFieldTypeName());
        assertEquals(emp_no, schemaParseResult.getUniqueKey());
        assertEquals(emp_no, schemaParseResult.getSharedKey());
        String birth_date = "birth_date";
        ISchemaField field = fields.get(birth_date);
        assertNotNull(field);
        assertEquals(ReflectSchemaFieldType.DATE.literia, field.getTisFieldTypeName());
        assertTrue(StringUtils.isEmpty(field.getTokenizerType()));
        // String first_name = "first_name";
        field = fields.get(FIELD_EMPLOYEES_FIRST_NAME);
        assertNotNull(field);
        assertEquals(ReflectSchemaFieldType.STRING.literia, field.getTisFieldTypeName());
        assertEquals(ReflectSchemaFieldType.LIKE.literia, field.getTokenizerType());
        // String last_name = "last_name";
        field = fields.get(FIELD_EMPLOYEES_LAST_NAME);
        assertNotNull(field);
        assertEquals(ReflectSchemaFieldType.STRING.literia, field.getTisFieldTypeName());
        assertEquals(ReflectSchemaFieldType.LIKE.literia, field.getTokenizerType());
        String gender = "gender";
        field = fields.get(gender);
        assertNotNull(field);
        assertEquals(ReflectSchemaFieldType.STRING.literia, field.getTisFieldTypeName());
        assertTrue(StringUtils.isEmpty(field.getTokenizerType()));
        String hire_date = "hire_date";
        field = fields.get(hire_date);
        assertNotNull(field);
        assertEquals(ReflectSchemaFieldType.DATE.literia, field.getTisFieldTypeName());
        assertTrue(StringUtils.isEmpty(field.getTokenizerType()));
        schemaParseResultProcessed.set(true);
    };
    this.replay();
    // 执行
    String result = proxy.execute();
    // assertEquals(Action.NONE, result);
    AjaxValve.ActionExecResult actionExecResult = showBizResult();
    CoreAction.TriggerBuildResult triggerResult = (CoreAction.TriggerBuildResult) actionExecResult.getBizResult();
    assertNotNull("triggerResult can not be null", triggerResult);
    assertTrue(triggerResult.success);
    assertEquals("taskId must large than 0", 1234, triggerResult.getTaskid());
    // SnapshotDomain snapshotDomain = HttpConfigFileReader.getResource(COLLECTION_NAME, targetSnapshotid, RunEnvironment.getSysRuntime(), ConfigFileReader.getAry);
    // 判断缓存中应该已经有snapshotDomain了
    assertNotNull("appKeyRef can not be null", appKeyRef.get());
    SnapshotDomain snapshotDomain = LoadSolrCoreConfigByAppNameServlet.getSnapshotDomain(ConfigFileReader.getConfigList(), appKeyRef.get().setFromCache(true), null);
    assertNotNull("snapshotDomain can not null", snapshotDomain);
    assertTrue(actionExecResult.isSuccess());
    assertTrue("schemaParseResultProcessed must be processd", schemaParseResultProcessed.get());
    this.verifyAll();
    AtomicBoolean executed = new AtomicBoolean(false);
    SolrFieldsParser.fieldTypeVisitor = (nodes) -> {
        NamedNodeMap tokenizerAttrs = null;
        outter: for (int i = 0; i < nodes.getLength(); i++) {
            Node node = nodes.item(i);
            NamedNodeMap attrs = node.getAttributes();
            String typeName = DOMUtil.getAttr(attrs, "name");
            if ("like".equals(typeName)) {
                NodeList childNodes = node.getChildNodes();
                for (int ii = 0; ii < childNodes.getLength(); ii++) {
                    Node item = childNodes.item(ii);
                    if ("analyzer".equals(item.getNodeName())) {
                        Node tokenizerNode = null;
                        NodeList analyzerChildNodes = item.getChildNodes();
                        for (int jj = 0; jj < analyzerChildNodes.getLength(); jj++) {
                            tokenizerNode = analyzerChildNodes.item(jj);
                            if ("tokenizer".equals(tokenizerNode.getNodeName())) {
                                tokenizerAttrs = tokenizerNode.getAttributes();
                                assertEquals(ISnapshotViewDAO.KEY_MIN_GRAM_SIZE, minGramSize, Integer.parseInt(DOMUtil.getAttr(tokenizerAttrs, ISnapshotViewDAO.KEY_MIN_GRAM_SIZE)));
                                assertEquals(ISnapshotViewDAO.KEY_MAX_GRAM_SIZE, maxGramSize, Integer.parseInt(DOMUtil.getAttr(tokenizerAttrs, ISnapshotViewDAO.KEY_MAX_GRAM_SIZE)));
                                break outter;
                            }
                        }
                        assertNotNull("tokenizerNode can not be null", tokenizerNode);
                        // =childNodes.item(0).getChildNodes().item(0);
                        break;
                    }
                }
            }
        }
        assertNotNull("tokenizerAttrs can not be null", tokenizerAttrs);
        executed.set(true);
    };
    SolrFieldsParser.parse(() -> {
        return snapshotDomain.getSolrSchema().getContent();
    });
    assertTrue("must have execute", executed.get());
}