Examples with SnapshotDomain com.qlangtech.tis.manage.common.SnapshotDomain used on opensource projects

Example 1 with SnapshotDomain

use of com.qlangtech.tis.manage.common.SnapshotDomain in project plugins by qlangtech.

the class TestLocalTableDumpAndIndex method startIndexBuild.

public void startIndexBuild(String solrCoreName, IJoinTaskContext execContext, ITISCoordinator zkCoordinator, String timePoint) throws Exception {
    LocalIndexBuilderTriggerFactory builderTriggerFactory = new LocalIndexBuilderTriggerFactory();
    File localOfflineDir = LocalTableDumpFactory.getLocalOfflineRootDir();
    String indexName = ITestDumpCommon.INDEX_COLLECTION;
    String groupNum = "0";
    Integer taskId = 123;
    ITISFileSystem fileSystem = builderTriggerFactory.getFileSystem();
    ImportDataProcessInfo buildParam = new ImportDataProcessInfo(taskId, fileSystem, zkCoordinator);
    buildParam.setIndexName(indexName);
    MockDataSourceFactory employeesDataSource = MockDataSourceFactory.getMockEmployeesDataSource();
    List<ColumnMetaData> eployeeTableMeta = employeesDataSource.getTableMetadata(TABLE_EMPLOYEES);
    String colsLiteria = eployeeTableMeta.stream().map((c) -> c.getKey()).collect(Collectors.joining(","));
    buildParam.setBuildTableTitleItems(colsLiteria);
    SnapshotDomain snapshot = com.qlangtech.tis.manage.common.SnapshotDomainUtils.mockEmployeeSnapshotDomain();
    snapshot.writeResource2fs(fileSystem, buildParam.getCoreName(Integer.parseInt(groupNum)), ConfigFileReader.FILE_SCHEMA);
    snapshot.writeResource2fs(fileSystem, buildParam.getCoreName(Integer.parseInt(groupNum)), ConfigFileReader.FILE_SOLR);
    IRemoteJobTrigger buildJob = builderTriggerFactory.createBuildJob(execContext, timePoint, indexName, groupNum, buildParam);
    buildJob.submitJob();
    /**
     * -----------------------------------------------------------
     * 开始执行索引build
     *         -----------------------------------------------------------
     */
    TestLocalTableDumpAndIndex.waitJobTerminatorAndAssert(buildJob);
    // long hdfsTimeStamp, String hdfsUser, SolrCore core, File indexDir, SolrQueryResponse rsp, String taskId
    indexFlowback2SolrEngineNode(solrCoreName, timePoint, localOfflineDir, taskId);
}

Example 2 with SnapshotDomain

use of com.qlangtech.tis.manage.common.SnapshotDomain in project tis by qlangtech.

the class DownloadServlet method getDownloadResource.

protected DownloadResource getDownloadResource(Matcher matcher) {
    Integer bizid = Integer.parseInt(matcher.group(1));
    Integer appid = Integer.parseInt(matcher.group(2));
    Short groupIndex = Short.parseShort(matcher.group(3));
    // 开发环境 线上 线下。。。。
    Short runtime = Short.parseShort(matcher.group(4));
    final String resourceName = matcher.group(5);
    final ServerGroup group = getServerGroup(appid, groupIndex, runtime, this.getContext().getServerGroupDAO());
    if (group == null) {
        throw new IllegalStateException("appid:" + appid + ",groupIndex:" + groupIndex + " runtime:" + AppDomainInfo.getRunEnvir(runtime) + " can not find a group");
    }
    if (group.getPublishSnapshotId() == null) {
        throw new IllegalStateException("group id:" + group.getGid() + " group have not set publishSnapshotId");
    }
    Application app = this.getContext().getApplicationDAO().selectByPrimaryKey(appid);
    if (bizid.intValue() != app.getDptId()) {
        throw new IllegalArgumentException("bizid.intValue()" + bizid.intValue() + " != app.getBizId()" + app.getDptId());
    }
    final SnapshotDomain snapshot = this.getContext().getSnapshotViewDAO().getView(group.getPublishSnapshotId());
    // Snapshot snapshot, String resourceName
    return new DownloadResource(app, snapshot, resourceName);
}

Example 3 with SnapshotDomain

use of com.qlangtech.tis.manage.common.SnapshotDomain in project tis by qlangtech.

the class ViewPojo method downloadResource.

public static boolean downloadResource(Context context, final AppDomainInfo appDomainInfo, BasicModule module, final Writer writer, ResourcePrep prepare) throws Exception {
    final SnapshotDomain snapshot = getSnapshotDoamin(module, appDomainInfo);
    // final StringWriter writer = new StringWriter();
    final IBuilderContext builderContext = new IBuilderContext() {

        @Override
        public void closeWriter(PrintWriter writer) {
        }

        @Override
        public Writer getOutputStream() throws Exception {
            return writer;
        }

        @Override
        public String getPojoName() {
            return StringUtils.capitalize(StringUtils.substringAfter(appDomainInfo.getAppName(), "search4"));
        }

        @Override
        public byte[] getResourceInputStream() {
            return snapshot.getSolrSchema().getContent();
        }

        @Override
        public String getTargetNameSpace() {
            return "com.qlangtech.tis";
        }
    };
    prepare.prepare(builderContext);
    SolrPojoBuilder builder = new SolrPojoBuilder(builderContext);
    builder.create();
    // return writer;
    return true;
}

Example 4 with SnapshotDomain

use of com.qlangtech.tis.manage.common.SnapshotDomain in project tis by qlangtech.

the class TestSchemaAction method testDoSaveByExpertModel.

/**
 * 测试专家模式Schema保存, 将mall_id类型由string改成test类型,并且去掉了一个动态字段pt_*
 *
 * @throws Exception
 */
public void testDoSaveByExpertModel() throws Exception {
    // event_submit_do_save_by_expert_model: y
    request.setParameter("emethod", "saveByExpertModel");
    request.setParameter("action", "schema_action");
    // request.addHeader("appname", collection);
    setCollection(collection);
    try (InputStream post = this.getClass().getResourceAsStream("schema-action-do-save-by-expert-model.json")) {
        request.setContent(IOUtils.toByteArray(post));
    }
    ActionProxy proxy = getActionProxy();
    String result = proxy.execute();
    assertEquals("SchemaAction_ajax", result);
    AjaxValve.ActionExecResult aResult = showBizResult();
    assertNotNull(aResult);
    assertTrue(aResult.isSuccess());
    // this.verifyAll();
    SchemaAction.CreateSnapshotResult bizResult = (SchemaAction.CreateSnapshotResult) aResult.getBizResult();
    assertNotNull(bizResult);
    Integer newSnapshotId = bizResult.getNewId();
    assertTrue("newSnapshotId can not be null", newSnapshotId > 0);
    SnapshotDomain snapshotView = runContext.getSnapshotViewDAO().getView(newSnapshotId);
    assertNotNull("snapshotView can not be null", snapshotView);
    UploadResource solrSchema = snapshotView.getSolrSchema();
    String mallIdTypeModifiedXml = StringUtils.remove(new String(solrSchema.getContent(), TisUTF8.get()), '\r');
    try (InputStream input = this.getClass().getResourceAsStream("schema-action-do-save-by-expert-model-modify-mallid-type-assert.xml")) {
        assertNotNull(input);
        String mallIdTypeModifiedXmlExpect = StringUtils.trimToEmpty(IOUtils.toString(input, TisUTF8.get()));
        assertEquals(mallIdTypeModifiedXmlExpect, StringUtils.trimToEmpty(mallIdTypeModifiedXml));
    }
// JsonUtil.assertJSONEqual(TestSchemaAction.class, "schema-action-do-save-by-expert-model-modify-mallid-type-assert.xml"
// , mallIdTypeModifiedXml, (msg, e, a) -> {
// assertEquals(msg, e, a);
// });
}

Example 5 with SnapshotDomain

use of com.qlangtech.tis.manage.common.SnapshotDomain in project tis by qlangtech.

the class AppSynAction method doInitAppFromDaily.

// private ITerminatorTriggerBizDalDAOFacade triggerContext;
/**
 * 接收从日常环境中推送上来的配置文件，<br>
 * 当第一次推送的时候线上还不存在索引实例的时候，会自动创建索引实例
 *
 * @param context
 * @throws Exception
 */
@Func(PermissionConstant.APP_ADD)
public void doInitAppFromDaily(Context context) throws Exception {
    String content = null;
    try (InputStream reader = this.getRequest().getInputStream()) {
        content = IOUtils.toString(reader, getEncode());
        if (StringUtils.isEmpty(content)) {
            throw new IllegalArgumentException("upload content can not be null");
        }
    }
    final ConfigPush configPush = (ConfigPush) HttpConfigFileReader.xstream.fromXML(content);
    final String collection = configPush.getCollection();
    ISchemaPluginContext schemaPlugin = SchemaAction.createSchemaPlugin(collection);
    // 校验当前的snapshot 版本是否就是传输上来的snapshot版本
    ServerGroup serverGroup = null;
    if (configPush.getRemoteSnapshotId() != null) {
        serverGroup = this.getServerGroupDAO().load(collection, (short) 0, /* groupIndex */
        RunEnvironment.ONLINE.getId());
        if (serverGroup.getPublishSnapshotId() != (configPush.getRemoteSnapshotId() + 0)) {
            this.addErrorMessage(context, "exist snapshotid:" + serverGroup.getPublishSnapshotId() + " is not equal push snapshotid:" + configPush.getRemoteSnapshotId());
            return;
        }
    }
    // List<UploadResource> resources = configPush.getUploadResources();
    Snapshot snapshot = null;
    SnapshotDomain snapshotDomain = null;
    Application app = null;
    ApplicationCriteria criteria = new ApplicationCriteria();
    criteria.createCriteria().andProjectNameEqualTo(collection);
    List<Application> apps = this.getApplicationDAO().selectByExample(criteria);
    for (Application p : apps) {
        app = p;
        break;
    }
    if (app == null) {
        // 在服务端创建新应用
        app = new Application();
    // Integer newAppid = this.createNewApp(context, configPush);
    // app.setAppId(newAppid);
    }
    String snycDesc = "NEW CREATE";
    serverGroup = this.getServerGroupDAO().load(collection, (short) 0, /* groupIndex */
    RunEnvironment.ONLINE.getId());
    boolean newSnapshot = false;
    if (serverGroup == null || serverGroup.getPublishSnapshotId() == null) {
        snapshot = new Snapshot();
        snapshot.setSnId(-1);
        snapshot.setPreSnId(-1);
        snapshot.setAppId(app.getAppId());
        newSnapshot = true;
    } else {
        snycDesc = "PUSH FROM DAILY";
        snapshotDomain = this.getSnapshotViewDAO().getView(configPush.getRemoteSnapshotId());
        snapshot = snapshotDomain.getSnapshot();
    }
    if (snapshot == null) {
        throw new IllegalStateException("snapshot can not be null,collection:" + collection);
    }
    snapshot.setCreateUserId(0l);
    snapshot.setCreateUserName(configPush.getReception());
    // ///////////////////////////////////
    // 组装新的snapshot
    PropteryGetter pGetter = null;
    for (UploadResource res : configPush.getUploadResources()) {
        pGetter = ConfigFileReader.createPropertyGetter(res.getResourceType());
        // 校验配置是否相等
        if (!newSnapshot) {
            final String md5 = ConfigFileReader.md5file(res.getContent());
            if (StringUtils.equals(md5, pGetter.getMd5CodeValue(snapshotDomain))) {
                this.addErrorMessage(context, "resource " + pGetter.getFileName() + " is newest,shall not be updated");
                return;
            }
        }
        Integer newResId = ResSynManager.createNewResource(context, schemaPlugin, res.getContent(), ConfigFileReader.md5file(res.getContent()), pGetter, this, this);
        snapshot = pGetter.createNewSnapshot(newResId, snapshot);
    }
    serverGroup = new ServerGroup();
    serverGroup.setPublishSnapshotId(SaveFileContentAction.createNewSnapshot(snapshot, snycDesc, this, 0l, configPush.getReception()));
    serverGroup.setUpdateTime(new Date());
    ServerGroupCriteria serverGroupCriteria = new ServerGroupCriteria();
    serverGroupCriteria.createCriteria().andAppIdEqualTo(app.getAppId()).andRuntEnvironmentEqualTo(RunEnvironment.ONLINE.getId()).andGroupIndexEqualTo((short) 0);
    this.getServerGroupDAO().updateByExampleSelective(serverGroup, serverGroupCriteria);
    // /////////////////////////////////////
    this.addActionMessage(context, "synsuccess");
}