Search in sources :

Example 1 with MockDataSourceFactory

use of com.qlangtech.tis.order.dump.task.MockDataSourceFactory in project plugins by qlangtech.

the class TestLocalTableDumpAndIndex method startIndexBuild.

public void startIndexBuild(String solrCoreName, IJoinTaskContext execContext, ITISCoordinator zkCoordinator, String timePoint) throws Exception {
    LocalIndexBuilderTriggerFactory builderTriggerFactory = new LocalIndexBuilderTriggerFactory();
    File localOfflineDir = LocalTableDumpFactory.getLocalOfflineRootDir();
    String indexName = ITestDumpCommon.INDEX_COLLECTION;
    String groupNum = "0";
    Integer taskId = 123;
    ITISFileSystem fileSystem = builderTriggerFactory.getFileSystem();
    ImportDataProcessInfo buildParam = new ImportDataProcessInfo(taskId, fileSystem, zkCoordinator);
    buildParam.setIndexName(indexName);
    MockDataSourceFactory employeesDataSource = MockDataSourceFactory.getMockEmployeesDataSource();
    List<ColumnMetaData> eployeeTableMeta = employeesDataSource.getTableMetadata(TABLE_EMPLOYEES);
    String colsLiteria = eployeeTableMeta.stream().map((c) -> c.getKey()).collect(Collectors.joining(","));
    buildParam.setBuildTableTitleItems(colsLiteria);
    SnapshotDomain snapshot = com.qlangtech.tis.manage.common.SnapshotDomainUtils.mockEmployeeSnapshotDomain();
    snapshot.writeResource2fs(fileSystem, buildParam.getCoreName(Integer.parseInt(groupNum)), ConfigFileReader.FILE_SCHEMA);
    snapshot.writeResource2fs(fileSystem, buildParam.getCoreName(Integer.parseInt(groupNum)), ConfigFileReader.FILE_SOLR);
    IRemoteJobTrigger buildJob = builderTriggerFactory.createBuildJob(execContext, timePoint, indexName, groupNum, buildParam);
    buildJob.submitJob();
    /**
     * -----------------------------------------------------------
     * 开始执行索引build
     *         -----------------------------------------------------------
     */
    TestLocalTableDumpAndIndex.waitJobTerminatorAndAssert(buildJob);
    // long hdfsTimeStamp, String hdfsUser, SolrCore core, File indexDir, SolrQueryResponse rsp, String taskId
    indexFlowback2SolrEngineNode(solrCoreName, timePoint, localOfflineDir, taskId);
}
Also used : IRemoteJobTrigger(com.qlangtech.tis.fullbuild.indexbuild.IRemoteJobTrigger) TaskContext(com.qlangtech.tis.fullbuild.indexbuild.TaskContext) ConfigFileReader(com.qlangtech.tis.manage.common.ConfigFileReader) Date(java.util.Date) ColumnMetaData(com.qlangtech.tis.plugin.ds.ColumnMetaData) TISTestCase(com.qlangtech.tis.test.TISTestCase) ITableDumpConstant(com.qlangtech.tis.order.dump.task.ITableDumpConstant) ITISCoordinator(com.qlangtech.tis.cloud.ITISCoordinator) ITISFileSystem(com.qlangtech.tis.fs.ITISFileSystem) MockTaskContextUtils(com.qlangtech.tis.fullbuild.indexbuild.MockTaskContextUtils) DataSourceFactory(com.qlangtech.tis.plugin.ds.DataSourceFactory) MockTisCoreAdminHandler(org.apache.solr.handler.admin.MockTisCoreAdminHandler) ITestDumpCommon(com.qlangtech.tis.order.dump.task.ITestDumpCommon) IJoinTaskContext(com.qlangtech.tis.order.center.IJoinTaskContext) MockDataSourceFactory(com.qlangtech.tis.order.dump.task.MockDataSourceFactory) Set(java.util.Set) FileUtils(org.apache.commons.io.FileUtils) IOException(java.io.IOException) EasyMock(org.easymock.EasyMock) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) File(java.io.File) MockZKUtils(com.qlangtech.tis.cloud.MockZKUtils) TimeUnit(java.util.concurrent.TimeUnit) LocalIndexBuilderTriggerFactory(com.qlangtech.tis.indexbuild.LocalIndexBuilderTriggerFactory) CountDownLatch(java.util.concurrent.CountDownLatch) Queues(com.google.common.collect.Queues) LocalTableDumpFactory(com.qlangtech.tis.dump.LocalTableDumpFactory) IParamContext(com.qlangtech.tis.order.center.IParamContext) SolrQueryResponse(org.apache.solr.response.SolrQueryResponse) List(java.util.List) SnapshotDomain(com.qlangtech.tis.manage.common.SnapshotDomain) RunningStatus(com.qlangtech.tis.fullbuild.indexbuild.RunningStatus) ArrayDeque(java.util.ArrayDeque) ImportDataProcessInfo(com.qlangtech.tis.trigger.jst.ImportDataProcessInfo) IRemoteJobTrigger(com.qlangtech.tis.fullbuild.indexbuild.IRemoteJobTrigger) SnapshotDomain(com.qlangtech.tis.manage.common.SnapshotDomain) LocalIndexBuilderTriggerFactory(com.qlangtech.tis.indexbuild.LocalIndexBuilderTriggerFactory) ITISFileSystem(com.qlangtech.tis.fs.ITISFileSystem) ImportDataProcessInfo(com.qlangtech.tis.trigger.jst.ImportDataProcessInfo) MockDataSourceFactory(com.qlangtech.tis.order.dump.task.MockDataSourceFactory) ColumnMetaData(com.qlangtech.tis.plugin.ds.ColumnMetaData) File(java.io.File)

Aggregations

Queues (com.google.common.collect.Queues)1 Sets (com.google.common.collect.Sets)1 ITISCoordinator (com.qlangtech.tis.cloud.ITISCoordinator)1 MockZKUtils (com.qlangtech.tis.cloud.MockZKUtils)1 LocalTableDumpFactory (com.qlangtech.tis.dump.LocalTableDumpFactory)1 ITISFileSystem (com.qlangtech.tis.fs.ITISFileSystem)1 IRemoteJobTrigger (com.qlangtech.tis.fullbuild.indexbuild.IRemoteJobTrigger)1 MockTaskContextUtils (com.qlangtech.tis.fullbuild.indexbuild.MockTaskContextUtils)1 RunningStatus (com.qlangtech.tis.fullbuild.indexbuild.RunningStatus)1 TaskContext (com.qlangtech.tis.fullbuild.indexbuild.TaskContext)1 LocalIndexBuilderTriggerFactory (com.qlangtech.tis.indexbuild.LocalIndexBuilderTriggerFactory)1 ConfigFileReader (com.qlangtech.tis.manage.common.ConfigFileReader)1 SnapshotDomain (com.qlangtech.tis.manage.common.SnapshotDomain)1 IJoinTaskContext (com.qlangtech.tis.order.center.IJoinTaskContext)1 IParamContext (com.qlangtech.tis.order.center.IParamContext)1 ITableDumpConstant (com.qlangtech.tis.order.dump.task.ITableDumpConstant)1 ITestDumpCommon (com.qlangtech.tis.order.dump.task.ITestDumpCommon)1 MockDataSourceFactory (com.qlangtech.tis.order.dump.task.MockDataSourceFactory)1 ColumnMetaData (com.qlangtech.tis.plugin.ds.ColumnMetaData)1 DataSourceFactory (com.qlangtech.tis.plugin.ds.DataSourceFactory)1