Search in sources :

Example 1 with LocalTableDumpFactory

use of com.qlangtech.tis.dump.LocalTableDumpFactory in project plugins by qlangtech.

the class TestLocalTableDumpAndIndex method testSingleTableDump.

public void testSingleTableDump() throws Exception {
    LocalTableDumpFactory tableDumpFactory = new LocalTableDumpFactory();
    File dumpRoot = LocalTableDumpFactory.getLocalOfflineRootDir();
    tableDumpFactory.name = "test";
    DataSourceFactory mockEmployeesDataSource = MockDataSourceFactory.getMockEmployeesDataSource();
    tableDumpFactory.setDataSourceFactoryGetter((tab) -> {
        return mockEmployeesDataSource;
    });
    ITISCoordinator zkCoordinator = MockZKUtils.createZkMock();
    // search4(.+?)_shard(\d+?)_replica_n(\d+?)
    String mockSolrCore = INDEX_COLLECTION + "_shard1_replica_n1";
    IJoinTaskContext execContext = this.mock("execContext", IJoinTaskContext.class);
    // EntityName targetTableName = EntityName.parse(DB_EMPLOYEES+"."); ctx.getAttribute(IParamContext.KEY_BUILD_TARGET_TABLE_NAME);
    EasyMock.expect(execContext.getAttribute(IParamContext.KEY_BUILD_TARGET_TABLE_NAME)).andReturn(getEmployeeTab()).anyTimes();
    replay();
    int round = 0;
    ArrayDeque<Date> createDates = Queues.newArrayDeque();
    // 一共测试5轮
    Date timestamp = null;
    while (round++ < 5) {
        timestamp = new Date();
        createDates.addLast(timestamp);
        TaskContext taskContext = MockTaskContextUtils.create(timestamp);
        taskContext.setCoordinator(zkCoordinator);
        /**
         * -----------------------------------------------------------
         * 开始执行数据导入流程
         *             -----------------------------------------------------------
         */
        startDump(tableDumpFactory, taskContext);
        /**
         * -----------------------------------------------------------
         * 开始执行索引构建流程
         *             -----------------------------------------------------------
         */
        startIndexBuild(mockSolrCore, execContext, zkCoordinator, MockTaskContextUtils.timeFormatYyyyMMddHHmmss.get().format(timestamp));
        Thread.sleep(1000);
    }
    int index = 0;
    File tableRoot = new File(dumpRoot, DB_EMPLOYEES + "/" + TABLE_EMPLOYEES + "/all");
    assertTrue(tableRoot.exists());
    String[] subTimeStampFiles = tableRoot.list();
    Set<String> timestamps = Sets.newHashSet();
    int maxHistorySave = ITableDumpConstant.MAX_PARTITION_SAVE + 1;
    while (index++ < maxHistorySave) {
        timestamps.add(MockTaskContextUtils.timeFormatYyyyMMddHHmmss.get().format(createDates.pollLast()));
    }
    assertEquals("maxHistorySave", maxHistorySave, subTimeStampFiles.length);
    for (String subFile : subTimeStampFiles) {
        assertTrue("shall contain file:" + new File(tableRoot, subFile), timestamps.contains(subFile));
    // TODO 继续校验文件夹中的内容是否正确
    }
    File indexBuildRoot = new File(dumpRoot, INDEX_COLLECTION + "/all/0/output");
    for (String indexBuildRootSub : indexBuildRoot.list()) {
        assertTrue("shall contain file:" + new File(indexBuildRoot, indexBuildRootSub), timestamps.contains(indexBuildRootSub));
    }
    verifyAll();
}
Also used : ITISCoordinator(com.qlangtech.tis.cloud.ITISCoordinator) DataSourceFactory(com.qlangtech.tis.plugin.ds.DataSourceFactory) MockDataSourceFactory(com.qlangtech.tis.order.dump.task.MockDataSourceFactory) TaskContext(com.qlangtech.tis.fullbuild.indexbuild.TaskContext) IJoinTaskContext(com.qlangtech.tis.order.center.IJoinTaskContext) LocalTableDumpFactory(com.qlangtech.tis.dump.LocalTableDumpFactory) IJoinTaskContext(com.qlangtech.tis.order.center.IJoinTaskContext) File(java.io.File) Date(java.util.Date)

Aggregations

ITISCoordinator (com.qlangtech.tis.cloud.ITISCoordinator)1 LocalTableDumpFactory (com.qlangtech.tis.dump.LocalTableDumpFactory)1 TaskContext (com.qlangtech.tis.fullbuild.indexbuild.TaskContext)1 IJoinTaskContext (com.qlangtech.tis.order.center.IJoinTaskContext)1 MockDataSourceFactory (com.qlangtech.tis.order.dump.task.MockDataSourceFactory)1 DataSourceFactory (com.qlangtech.tis.plugin.ds.DataSourceFactory)1 File (java.io.File)1 Date (java.util.Date)1