Examples with ITISFileSystem com.qlangtech.tis.fs.ITISFileSystem used on opensource projects

Example 6 with ITISFileSystem

use of com.qlangtech.tis.fs.ITISFileSystem in project tis by qlangtech.

the class AbstractIndexBuildJob method startBuildIndex.

/**
 * 执行单组build任務
 *
 * @return
 * @throws Exception
 */
@SuppressWarnings("all")
public final BuildResult startBuildIndex() throws Exception {
    // final String coreName = state.getIndexName() + '-' + groupNum;
    // + '-' + groupNum;
    final String coreName = state.getCoreName(groupNum);
    final String timePoint = state.getTimepoint();
    final DumpJobStatus status = new DumpJobStatus();
    // status.setUserName(userName);
    status.setTimepoint(state.getTimepoint());
    status.setDumpType("remote");
    DumpJobId dumpJobId = new DumpJobId("jtIdentifier", jobid++);
    status.setDumpJobID(dumpJobId);
    status.setCoreName(coreName);
    RunEnvironment runtime = RunEnvironment.getSysRuntime();
    long now = System.currentTimeMillis();
    final String outPath = state.getIndexBuildOutputPath((this.groupNum));
    logger.info("build out path:" + outPath);
    ITISFileSystem fileSystem = indexBuildFS;
    appDomain.writeResource2fs(fileSystem, coreName, ConfigFileReader.FILE_SCHEMA);
    appDomain.writeResource2fs(fileSystem, coreName, ConfigFileReader.FILE_SOLR);
    // writeResource2Hdfs(coreName, domain, ConfigFileReader.FILE_CORE_PROPERTIES, "config");
    // // TODO 为了兼容老的索引先加上，到时候要删除掉的
    // writeResource2Hdfs(coreName, domain, ConfigFileReader.FILE_SCHEMA, SCHEMA);
    // writeResource2Hdfs(coreName, domain, ConfigFileReader.FILE_APPLICATION, "app");
    // writeResource2Hdfs(coreName, domain, ConfigFileReader.FILE_CORE_PROPERTIES, "core");
    // TODO 为了兼容老的索引先加上，到时候要删除掉的 end
    logger.info("Excute  RemoteDumpJob: Sbumit Remote Job .....  ");
    status.setStartTime(now);
    // String[] core = this.coreName.split("-");
    String serviceName = state.getIndexName();
    // ///////////////////////////////////////////
    logger.info("Excute Remote Dump Job Status: Sbumit  ");
    return buildSliceIndex(coreName, timePoint, status, outPath, serviceName);
}

Example 7 with ITISFileSystem

use of com.qlangtech.tis.fs.ITISFileSystem in project plugins by qlangtech.

the class TestHdfsFileSystemFactory method testCreate.

public void testCreate() {
    PluginStore pluginStore = TIS.getPluginStore(FileSystemFactory.class);
    assertNotNull(pluginStore);
    Describable<FileSystemFactory> plugin = pluginStore.getPlugin();
    assertNotNull(plugin);
    FileSystemFactory fsFactory = (FileSystemFactory) plugin;
    ITISFileSystem fileSystem = fsFactory.getFileSystem();
    List<IPathInfo> paths = fileSystem.listChildren(fileSystem.getPath("/"));
    for (IPathInfo i : paths) {
        System.out.println(i.getName());
    }
// plugin.
// 
// assertTrue("real class:" + plugin.getClass().getName(), plugin instanceof HdfsFileSystemFactory);
// HdfsFileSystemFactory fsFactory = (HdfsFileSystemFactory) plugin;
// ITISFileSystem fileSystem = fsFactory.getFileSystem();
// List<IPathInfo> paths = fileSystem.listChildren(fileSystem.getPath(fsFactory.getRootDir() + "/"));
// for (IPathInfo i : paths) {
// System.out.println(i.getName());
// }
}

Example 8 with ITISFileSystem

use of com.qlangtech.tis.fs.ITISFileSystem in project plugins by qlangtech.

the class RemoveJoinHistoryDataTask method deleteHistoryJoinTable.

/**
 * 删除宽表历史数据
 *
 * @param dumpTable
 * @throws Exception
 */
public static void deleteHistoryJoinTable(EntityName dumpTable, ITISFileSystem fileSys, Integer partitionRetainNum) throws Exception {
    final String path = FSHistoryFileUtils.getJoinTableStorePath(fileSys.getRootDir(), dumpTable).replaceAll("\\.", Path.SEPARATOR);
    if (fileSys == null) {
        throw new IllegalStateException("fileSys can not be null");
    }
    ITISFileSystem fs = fileSys;
    // new Path(hdfsPath);
    IPath parent = fs.getPath(path);
    if (!fs.exists(parent)) {
        return;
    }
    List<IPathInfo> child = fs.listChildren(parent);
    FSHistoryFileUtils.PathInfo pathinfo;
    List<PathInfo> timestampList = new ArrayList<>();
    Matcher matcher;
    for (IPathInfo c : child) {
        matcher = ITISFileSystem.DATE_PATTERN.matcher(c.getPath().getName());
        if (matcher.find()) {
            pathinfo = new PathInfo();
            pathinfo.setPathName(c.getPath().getName());
            pathinfo.setTimeStamp(Long.parseLong(matcher.group()));
            timestampList.add(pathinfo);
        }
    }
    FSHistoryFileUtils.deleteOldHdfsfile(fs, parent, timestampList, partitionRetainNum);
}

Example 9 with ITISFileSystem

use of com.qlangtech.tis.fs.ITISFileSystem in project plugins by qlangtech.

the class TestHiveRemoveHistoryDataTask method testDropHistoryHiveTable.

public void testDropHistoryHiveTable() throws Exception {
    String dbName = "testdb";
    String tabName = "order";
    ITISFileSystem fileSystem = EasyMock.createMock("fileSystem", ITISFileSystem.class);
    Connection hiveConn = EasyMock.createMock("hiveConn", Connection.class);
    Statement showDBStatment = EasyMock.createMock("showDBStatment", Statement.class);
    ResultSet resultSet = EasyMock.createMock("resultSet", ResultSet.class);
    EasyMock.expect(showDBStatment.executeQuery("show databases")).andReturn(resultSet);
    EasyMock.expect(resultSet.next()).andReturn(true);
    EasyMock.expect(resultSet.getString(1)).andReturn(dbName);
    EasyMock.expect(resultSet.next()).andReturn(false);
    EasyMock.expect(hiveConn.createStatement()).andReturn(showDBStatment);
    showDBStatment.close();
    resultSet.close();
    // 查询表
    Statement showTabsStatement = EasyMock.createMock("showTabsStatement", Statement.class);
    EasyMock.expect(hiveConn.createStatement()).andReturn(showTabsStatement);
    ResultSet tabsResult = EasyMock.createMock("tabsResult", ResultSet.class);
    EasyMock.expect(tabsResult.next()).andReturn(true);
    EasyMock.expect(tabsResult.getString(1)).andReturn(tabName);
    EasyMock.expect(tabsResult.next()).andReturn(false);
    tabsResult.close();
    EasyMock.expect(showTabsStatement.executeQuery("show tables in testdb")).andReturn(tabsResult);
    showTabsStatement.close();
    String minPt1 = "20210529142016";
    String minPt2 = "20210529152016";
    String retainPt1 = "20210529162016";
    String retainPt2 = "20210529172016";
    Set<String> removePts = Sets.newHashSet(minPt1, minPt2);
    Statement showPartitionsStatement = EasyMock.createMock("showPartitionsStatement", Statement.class);
    ResultSet showPartitionsResult = EasyMock.createMock("showPartitionsResultSet", ResultSet.class);
    EasyMock.expect(hiveConn.createStatement()).andReturn(showPartitionsStatement);
    EasyMock.expect(showPartitionsStatement.executeQuery("show partitions testdb.order")).andReturn(showPartitionsResult);
    EasyMock.expect(showPartitionsResult.next()).andReturn(true);
    EasyMock.expect(showPartitionsResult.getString(1)).andReturn("pt=" + minPt1);
    EasyMock.expect(showPartitionsResult.next()).andReturn(true);
    EasyMock.expect(showPartitionsResult.getString(1)).andReturn("pt=" + minPt2);
    EasyMock.expect(showPartitionsResult.next()).andReturn(true);
    EasyMock.expect(showPartitionsResult.getString(1)).andReturn("pt=" + retainPt1);
    EasyMock.expect(showPartitionsResult.next()).andReturn(true);
    EasyMock.expect(showPartitionsResult.getString(1)).andReturn("pt=" + retainPt2);
    EasyMock.expect(showPartitionsResult.next()).andReturn(false);
    showPartitionsResult.close();
    showPartitionsStatement.close();
    // ====================================
    Statement dropPtStatement2 = EasyMock.createMock("dropStatement_" + minPt2, Statement.class);
    EasyMock.expect(dropPtStatement2.execute("alter table testdb.order drop partition (  pt = '" + minPt2 + "' )")).andReturn(true);
    EasyMock.expect(hiveConn.createStatement()).andReturn(dropPtStatement2);
    dropPtStatement2.close();
    Statement dropPtStatement1 = EasyMock.createMock("dropStatement_" + minPt1, Statement.class);
    EasyMock.expect(dropPtStatement1.execute("alter table testdb.order drop partition (  pt = '" + minPt1 + "' )")).andReturn(true);
    EasyMock.expect(hiveConn.createStatement()).andReturn(dropPtStatement1);
    dropPtStatement1.close();
    int partitionRetainNum = 2;
    EntityName tabOrder = EntityName.create(dbName, tabName);
    // hiveConn.close();
    EasyMock.replay(fileSystem, hiveConn, showDBStatment, resultSet, showTabsStatement, tabsResult, showPartitionsStatement, showPartitionsResult, dropPtStatement1, dropPtStatement2);
    List<FSHistoryFileUtils.PathInfo> deletePts = (new HiveRemoveHistoryDataTask(fileSystem, MREngine.HIVE)).dropHistoryHiveTable(tabOrder, hiveConn, partitionRetainNum);
    assertEquals(2, deletePts.size());
    deletePts.forEach((pt) -> {
        assertTrue("removed partition shall exist:" + pt.getPathName(), removePts.contains(pt.getPathName()));
    });
    EasyMock.verify(fileSystem, hiveConn, showDBStatment, resultSet, showTabsStatement, tabsResult, showPartitionsStatement, showPartitionsResult, dropPtStatement1, dropPtStatement2);
}

Example 10 with ITISFileSystem

use of com.qlangtech.tis.fs.ITISFileSystem in project plugins by qlangtech.

the class LocalIndexBuilderTriggerFactory method createBuildJob.

/**
 * 执行索引触发任务
 *
 * @param timePoint
 * @param indexName
 * @param groupNum
 * @param buildParam
 * @return
 * @throws Exception
 */
@Override
public IRemoteJobTrigger createBuildJob(IJoinTaskContext execContext, String timePoint, String indexName, String groupNum, IIndexBuildParam buildParam) throws Exception {
    String coreName = buildParam.getCoreName(Integer.parseInt(groupNum));
    Map<String, String> params = Maps.newHashMap();
    params.put(IndexBuildParam.INDEXING_BUILD_TABLE_TITLE_ITEMS, buildParam.getBuildTableTitleItems());
    params.put(IndexBuildParam.JOB_TYPE, IndexBuildParam.JOB_TYPE_INDEX_BUILD);
    params.put(IndexBuildParam.INDEXING_MAX_DOC_FAILD_LIMIT, "100");
    ITISFileSystem fs = this.getFileSystem();
    params.put(IndexBuildParam.INDEXING_SOLRCONFIG_PATH, ConfigFileReader.FILE_SOLR.getFsPath(fs, coreName));
    params.put(IndexBuildParam.INDEXING_SCHEMA_PATH, ConfigFileReader.FILE_SCHEMA.getFsPath(fs, coreName));
    params.put(IndexBuildParam.INDEXING_MAX_NUM_SEGMENTS, "1");
    params.put(IndexBuildParam.INDEXING_GROUP_NUM, "0");
    params.put(IndexBuildParam.INDEXING_SERVICE_NAME, indexName);
    // params.put(IndexBuildParam.INDEXING_BUILDER_TRIGGER_FACTORY, "yarn-index-build");
    params.put(IndexBuildParam.INDEXING_INCR_TIME, timePoint);
    // IndexBuildSourcePathCreator pathCreator = buildParam.getHdfsSourcePath();
    // Objects.requireNonNull(pathCreator, "pathCreator can not be null");
    // File dumpRoot = LocalTableDumpFactory.getLocalOfflineRootDir();
    // File tableRoot = new File(dumpRoot, DB_EMPLOYEES + "/" + TABLE_EMPLOYEES + "/all");
    // IJoinTaskContext execContext, String group, ITabPartition ps
    IndexBuildSourcePathCreator indexBuildSourcePathCreator = this.createIndexBuildSourcePathCreator(execContext, () -> timePoint);
    String sourcePath = URLEncoder.encode(indexBuildSourcePathCreator.build(groupNum), TisUTF8.getName());
    params.put(IndexBuildParam.INDEXING_SOURCE_PATH, sourcePath);
    // "/user/admin/search4totalpay/all/0/output/20200525134425"
    params.put(IndexBuildParam.INDEXING_OUTPUT_PATH, ImportDataProcessInfo.createIndexDir(this.getFileSystem(), timePoint, groupNum, indexName, false));
    params.put(IndexBuildParam.INDEXING_CORE_NAME, coreName);
    params.put(IParamContext.KEY_TASK_ID, String.valueOf(buildParam.getTaskId()));
    params.put(IndexBuildParam.INDEXING_ROW_COUNT, "999");
    TaskContext context = TaskContext.create(params);
    context.setCoordinator(buildParam.getCoordinator());
    return LocalTableDumpFactory.triggerTask(context, (rpc) -> {
        IndexBuilderTriggerFactory buildTrigger = LocalIndexBuilderTriggerFactory.this;
        IndexBuildNodeMaster.executeIndexBuild(context, buildTrigger, rpc, false);
    });
}