Examples with IPathInfo com.qlangtech.tis.fs.IPathInfo used on opensource projects

Example 1 with IPathInfo

use of com.qlangtech.tis.fs.IPathInfo in project plugins by qlangtech.

the class TestHdfsFileSystemFactory method testCreate.

public void testCreate() {
    PluginStore pluginStore = TIS.getPluginStore(FileSystemFactory.class);
    assertNotNull(pluginStore);
    Describable<FileSystemFactory> plugin = pluginStore.getPlugin();
    assertNotNull(plugin);
    FileSystemFactory fsFactory = (FileSystemFactory) plugin;
    ITISFileSystem fileSystem = fsFactory.getFileSystem();
    List<IPathInfo> paths = fileSystem.listChildren(fileSystem.getPath("/"));
    for (IPathInfo i : paths) {
        System.out.println(i.getName());
    }
// plugin.
// 
// assertTrue("real class:" + plugin.getClass().getName(), plugin instanceof HdfsFileSystemFactory);
// HdfsFileSystemFactory fsFactory = (HdfsFileSystemFactory) plugin;
// ITISFileSystem fileSystem = fsFactory.getFileSystem();
// List<IPathInfo> paths = fileSystem.listChildren(fileSystem.getPath(fsFactory.getRootDir() + "/"));
// for (IPathInfo i : paths) {
// System.out.println(i.getName());
// }
}

Example 2 with IPathInfo

use of com.qlangtech.tis.fs.IPathInfo in project plugins by qlangtech.

the class RemoveJoinHistoryDataTask method deleteHistoryJoinTable.

/**
 * 删除宽表历史数据
 *
 * @param dumpTable
 * @throws Exception
 */
public static void deleteHistoryJoinTable(EntityName dumpTable, ITISFileSystem fileSys, Integer partitionRetainNum) throws Exception {
    final String path = FSHistoryFileUtils.getJoinTableStorePath(fileSys.getRootDir(), dumpTable).replaceAll("\\.", Path.SEPARATOR);
    if (fileSys == null) {
        throw new IllegalStateException("fileSys can not be null");
    }
    ITISFileSystem fs = fileSys;
    // new Path(hdfsPath);
    IPath parent = fs.getPath(path);
    if (!fs.exists(parent)) {
        return;
    }
    List<IPathInfo> child = fs.listChildren(parent);
    FSHistoryFileUtils.PathInfo pathinfo;
    List<PathInfo> timestampList = new ArrayList<>();
    Matcher matcher;
    for (IPathInfo c : child) {
        matcher = ITISFileSystem.DATE_PATTERN.matcher(c.getPath().getName());
        if (matcher.find()) {
            pathinfo = new PathInfo();
            pathinfo.setPathName(c.getPath().getName());
            pathinfo.setTimeStamp(Long.parseLong(matcher.group()));
            timestampList.add(pathinfo);
        }
    }
    FSHistoryFileUtils.deleteOldHdfsfile(fs, parent, timestampList, partitionRetainNum);
}

Example 3 with IPathInfo

use of com.qlangtech.tis.fs.IPathInfo in project plugins by qlangtech.

the class HudiTableMeta method getHistoryBatchs.

public static List<Option> getHistoryBatchs(ITISFileSystem fs, IHiveConnGetter hiveConn) {
    IPath path = fs.getPath(fs.getRootDir(), hiveConn.getDbName());
    List<IPathInfo> child = fs.listChildren(path);
    return child.stream().map((c) -> new Option(c.getName())).collect(Collectors.toList());
}

Example 4 with IPathInfo

use of com.qlangtech.tis.fs.IPathInfo in project plugins by qlangtech.

the class TestDataXHdfsWriter method testdataDump.

// @Test
public void testdataDump() throws Exception {
    // final DataxWriter dataxWriter = DataxWriter.load(null, mysql2hdfsDataXName);
    HdfsFileSystemFactory fsFactory = HdfsFileSystemFactoryTestUtils.getFileSystemFactory();
    ITISFileSystem fileSystem = fsFactory.getFileSystem();
    // assertNotNull("fileSystem can not be null", fileSystem);
    // new Path(fsFactory.rootDir
    // , this.cfg.getNecessaryValue(Key.PATH, HdfsWriterErrorCode.REQUIRED_VALUE));
    // 
    // fileSystem.getPath("");
    HdfsPath p = new HdfsPath(fsFactory.rootDir + "/tis/order");
    HdfsPath subWriterPath = new HdfsPath(p, "test");
    try (TISFSDataOutputStream outputStream = fileSystem.create(subWriterPath, true)) {
        org.apache.commons.io.IOUtils.write(IOUtils.loadResourceFromClasspath(DataXHdfsWriter.class, "hdfs-datax-writer-assert-without-option-val.json"), outputStream, TisUTF8.get());
    }
    System.out.println("write file success");
    List<IPathInfo> iPathInfos = fileSystem.listChildren(p);
    for (IPathInfo child : iPathInfos) {
        fileSystem.delete(child.getPath(), true);
    }
    final DataXHdfsWriter hdfsWriter = new DataXHdfsWriter() {

        @Override
        public FileSystemFactory getFs() {
            return fsFactory;
        }

        @Override
        public Class<?> getOwnerClass() {
            return DataXHdfsWriter.class;
        }
    };
    DataxWriter.dataxWriterGetter = (name) -> {
        assertEquals("mysql2hdfs", name);
        return hdfsWriter;
    };
    // IPath path = fileSystem.getPath(fileSystem.getPath(fileSystem.getRootDir()), hdfsRelativePath);
    // System.out.println("clear path:" + path);
    // fileSystem.delete(path, true);
    // 
    WriterTemplate.realExecuteDump("hdfs-datax-writer-assert-without-option-val.json", hdfsWriter);
}