Examples with ITISFileSystem com.qlangtech.tis.fs.ITISFileSystem used on opensource projects

Example 11 with ITISFileSystem

use of com.qlangtech.tis.fs.ITISFileSystem in project plugins by qlangtech.

the class HudiDumpPostTask method run.

@Override
public void run() {
    ITISFileSystem fs = this.hudiWriter.getFs().getFileSystem();
    IPath dumpDir = HudiTableMeta.getDumpDir(fs, this.hudiTab.getName(), execContext.getPartitionTimestamp(), this.hiveConnMeta);
    IPath fsSourcePropsPath = fs.getPath(dumpDir, "meta/" + this.hudiTab.getName() + "-source.properties");
    try {
        this.writeSourceProps(fs, dumpDir, fsSourcePropsPath);
        this.launchSparkRddConvert(fs, dumpDir, fsSourcePropsPath);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 12 with ITISFileSystem

use of com.qlangtech.tis.fs.ITISFileSystem in project plugins by qlangtech.

the class HudiTableMeta method getHistoryBatchs.

public static List<Option> getHistoryBatchs(ITISFileSystem fs, IHiveConnGetter hiveConn) {
    IPath path = fs.getPath(fs.getRootDir(), hiveConn.getDbName());
    List<IPathInfo> child = fs.listChildren(path);
    return child.stream().map((c) -> new Option(c.getName())).collect(Collectors.toList());
}

Example 13 with ITISFileSystem

use of com.qlangtech.tis.fs.ITISFileSystem in project plugins by qlangtech.

the class TestDataXHdfsWriter method testdataDump.

// @Test
public void testdataDump() throws Exception {
    // final DataxWriter dataxWriter = DataxWriter.load(null, mysql2hdfsDataXName);
    HdfsFileSystemFactory fsFactory = HdfsFileSystemFactoryTestUtils.getFileSystemFactory();
    ITISFileSystem fileSystem = fsFactory.getFileSystem();
    // assertNotNull("fileSystem can not be null", fileSystem);
    // new Path(fsFactory.rootDir
    // , this.cfg.getNecessaryValue(Key.PATH, HdfsWriterErrorCode.REQUIRED_VALUE));
    // 
    // fileSystem.getPath("");
    HdfsPath p = new HdfsPath(fsFactory.rootDir + "/tis/order");
    HdfsPath subWriterPath = new HdfsPath(p, "test");
    try (TISFSDataOutputStream outputStream = fileSystem.create(subWriterPath, true)) {
        org.apache.commons.io.IOUtils.write(IOUtils.loadResourceFromClasspath(DataXHdfsWriter.class, "hdfs-datax-writer-assert-without-option-val.json"), outputStream, TisUTF8.get());
    }
    System.out.println("write file success");
    List<IPathInfo> iPathInfos = fileSystem.listChildren(p);
    for (IPathInfo child : iPathInfos) {
        fileSystem.delete(child.getPath(), true);
    }
    final DataXHdfsWriter hdfsWriter = new DataXHdfsWriter() {

        @Override
        public FileSystemFactory getFs() {
            return fsFactory;
        }

        @Override
        public Class<?> getOwnerClass() {
            return DataXHdfsWriter.class;
        }
    };
    DataxWriter.dataxWriterGetter = (name) -> {
        assertEquals("mysql2hdfs", name);
        return hdfsWriter;
    };
    // IPath path = fileSystem.getPath(fileSystem.getPath(fileSystem.getRootDir()), hdfsRelativePath);
    // System.out.println("clear path:" + path);
    // fileSystem.delete(path, true);
    // 
    WriterTemplate.realExecuteDump("hdfs-datax-writer-assert-without-option-val.json", hdfsWriter);
}

Example 14 with ITISFileSystem

use of com.qlangtech.tis.fs.ITISFileSystem in project tis by qlangtech.

the class MockTest method testMock.

public void testMock() {
    final String fsPath = "/user/admin/test";
    FileSystemFactory indexBuilderFileSystemFactory = EasyMock.createMock("indexBuildFileSystem", FileSystemFactory.class);
    ITISFileSystem tisFileSystem = EasyMock.createMock("tisFileSystem", ITISFileSystem.class);
    EasyMock.expect(tisFileSystem.getName()).andReturn("mocktest");
    IPath path = EasyMock.createMock("mockpath", IPath.class);
    EasyMock.expect(tisFileSystem.getPath(fsPath)).andReturn(path);
    EasyMock.expect(indexBuilderFileSystemFactory.getFileSystem()).andReturn(tisFileSystem);
    EasyMock.replay(indexBuilderFileSystemFactory, tisFileSystem, path);
    ITISFileSystem fs = indexBuilderFileSystemFactory.getFileSystem();
    assertEquals("mocktest", fs.getName());
    assertNotNull(fs.getPath(fsPath));
}