Examples with FileSystemFactory com.qlangtech.tis.offline.FileSystemFactory used on opensource projects

Example 1 with FileSystemFactory

use of com.qlangtech.tis.offline.FileSystemFactory in project plugins by qlangtech.

the class TestDataXHudiWriter method createDataXWriter.

private static HudiTest createDataXWriter(Optional<FileSystemFactory> fsFactory) {
    final DefaultSparkConnGetter sparkConnGetter = new DefaultSparkConnGetter();
    sparkConnGetter.name = "default";
    sparkConnGetter.master = "spark://sparkmaster:7077";
    DataXHudiWriter writer = new DataXHudiWriter() {

        @Override
        public Class<?> getOwnerClass() {
            return DataXHudiWriter.class;
        }

        @Override
        public IHiveConnGetter getHiveConnMeta() {
            return HdfsFileSystemFactoryTestUtils.createHiveConnGetter();
        }

        @Override
        public ISparkConnGetter getSparkConnGetter() {
            return sparkConnGetter;
        }

        @Override
        public FileSystemFactory getFs() {
            return fsFactory.isPresent() ? fsFactory.get() : HdfsFileSystemFactoryTestUtils.getFileSystemFactory();
        }
    };
    writer.template = DataXHudiWriter.getDftTemplate();
    writer.fsName = HdfsFileSystemFactoryTestUtils.FS_NAME;
    writer.setKey(new KeyedPluginStore.Key(null, HdfsFileSystemFactoryTestUtils.testDataXName.getName(), null));
    writer.tabType = HudiWriteTabType.COW.getValue();
    writer.batchOp = BatchOpMode.BULK_INSERT.getValue();
    writer.shuffleParallelism = 3;
    writer.partitionedBy = "pt";
    // writer.batchByteSize = 3456;
    // writer.batchSize = 9527;
    // writer.dbName = dbName;
    writer.writeMode = "insert";
    // writer.autoCreateTable = true;
    // writer.postSql = "drop table @table";
    // writer.preSql = "drop table @table";
    // writer.dataXName = HdfsFileSystemFactoryTestUtils.testDataXName.getName();
    // writer.dbName = dbName;
    // HudiSelectedTab hudiTab = new HudiSelectedTab() {
    // @Override
    // public List<ColMeta> getCols() {
    // return WriterTemplate.createColMetas();
    // }
    // };
    // //hudiTab.partitionPathField = WriterTemplate.kind;
    // hudiTab.recordField = WriterTemplate.customerregisterId;
    // hudiTab.sourceOrderingField = WriterTemplate.lastVer;
    // hudiTab.setWhere("1=1");
    // hudiTab.name = WriterTemplate.TAB_customer_order_relation;
    List<HdfsColMeta> colsMeta = HdfsColMeta.getColsMeta(Configuration.from(IOUtils.loadResourceFromClasspath(writer.getClass(), hudi_datax_writer_assert_without_optional)).getConfiguration(cfgPathParameter));
    HudiSelectedTab tab = new HudiSelectedTab() {

        @Override
        public List<ColMeta> getCols() {
            return colsMeta.stream().map((c) -> {
                ColMeta col = new ColMeta();
                col.setName(c.getName());
                col.setPk(c.pk);
                col.setType(c.type);
                col.setNullable(c.nullable);
                return col;
            }).collect(Collectors.toList());
        }
    };
    tab.name = WriterTemplate.TAB_customer_order_relation;
    tab.partition = new OffPartition();
    tab.sourceOrderingField = "last_ver";
    tab.recordField = "customerregister_id";
    return new HudiTest(writer, WriterTemplate.createCustomer_order_relationTableMap(Optional.of(tab)), tab);
}

Example 2 with FileSystemFactory

use of com.qlangtech.tis.offline.FileSystemFactory in project plugins by qlangtech.

the class TestDataXHudiWriter method testFlinkSqlTableDDLCreate.

@Test
public void testFlinkSqlTableDDLCreate() throws Exception {
    FileSystemFactory fsFactory = EasyMock.createMock("fsFactory", FileSystemFactory.class);
    ITISFileSystem fs = EasyMock.createMock("fileSystem", ITISFileSystem.class);
    // fs.getRootDir()
    String child = "default/customer_order_relation";
    String dataDir = "hudi";
    IPath rootPath = new HdfsPath(HdfsFileSystemFactoryTestUtils.DEFAULT_HDFS_ADDRESS + "/user/admin");
    IPath tabPath = new HdfsPath(rootPath, child);
    IPath hudiDataPath = new HdfsPath(tabPath, dataDir);
    EasyMock.expect(fs.getPath(rootPath, child)).andReturn(tabPath);
    EasyMock.expect(fs.getPath(tabPath, dataDir)).andReturn(hudiDataPath);
    EasyMock.expect(fs.getRootDir()).andReturn(rootPath);
    EasyMock.expect(fsFactory.getFileSystem()).andReturn(fs);
    HudiTest forTest = createDataXWriter(Optional.of(fsFactory));
    DataxProcessor dataXProcessor = EasyMock.mock("dataXProcessor", DataxProcessor.class);
    File dataXCfg = folder.newFile();
    FileUtils.writeStringToFile(dataXCfg, "{job:{content:[{\"writer\":" + IOUtils.loadResourceFromClasspath(this.getClass(), hudi_datax_writer_assert_without_optional) + "}]}}", TisUTF8.get());
    List<File> dataXFiles = Lists.newArrayList(dataXCfg);
    EasyMock.expect(dataXProcessor.getDataxCfgFileNames(null)).andReturn(dataXFiles);
    DataxProcessor.processorGetter = (dataXName) -> {
        Assert.assertEquals(HdfsFileSystemFactoryTestUtils.testDataXName.getName(), dataXName);
        return dataXProcessor;
    };
    EasyMock.replay(dataXProcessor, fsFactory, fs);
    // IStreamTableCreator.IStreamTableMeta
    // streamTableMeta = forTest.writer.getStreamTableMeta(HudiWriter.targetTableName);
    // Assert.assertNotNull("streamTableMeta can not be null", streamTableMeta);
    // streamTableMeta.getColsMeta();
    // System.out.println(streamTableMeta.createFlinkTableDDL());
    // DataXHudiWriter.HudiStreamTemplateData tplData
    // = (DataXHudiWriter.HudiStreamTemplateData) forTest.writer.decorateMergeData(
    // new TestStreamTemplateData(HdfsFileSystemFactoryTestUtils.testDataXName, HudiWriter.targetTableName));
    // 
    // 
    // StringBuffer createTabDdl = tplData.getSinkFlinkTableDDL(HudiWriter.targetTableName);
    // Assert.assertNotNull(createTabDdl);
    // 
    // System.out.println(createTabDdl);
    EasyMock.verify(dataXProcessor, fsFactory, fs);
}

Example 3 with FileSystemFactory

use of com.qlangtech.tis.offline.FileSystemFactory in project plugins by qlangtech.

the class TestHdfsFileSystemFactory method testCreate.

public void testCreate() {
    PluginStore pluginStore = TIS.getPluginStore(FileSystemFactory.class);
    assertNotNull(pluginStore);
    Describable<FileSystemFactory> plugin = pluginStore.getPlugin();
    assertNotNull(plugin);
    FileSystemFactory fsFactory = (FileSystemFactory) plugin;
    ITISFileSystem fileSystem = fsFactory.getFileSystem();
    List<IPathInfo> paths = fileSystem.listChildren(fileSystem.getPath("/"));
    for (IPathInfo i : paths) {
        System.out.println(i.getName());
    }
// plugin.
// 
// assertTrue("real class:" + plugin.getClass().getName(), plugin instanceof HdfsFileSystemFactory);
// HdfsFileSystemFactory fsFactory = (HdfsFileSystemFactory) plugin;
// ITISFileSystem fileSystem = fsFactory.getFileSystem();
// List<IPathInfo> paths = fileSystem.listChildren(fileSystem.getPath(fsFactory.getRootDir() + "/"));
// for (IPathInfo i : paths) {
// System.out.println(i.getName());
// }
}

Example 4 with FileSystemFactory

use of com.qlangtech.tis.offline.FileSystemFactory in project plugins by qlangtech.

the class BasicHdfsWriterJob method init.

@Override
public void init() {
    getPluginJobConf();
    try {
        if (this.tabDumpParentPath == null) {
            Path path = createPath();
            this.path = String.valueOf(path);
            Objects.requireNonNull(this.tabDumpParentPath, "tabDumpParentPath can not be null");
            FileSystemFactory hdfsFactory = writerPlugin.getFs();
            logger.info("config param {}:{}", Key.PATH, hdfsFactory.getFSAddress());
            cfg.set(Key.DEFAULT_FS, hdfsFactory.getFSAddress());
        }
    } catch (Throwable e) {
        throw new JobPropInitializeException("pmodPath initial error", e);
    }
    Objects.requireNonNull(writerPlugin, "writerPlugin can not be null");
    super.init();
}

Example 5 with FileSystemFactory

use of com.qlangtech.tis.offline.FileSystemFactory in project plugins by qlangtech.

the class BasicHdfsWriterJob method createHdfsHelper.

public static HdfsHelper createHdfsHelper(Configuration pluginJobConf, BasicFSWriter hiveWriter) {
    Objects.requireNonNull(pluginJobConf, "pluginJobConf can not be null");
    Objects.requireNonNull(hiveWriter, "hiveWriter can not be null");
    try {
        FileSystemFactory fs = hiveWriter.getFs();
        HdfsHelper hdfsHelper = new HdfsHelper(fs.getFileSystem().unwrap());
        org.apache.hadoop.conf.Configuration cfg = new org.apache.hadoop.conf.Configuration();
        cfg.setClassLoader(TIS.get().getPluginManager().uberClassLoader);
        org.apache.hadoop.mapred.JobConf conf = new JobConf(cfg);
        conf.set(FileSystem.FS_DEFAULT_NAME_KEY, pluginJobConf.getString("defaultFS"));
        conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
        conf.set(JobContext.WORKING_DIR, (fs.getFileSystem().getRootDir().unwrap(Path.class).toString()));
        hdfsHelper.conf = conf;
        return hdfsHelper;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}