Examples with DataxReader com.qlangtech.tis.datax.impl.DataxReader used on opensource projects

Example 11 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project plugins by qlangtech.

the class TestDataXDorisWriter method testDescriptorsJSONGenerate.

public void testDescriptorsJSONGenerate() {
    DataxReader dataxReader = EasyMock.createMock("dataxReader", DataxReader.class);
    List<ISelectedTab> selectedTabs = TestSelectedTabs.createSelectedTabs(1).stream().map((t) -> t).collect(Collectors.toList());
    for (ISelectedTab tab : selectedTabs) {
        for (ISelectedTab.ColMeta cm : tab.getCols()) {
            cm.setType(DataXReaderColType.STRING.dataType);
        }
    }
    // EasyMock.expect(dataxReader.getSelectedTabs()).andReturn(selectedTabs).anyTimes();
    DataxReader.dataxReaderThreadLocal.set(dataxReader);
    EasyMock.replay(dataxReader);
    DataXDorisWriter writer = new DataXDorisWriter();
    DescriptorsJSON descJson = new DescriptorsJSON(writer.getDescriptor());
    JsonUtil.assertJSONEqual(DataXDorisWriter.class, "doris-datax-writer-descriptor.json", descJson.getDescriptorsJSON(), (m, e, a) -> {
        assertEquals(m, e, a);
    });
    JsonUtil.assertJSONEqual(DataXDorisWriter.class, "doris-datax-writer-descriptor.json", descJson.getDescriptorsJSON(), (m, e, a) -> {
        assertEquals(m, e, a);
    });
    EasyMock.verify(dataxReader);
}

Example 12 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project plugins by qlangtech.

the class TestDataXStarRocksWriter method testDescriptorsJSONGenerate.

public void testDescriptorsJSONGenerate() {
    DataxReader dataxReader = EasyMock.createMock("dataxReader", DataxReader.class);
    List<ISelectedTab> selectedTabs = TestSelectedTabs.createSelectedTabs(1).stream().map((t) -> t).collect(Collectors.toList());
    for (ISelectedTab tab : selectedTabs) {
        for (ISelectedTab.ColMeta cm : tab.getCols()) {
            cm.setType(DataXReaderColType.STRING.dataType);
        }
    }
    // EasyMock.expect(dataxReader.getSelectedTabs()).andReturn(selectedTabs).anyTimes();
    DataxReader.dataxReaderThreadLocal.set(dataxReader);
    EasyMock.replay(dataxReader);
    DataXStarRocksWriter writer = new DataXStarRocksWriter();
    DescriptorsJSON descJson = new DescriptorsJSON(writer.getDescriptor());
    JsonUtil.assertJSONEqual(DataXStarRocksWriter.class, "starrocks-datax-writer-descriptor.json", descJson.getDescriptorsJSON(), (m, e, a) -> {
        assertEquals(m, e, a);
    });
    JsonUtil.assertJSONEqual(DataXStarRocksWriter.class, "starrocks-datax-writer-descriptor.json", descJson.getDescriptorsJSON(), (m, e, a) -> {
        assertEquals(m, e, a);
    });
    EasyMock.verify(dataxReader);
}

Example 13 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project plugins by qlangtech.

the class TestDataXElasticsearchWriter method testTemplateGenerate.

public void testTemplateGenerate() throws Exception {
    DataxReader dataxReader = EasyMock.createMock("dataxReader", DataxReader.class);
    // List<ISelectedTab> selectedTabs = createSelectedTabs(dataxReader);
    // EasyMock.expect(dataxReader.getSelectedTabs()).andReturn(selectedTabs);
    final TestAliyunToken token = new TestAliyunToken("xxxxxxxxxxx", "accessKeykkkkkkkkkkkkkk");
    DataXElasticsearchWriter dataXWriter = new DataXElasticsearchWriter() {

        @Override
        public IHttpToken getToken() {
            return token;
        }

        @Override
        public String getTemplate() {
            return DataXElasticsearchWriter.getDftTemplate();
        }

        @Override
        public Class<?> getOwnerClass() {
            return DataXElasticsearchWriter.class;
        }
    };
    EasyMock.replay(dataxReader);
    dataXWriter.endpoint = "aliyun-bj-endpoint";
    dataXWriter.alias = "application2";
    dataXWriter.index = "application";
    dataXWriter.type = "specific_type";
    dataXWriter.cleanup = true;
    dataXWriter.batchSize = 9999;
    dataXWriter.trySize = 22;
    dataXWriter.timeout = 6666;
    dataXWriter.discovery = true;
    dataXWriter.compression = true;
    dataXWriter.multiThread = true;
    dataXWriter.ignoreParseError = true;
    dataXWriter.aliasMode = "append";
    dataXWriter.settings = "{\"index\" :{\"number_of_shards\": 1, \"number_of_replicas\": 0}}";
    dataXWriter.splitter = ",";
    dataXWriter.dynamic = true;
    ESTableAlias tableMap = new ESTableAlias();
    String esSchema = IOUtils.loadResourceFromClasspath(DataXElasticsearchWriter.class, "es-schema-content.json");
    tableMap.setSchemaContent(esSchema);
    WriterTemplate.valiateCfgGenerate("es-datax-writer-assert.json", dataXWriter, tableMap);
    token.accessKeyId = null;
    token.sccessKeySecret = null;
    WriterTemplate.valiateCfgGenerate("es-datax-writer-assert-without-option.json", dataXWriter, tableMap);
    EasyMock.verify(dataxReader);
}

Example 14 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project plugins by qlangtech.

the class TestDataxMySQLReader method testUpdateDataxReader.

/**
 * 这个单元测试很重要
 *
 * @throws Exception
 */
public void testUpdateDataxReader() throws Exception {
    KeyedPluginStore<DataxReader> readerStore = DataxReader.getPluginStore(null, dataXName);
    final SuFormProperties props = EasyMock.createMock("subformProp", SuFormProperties.class);
    EasyMock.expect(props.getSubFormFieldName()).andReturn("selectedTabs").anyTimes();
    EasyMock.replay(props);
    DataxReader.SubFieldFormAppKey<DataxReader> subFieldKey = new DataxReader.SubFieldFormAppKey<>(null, false, dataXName, props, DataxReader.class);
    KeyedPluginStore<DataxReader> subFieldStore = KeyedPluginStore.getPluginStore(subFieldKey);
    File targetFile = readerStore.getTargetFile();
    FileUtils.deleteQuietly(targetFile);
    FileUtils.deleteQuietly(subFieldStore.getTargetFile());
    DataxMySQLReader dataxReader = (DataxMySQLReader) readerStore.getPlugin();
    assertNull("targetFile:" + targetFile.getAbsolutePath(), dataxReader);
    List<Descriptor.ParseDescribable<DataxReader>> dlist = Lists.newArrayList();
    dataxReader = new DataxMySQLReader();
    dataxReader.dbName = dbName;
    dataxReader.fetchSize = 1999;
    dataxReader.template = DataxMySQLReader.getDftTemplate();
    dataxReader.dataXName = dataXName;
    dlist.add(new Descriptor.ParseDescribable<>(dataxReader));
    readerStore.setPlugins(null, Optional.empty(), dlist);
    dataxReader = (DataxMySQLReader) readerStore.getPlugin();
    assertNotNull(dataxReader);
    assertTrue("dataxReader.getSelectedTabs() must be empty", CollectionUtils.isEmpty(dataxReader.getSelectedTabs()));
    dlist = Lists.newArrayList();
    DataxMySQLReader subformReader = new DataxMySQLReader();
    SelectedTab tabOrder = new SelectedTab();
    tabOrder.name = TestSelectedTabs.tabNameOrderDetail;
    tabOrder.setCols(TestSelectedTabs.tabColsMetaOrderDetail.stream().map((c) -> c.getName()).collect(Collectors.toList()));
    subformReader.setSelectedTabs(Collections.singletonList(tabOrder));
    dlist.add(new Descriptor.ParseDescribable(subformReader));
    subFieldStore.setPlugins(null, Optional.empty(), dlist);
    assertFalse("must not dirty", dataxReader.isDirty());
    // 这里有点像变魔术，这是通过 DataXReader中addPluginsUpdateListener添加的回调函数实现的
    assertTrue(dataxReader.getSelectedTabs().size() > 0);
    dataxReader.getSelectedTabs().forEach((t) -> {
        t.getCols().forEach((c) -> {
            assertNotNull("col:" + c.getName(), c.isPk());
            assertNotNull("col:" + c.getName(), c.getType());
        });
    });
    // 子form中再加一个tab
    SelectedTab tabTotalpayinfo = new SelectedTab();
    tabTotalpayinfo.name = TestSelectedTabs.tabNameTotalpayinfo;
    tabTotalpayinfo.setCols(TestSelectedTabs.tabColsMetaTotalpayinfo.stream().map((c) -> c.getName()).collect(Collectors.toList()));
    subformReader.setSelectedTabs(Lists.newArrayList(tabOrder, tabTotalpayinfo));
    dlist = Lists.newArrayList();
    dlist.add(new Descriptor.ParseDescribable(subformReader));
    subFieldStore.setPlugins(null, Optional.empty(), dlist);
    assertEquals(2, dataxReader.getSelectedTabs().size());
    dataxReader.getSelectedTabs().forEach((t) -> {
        t.getCols().forEach((c) -> {
            assertNotNull("tab:" + t.getName() + ",col:" + c.getName() + " relevant PK can not be null", c.isPk());
            assertNotNull("tab:" + t.getName() + ",col:" + c.getName() + " relevant Type can not be null", c.getType());
        });
    });
    DataxMySQLReader oldReader = dataxReader;
    dataxReader = new DataxMySQLReader();
    dataxReader.dbName = dbName;
    dataxReader.fetchSize = 2999;
    dataxReader.template = DataxMySQLReader.getDftTemplate();
    dataxReader.dataXName = dataXName;
    readerStore.setPlugins(null, Optional.empty(), Collections.singletonList(new Descriptor.ParseDescribable(dataxReader)));
    assertTrue("oldReader must be dirty", oldReader.isDirty());
    assertTrue(dataxReader.getSelectedTabs().size() > 0);
    // 清空测试
    dlist = Lists.newArrayList();
    subformReader = new DataxMySQLReader();
    dlist.add(new Descriptor.ParseDescribable(subformReader));
    subFieldStore.setPlugins(null, Optional.empty(), dlist);
    assertTrue("getSelectedTabs shall be empty", dataxReader.getSelectedTabs().size() < 1);
    EasyMock.verify(props);
}

Example 15 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project plugins by qlangtech.

the class DataxMySQLWriter method generateCreateDDL.

@Override
public StringBuffer generateCreateDDL(IDataxProcessor.TableMap tableMapper) {
    if (!this.autoCreateTable) {
        return null;
    }
    StringBuffer script = new StringBuffer();
    DataxReader threadBingDataXReader = DataxReader.getThreadBingDataXReader();
    Objects.requireNonNull(threadBingDataXReader, "getThreadBingDataXReader can not be null");
    if (threadBingDataXReader instanceof DataxMySQLReader) {
        DataxMySQLReader mySQLReader = (DataxMySQLReader) threadBingDataXReader;
        MySQLDataSourceFactory dsFactory = mySQLReader.getDataSourceFactory();
        dsFactory.visitFirstConnection((conn) -> {
            Statement statement = conn.createStatement();
            ResultSet resultSet = statement.executeQuery("show create table " + tableMapper.getFrom());
            if (!resultSet.next()) {
                throw new IllegalStateException("table:" + tableMapper.getFrom() + " can not exec show create table script");
            }
            String ddl = resultSet.getString(2);
            script.append(ddl);
        });
        return script;
    }
    final CreateTableSqlBuilder createTableSqlBuilder = new CreateTableSqlBuilder(tableMapper) {

        @Override
        protected void appendExtraColDef(List<ColWrapper> pks) {
            if (!pks.isEmpty()) {
                script.append("  PRIMARY KEY (").append(pks.stream().map((pk) -> "`" + pk.getName() + "`").collect(Collectors.joining(","))).append(")").append("\n");
            }
        }

        @Override
        protected void appendTabMeta(List<ColWrapper> pks) {
            script.append(" ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci").append("\n");
        }

        @Override
        protected ColWrapper createColWrapper(ISelectedTab.ColMeta c) {
            return new ColWrapper(c) {

                @Override
                public String getMapperType() {
                    return convertType(this.meta);
                }
            };
        }

        // @Override
        // protected String convertType(ISelectedTab.ColMeta col) {
        // switch (col.getType()) {
        // case Long:
        // return "bigint(20)";
        // case INT:
        // return "int(11)";
        // case Double:
        // return "decimal(18,2)";
        // case Date:
        // return "date";
        // case STRING:
        // case Boolean:
        // case Bytes:
        // default:
        // return "varchar(50)";
        // }
        // }
        /**
         * https://www.runoob.com/mysql/mysql-data-types.html
         * @param col
         * @return
         */
        private String convertType(ISelectedTab.ColMeta col) {
            DataType type = col.getType();
            switch(type.type) {
                case Types.BIT:
                case Types.BOOLEAN:
                case Types.TINYINT:
                    return "TINYINT";
                case Types.SMALLINT:
                    return "SMALLINT";
                case Types.INTEGER:
                    return "int(11)";
                case Types.BIGINT:
                    return "BIGINT(20)";
                case Types.FLOAT:
                    return "FLOAT";
                case Types.DOUBLE:
                    return "DOUBLE";
                case Types.DECIMAL:
                    return "DECIMAL";
                case Types.DATE:
                    return "DATE";
                case Types.TIME:
                    return "TIME";
                case Types.TIMESTAMP:
                    return "TIMESTAMP";
                case Types.BLOB:
                case Types.BINARY:
                case Types.LONGVARBINARY:
                case Types.VARBINARY:
                    return "BLOB";
                case Types.VARCHAR:
                    return "VARCHAR(" + type.columnSize + ")";
                default:
                    return "TINYTEXT";
            }
        }
    };
    return createTableSqlBuilder.build();
}