Examples with DataType com.qlangtech.tis.plugin.ds.DataType used on opensource projects

Example 1 with DataType

use of com.qlangtech.tis.plugin.ds.DataType in project plugins by qlangtech.

the class DataXPostgresqlWriter method generateCreateDDL.

@Override
public StringBuffer generateCreateDDL(IDataxProcessor.TableMap tableMapper) {
    if (!this.autoCreateTable) {
        return null;
    }
    // 多个主键
    boolean multiPk = tableMapper.getSourceCols().stream().filter((col) -> col.isPk()).count() > 1;
    final CreateTableSqlBuilder createTableSqlBuilder = new CreateTableSqlBuilder(tableMapper) {

        @Override
        protected void appendExtraColDef(List<ColWrapper> pks) {
            // }
            if (multiPk) {
                this.script.append(", CONSTRAINT ").append("uk_" + getCreateTableName() + "_unique_" + pks.stream().map((c) -> c.getName()).collect(Collectors.joining("_"))).append(" UNIQUE(").append(pks.stream().map((c) -> c.getName()).collect(Collectors.joining(","))).append(")");
            }
        }

        @Override
        protected char colEscapeChar() {
            return '\"';
        }

        @Override
        protected ColWrapper createColWrapper(ISelectedTab.ColMeta c) {
            return new ColWrapper(c) {

                @Override
                public String getMapperType() {
                    return convertType(this.meta);
                }
            };
        }

        @Override
        protected void appendTabMeta(List<ColWrapper> pks) {
        }

        /**
         * https://www.runoob.com/mysql/mysql-data-types.html
         * @param col
         * @return
         */
        private String convertType(ISelectedTab.ColMeta col) {
            DataType type = col.getType();
            String colType = type.accept(new DataType.TypeVisitor<String>() {

                @Override
                public String longType(DataType type) {
                    return "BIGINT";
                }

                @Override
                public String doubleType(DataType type) {
                    return "FLOAT8";
                }

                @Override
                public String dateType(DataType type) {
                    return "DATE";
                }

                @Override
                public String timestampType(DataType type) {
                    return "TIMESTAMP";
                }

                @Override
                public String bitType(DataType type) {
                    return "BIT";
                }

                @Override
                public String blobType(DataType type) {
                    return "BYTEA";
                }

                @Override
                public String varcharType(DataType type) {
                    return "VARCHAR(" + type.columnSize + ")";
                }

                @Override
                public String intType(DataType type) {
                    return "INTEGER";
                }

                @Override
                public String floatType(DataType type) {
                    return "FLOAT4";
                }

                @Override
                public String decimalType(DataType type) {
                    return "DECIMAL";
                }

                @Override
                public String timeType(DataType type) {
                    return "TIME";
                }

                @Override
                public String tinyIntType(DataType dataType) {
                    return smallIntType(dataType);
                }

                @Override
                public String smallIntType(DataType dataType) {
                    return "SMALLINT";
                }
            });
            return colType + (!multiPk && col.isPk() ? " PRIMARY KEY" : StringUtils.EMPTY);
        }
    };
    return createTableSqlBuilder.build();
}

Example 2 with DataType

use of com.qlangtech.tis.plugin.ds.DataType in project plugins by qlangtech.

the class TestElasticSearchSinkFactory method testCreateSinkFunction.

/**
 * 参考：ElasticsearchSinkTestBase
 *
 * @throws Exception
 */
@Test
public void testCreateSinkFunction() throws Exception {
    String tableName = "totalpayinfo";
    String colEntityId = "entity_id";
    String colNum = "num";
    String colId = "id";
    String colCreateTime = "create_time";
    IDataxProcessor dataxProcessor = mock("dataxProcessor", IDataxProcessor.class);
    IDataxReader dataxReader = mock("dataxReader", IDataxReader.class);
    List<ISelectedTab> selectedTabs = Lists.newArrayList();
    SelectedTab totalpayinfo = mock(tableName, SelectedTab.class);
    EasyMock.expect(totalpayinfo.getName()).andReturn(tableName);
    List<ISelectedTab.ColMeta> cols = Lists.newArrayList();
    ISelectedTab.ColMeta cm = new ISelectedTab.ColMeta();
    cm.setName(colEntityId);
    cm.setType(new DataType(Types.VARCHAR, 6));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colNum);
    cm.setType(new DataType(Types.INTEGER));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colId);
    cm.setType(new DataType(Types.VARCHAR, 32));
    cm.setPk(true);
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colCreateTime);
    cm.setType(new DataType(Types.BIGINT));
    cols.add(cm);
    EasyMock.expect(totalpayinfo.getCols()).andReturn(cols).anyTimes();
    selectedTabs.add(totalpayinfo);
    EasyMock.expect(dataxReader.getSelectedTabs()).andReturn(selectedTabs);
    EasyMock.expect(dataxProcessor.getReader(null)).andReturn(dataxReader);
    DataXElasticsearchWriter dataXWriter = mock("dataXWriter", DataXElasticsearchWriter.class);
    ESTableAlias esTableAlias = new ESTableAlias();
    dataXWriter.initialIndex(esTableAlias);
    EasyMock.expect(dataxProcessor.getWriter(null)).andReturn(dataXWriter);
    Map<String, IDataxProcessor.TableAlias> aliasMap = new HashMap<>();
    IDataxProcessor.TableAlias tab = new IDataxProcessor.TableAlias(tableName);
    aliasMap.put(tableName, tab);
    EasyMock.expect(dataxProcessor.getTabAlias()).andReturn(aliasMap);
    this.replay();
    ElasticSearchSinkFactory clickHouseSinkFactory = new ElasticSearchSinkFactory();
    Map<IDataxProcessor.TableAlias, SinkFunction<DTO>> sinkFuncs = clickHouseSinkFactory.createSinkFunction(dataxProcessor);
    Assert.assertTrue("sinkFuncs must > 0", sinkFuncs.size() > 0);
    // StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // env.setParallelism(1);
    DTO d = new DTO();
    d.setTableName(tableName);
    d.setEventType(DTO.EventType.ADD);
    Map<String, Object> after = Maps.newHashMap();
    after.put(colEntityId, "334556");
    after.put(colNum, "5");
    after.put(colId, "123dsf124325253dsf123");
    after.put(colCreateTime, "20211113115959");
    d.setAfter(after);
    Assert.assertEquals(1, sinkFuncs.size());
    for (Map.Entry<IDataxProcessor.TableAlias, SinkFunction<DTO>> entry : sinkFuncs.entrySet()) {
        // env.fromElements(new DTO[]{d}).addSink(entry.getValue()).name("clickhouse");
        runElasticSearchSinkTest("elasticsearch-sink-test-json-index", entry.getValue());
        break;
    }
    // env.execute("testJob");
    Thread.sleep(5000);
    this.verifyAll();
    Client client = getClient();
}

Example 3 with DataType

use of com.qlangtech.tis.plugin.ds.DataType in project plugins by qlangtech.

the class DataXClickhouseWriter method generateCreateDDL.

@Override
public StringBuffer generateCreateDDL(IDataxProcessor.TableMap tableMapper) {
    if (!this.autoCreateTable) {
        return null;
    }
    final CreateTableSqlBuilder createTableSqlBuilder = new CreateTableSqlBuilder(tableMapper) {

        @Override
        protected void appendExtraColDef(List<ColWrapper> pks) {
            script.append("   ,`__cc_ck_sign` Int8 DEFAULT 1").append("\n");
        }

        @Override
        protected ColWrapper createColWrapper(ISelectedTab.ColMeta c) {
            return new ColWrapper(c) {

                @Override
                public String getMapperType() {
                    return convertType(this.meta);
                }
            };
        }

        @Override
        protected void appendTabMeta(List<ColWrapper> pk) {
            script.append(" ENGINE = CollapsingMergeTree(__cc_ck_sign)").append("\n");
            if (CollectionUtils.isNotEmpty(pk)) {
                script.append(" ORDER BY ").append(pk.stream().map((p) -> "`" + p.getName() + "`").collect(Collectors.joining(","))).append("\n");
            }
            script.append(" SETTINGS index_granularity = 8192");
        }

        private String convertType(ISelectedTab.ColMeta col) {
            DataType type = col.getType();
            switch(type.type) {
                case Types.INTEGER:
                case Types.TINYINT:
                case Types.SMALLINT:
                    return "Int32";
                case Types.BIGINT:
                    return "Int64";
                case Types.FLOAT:
                    return "Float32";
                case Types.DOUBLE:
                case Types.DECIMAL:
                    return "Float64";
                case Types.DATE:
                    return "Date";
                case Types.TIME:
                case Types.TIMESTAMP:
                    return "DateTime";
                case Types.BIT:
                case Types.BOOLEAN:
                    return "UInt8";
                case Types.BLOB:
                case Types.BINARY:
                case Types.LONGVARBINARY:
                case Types.VARBINARY:
                default:
                    return "String";
            }
        }
    };
    return createTableSqlBuilder.build();
// List<ColumnMetaData> tableMetadata = this.getDataSourceFactory().getTableMetadata(tableMapper.getTo());
// Set<String> pks = tableMetadata.stream().filter((t) -> t.isPk()).map((t) -> t.getName()).collect(Collectors.toSet());
// StringBuffer script = new StringBuffer();
// script.append("CREATE TABLE ").append(tableMapper.getTo()).append("\n");
// script.append("(\n");
// ISelectedTab.ColMeta pk = null;
// int maxColNameLength = 0;
// for (ISelectedTab.ColMeta col : tableMapper.getSourceCols()) {
// int m = StringUtils.length(col.getName());
// if (m > maxColNameLength) {
// maxColNameLength = m;
// }
// }
// maxColNameLength += 4;
// for (ISelectedTab.ColMeta col : tableMapper.getSourceCols()) {
// if (pk == null && col.isPk()) {
// pk = col;
// }
// script.append("    `").append(String.format("%-" + (maxColNameLength) + "s", col.getName() + "`"))
// .append(convert2ClickhouseType(col.getType())).append(",").append("\n");
// }
// script.append("    `__cc_ck_sign` Int8 DEFAULT 1").append("\n");
// script.append(")\n");
// script.append(" ENGINE = CollapsingMergeTree(__cc_ck_sign)").append("\n");
// // Objects.requireNonNull(pk, "pk can not be null");
// if (pk != null) {
// script.append(" ORDER BY `").append(pk.getName()).append("`\n");
// }
// script.append(" SETTINGS index_granularity = 8192");
// CREATE TABLE tis.customer_order_relation
// (
// `customerregister_id` String,
// `waitingorder_id` String,
// `worker_id` String,
// `kind` Int8,
// `create_time` Int64,
// `last_ver` Int8,
// `__cc_ck_sign` Int8 DEFAULT 1
// )
// ENGINE = CollapsingMergeTree(__cc_ck_sign)
// ORDER BY customerregister_id
// SETTINGS index_granularity = 8192
// return script;
}

Example 4 with DataType

use of com.qlangtech.tis.plugin.ds.DataType in project tis by qlangtech.

the class SqlTaskNode method reflectTableCols.

// 依赖的数据库表，可能有相同的表明，但是来自两个数据库的情况
// private static final Map<String /* tableName */, List<TableTupleCreator>> dumpNodes;
// static {
// try {
// 
// List<TableTupleCreator> tables = null;
// Map<String /* tableName */, List<TableTupleCreator>> builder = Maps.newHashMap();
// 
// File f = new File(
// "D:\\j2ee_solution\\eclipse-java-oxygen-mars-develop\\workspace\\tis-mars\\tis-sql-parser\\src\\main\\resources\\dump_tabs\\dump_tabs.txt");
// LineIterator lineIt = FileUtils.lineIterator(f, "utf8");
// String line = null;
// TableTupleCreator tupleCreator = null;
// 
// EntityName entityName = null;
// while (lineIt.hasNext()) {
// line = lineIt.nextLine();
// 
// entityName = EntityName.parse(line);
// 
// tupleCreator = new TableTupleCreator(line, NodeType.DUMP);
// 
// tables = builder.get(entityName.getTabName());
// if (tables == null) {
// tables = Lists.newArrayList();
// builder.put(entityName.getTabName(), tables);
// }
// 
// tupleCreator.setRealEntityName(entityName);
// tables.add(tupleCreator);
// 
// }
// dumpNodes = Collections.unmodifiableMap(builder);// builder.build();
// } catch (IOException e) {
// throw new RuntimeException(e);
// }
// 
// }
public static List<ColumnMetaData> reflectTableCols(String sql) {
    if (StringUtils.isEmpty(sql)) {
        throw new IllegalArgumentException("param sql can not be null");
    }
    List<ColumnMetaData> result = Lists.newArrayList();
    Query query = parseQuery(sql);
    StreamTransformVisitor v = new StreamTransformVisitor(null);
    query.accept(v, new StackableAstVisitorContext<>(1));
    ColRef colsRef = v.getColsRef();
    int index = 0;
    for (Map.Entry<ColName, IDataTupleCreator> /* colName */
    entry : colsRef.getColRefMap().entrySet()) {
        // int index, String key, int type, boolean pk
        result.add(new ColumnMetaData(index++, StringUtils.lowerCase(entry.getKey().getName()), new DataType(-1), /**
         * 暂时无法取到类型，先用-1占一下位置
         */
        false));
    }
    return result;
}

Example 5 with DataType

use of com.qlangtech.tis.plugin.ds.DataType in project tis by qlangtech.

the class ESTableAlias method getSourceCols.

@Override
public List<ISelectedTab.ColMeta> getSourceCols() {
    List<ISelectedTab.ColMeta> colsMeta = Lists.newArrayList();
    ISelectedTab.ColMeta colMeta = null;
    JSONArray cols = getSchemaCols();
    JSONObject col = null;
    for (int i = 0; i < cols.size(); i++) {
        col = cols.getJSONObject(i);
        colMeta = new ISelectedTab.ColMeta() {

            @Override
            public DataType getType() {
                // return super.getType();
                throw new UnsupportedOperationException();
            }
        };
        colMeta.setName(col.getString("name"));
        colMeta.setPk(col.getBoolean("pk"));
        colsMeta.add(colMeta);
    }
    return colsMeta;
}