Examples with ColumnMetaData com.qlangtech.tis.plugin.ds.ColumnMetaData used on opensource projects

Example 6 with ColumnMetaData

use of com.qlangtech.tis.plugin.ds.ColumnMetaData in project plugins by qlangtech.

the class TestLocalTableDumpAndIndex method startIndexBuild.

public void startIndexBuild(String solrCoreName, IJoinTaskContext execContext, ITISCoordinator zkCoordinator, String timePoint) throws Exception {
    LocalIndexBuilderTriggerFactory builderTriggerFactory = new LocalIndexBuilderTriggerFactory();
    File localOfflineDir = LocalTableDumpFactory.getLocalOfflineRootDir();
    String indexName = ITestDumpCommon.INDEX_COLLECTION;
    String groupNum = "0";
    Integer taskId = 123;
    ITISFileSystem fileSystem = builderTriggerFactory.getFileSystem();
    ImportDataProcessInfo buildParam = new ImportDataProcessInfo(taskId, fileSystem, zkCoordinator);
    buildParam.setIndexName(indexName);
    MockDataSourceFactory employeesDataSource = MockDataSourceFactory.getMockEmployeesDataSource();
    List<ColumnMetaData> eployeeTableMeta = employeesDataSource.getTableMetadata(TABLE_EMPLOYEES);
    String colsLiteria = eployeeTableMeta.stream().map((c) -> c.getKey()).collect(Collectors.joining(","));
    buildParam.setBuildTableTitleItems(colsLiteria);
    SnapshotDomain snapshot = com.qlangtech.tis.manage.common.SnapshotDomainUtils.mockEmployeeSnapshotDomain();
    snapshot.writeResource2fs(fileSystem, buildParam.getCoreName(Integer.parseInt(groupNum)), ConfigFileReader.FILE_SCHEMA);
    snapshot.writeResource2fs(fileSystem, buildParam.getCoreName(Integer.parseInt(groupNum)), ConfigFileReader.FILE_SOLR);
    IRemoteJobTrigger buildJob = builderTriggerFactory.createBuildJob(execContext, timePoint, indexName, groupNum, buildParam);
    buildJob.submitJob();
    /**
     * -----------------------------------------------------------
     * 开始执行索引build
     *         -----------------------------------------------------------
     */
    TestLocalTableDumpAndIndex.waitJobTerminatorAndAssert(buildJob);
    // long hdfsTimeStamp, String hdfsUser, SolrCore core, File indexDir, SolrQueryResponse rsp, String taskId
    indexFlowback2SolrEngineNode(solrCoreName, timePoint, localOfflineDir, taskId);
}

Example 7 with ColumnMetaData

use of com.qlangtech.tis.plugin.ds.ColumnMetaData in project plugins by qlangtech.

the class TestOracleDataSourceFactory method testShowTableInDB.

public void testShowTableInDB() {
    // System.out.println("SELECT NULL AS table_cat,\n       c.owner AS table_schem,\n       c.table_name,\n       c.column_name,\n       c.position AS key_seq,\n       c.constraint_name AS pk_name\nFROM all_cons_columns c, all_constraints k\nWHERE k.constraint_type = 'P'\n  AND k.table_name = :1\n  AND k.owner like :2 escape '/'\n  AND k.constraint_name = c.constraint_name \n  AND k.table_name = c.table_name \n  AND k.owner = c.owner \nORDER BY column_name\n");
    String createDDL = IOUtils.loadResourceFromClasspath(TestOracleDataSourceFactory.class, "create-sql-instancedetail.sql");
    System.out.println(createDDL);
    OracleDataSourceFactory dsFactory = createOracleDataSourceFactory();
    List<String> tablesInDB = dsFactory.getTablesInDB();
    assertTrue(tablesInDB.size() > 1);
    // tablesInDB.forEach((tab) -> System.out.println(tab));
    List<ColumnMetaData> cols = dsFactory.getTableMetadata(StringUtils.upperCase("instancedetail"));
    assertTrue(cols.size() > 0);
    for (ColumnMetaData col : cols) {
        System.out.println(col.getKey() + " " + col.isPk() + " " + col.getType());
    }
}

Example 8 with ColumnMetaData

use of com.qlangtech.tis.plugin.ds.ColumnMetaData in project plugins by qlangtech.

the class DataXClickhouseWriter method generateCreateDDL.

@Override
public StringBuffer generateCreateDDL(IDataxProcessor.TableMap tableMapper) {
    if (!this.autoCreateTable) {
        return null;
    }
    final CreateTableSqlBuilder createTableSqlBuilder = new CreateTableSqlBuilder(tableMapper) {

        @Override
        protected void appendExtraColDef(List<ColWrapper> pks) {
            script.append("   ,`__cc_ck_sign` Int8 DEFAULT 1").append("\n");
        }

        @Override
        protected ColWrapper createColWrapper(ISelectedTab.ColMeta c) {
            return new ColWrapper(c) {

                @Override
                public String getMapperType() {
                    return convertType(this.meta);
                }
            };
        }

        @Override
        protected void appendTabMeta(List<ColWrapper> pk) {
            script.append(" ENGINE = CollapsingMergeTree(__cc_ck_sign)").append("\n");
            if (CollectionUtils.isNotEmpty(pk)) {
                script.append(" ORDER BY ").append(pk.stream().map((p) -> "`" + p.getName() + "`").collect(Collectors.joining(","))).append("\n");
            }
            script.append(" SETTINGS index_granularity = 8192");
        }

        private String convertType(ISelectedTab.ColMeta col) {
            DataType type = col.getType();
            switch(type.type) {
                case Types.INTEGER:
                case Types.TINYINT:
                case Types.SMALLINT:
                    return "Int32";
                case Types.BIGINT:
                    return "Int64";
                case Types.FLOAT:
                    return "Float32";
                case Types.DOUBLE:
                case Types.DECIMAL:
                    return "Float64";
                case Types.DATE:
                    return "Date";
                case Types.TIME:
                case Types.TIMESTAMP:
                    return "DateTime";
                case Types.BIT:
                case Types.BOOLEAN:
                    return "UInt8";
                case Types.BLOB:
                case Types.BINARY:
                case Types.LONGVARBINARY:
                case Types.VARBINARY:
                default:
                    return "String";
            }
        }
    };
    return createTableSqlBuilder.build();
// List<ColumnMetaData> tableMetadata = this.getDataSourceFactory().getTableMetadata(tableMapper.getTo());
// Set<String> pks = tableMetadata.stream().filter((t) -> t.isPk()).map((t) -> t.getName()).collect(Collectors.toSet());
// StringBuffer script = new StringBuffer();
// script.append("CREATE TABLE ").append(tableMapper.getTo()).append("\n");
// script.append("(\n");
// ISelectedTab.ColMeta pk = null;
// int maxColNameLength = 0;
// for (ISelectedTab.ColMeta col : tableMapper.getSourceCols()) {
// int m = StringUtils.length(col.getName());
// if (m > maxColNameLength) {
// maxColNameLength = m;
// }
// }
// maxColNameLength += 4;
// for (ISelectedTab.ColMeta col : tableMapper.getSourceCols()) {
// if (pk == null && col.isPk()) {
// pk = col;
// }
// script.append("    `").append(String.format("%-" + (maxColNameLength) + "s", col.getName() + "`"))
// .append(convert2ClickhouseType(col.getType())).append(",").append("\n");
// }
// script.append("    `__cc_ck_sign` Int8 DEFAULT 1").append("\n");
// script.append(")\n");
// script.append(" ENGINE = CollapsingMergeTree(__cc_ck_sign)").append("\n");
// // Objects.requireNonNull(pk, "pk can not be null");
// if (pk != null) {
// script.append(" ORDER BY `").append(pk.getName()).append("`\n");
// }
// script.append(" SETTINGS index_granularity = 8192");
// CREATE TABLE tis.customer_order_relation
// (
// `customerregister_id` String,
// `waitingorder_id` String,
// `worker_id` String,
// `kind` Int8,
// `create_time` Int64,
// `last_ver` Int8,
// `__cc_ck_sign` Int8 DEFAULT 1
// )
// ENGINE = CollapsingMergeTree(__cc_ck_sign)
// ORDER BY customerregister_id
// SETTINGS index_granularity = 8192
// return script;
}

Example 9 with ColumnMetaData

use of com.qlangtech.tis.plugin.ds.ColumnMetaData in project plugins by qlangtech.

the class TestCassandraDatasourceFactory method testGetTableMetadata.

public void testGetTableMetadata() {
    Set<String> keys = Sets.newHashSet("city", "user_id", "user_name");
    CassandraDatasourceFactory ds = getDS();
    List<ColumnMetaData> colsMeta = ds.getTableMetadata("user_dtl");
    assertEquals(3, colsMeta.size());
    for (ColumnMetaData col : colsMeta) {
        assertTrue(keys.contains(col.getKey()));
    }
}

Example 10 with ColumnMetaData

use of com.qlangtech.tis.plugin.ds.ColumnMetaData in project tis by qlangtech.

the class SqlTaskNode method reflectTableCols.

// 依赖的数据库表，可能有相同的表明，但是来自两个数据库的情况
// private static final Map<String /* tableName */, List<TableTupleCreator>> dumpNodes;
// static {
// try {
// 
// List<TableTupleCreator> tables = null;
// Map<String /* tableName */, List<TableTupleCreator>> builder = Maps.newHashMap();
// 
// File f = new File(
// "D:\\j2ee_solution\\eclipse-java-oxygen-mars-develop\\workspace\\tis-mars\\tis-sql-parser\\src\\main\\resources\\dump_tabs\\dump_tabs.txt");
// LineIterator lineIt = FileUtils.lineIterator(f, "utf8");
// String line = null;
// TableTupleCreator tupleCreator = null;
// 
// EntityName entityName = null;
// while (lineIt.hasNext()) {
// line = lineIt.nextLine();
// 
// entityName = EntityName.parse(line);
// 
// tupleCreator = new TableTupleCreator(line, NodeType.DUMP);
// 
// tables = builder.get(entityName.getTabName());
// if (tables == null) {
// tables = Lists.newArrayList();
// builder.put(entityName.getTabName(), tables);
// }
// 
// tupleCreator.setRealEntityName(entityName);
// tables.add(tupleCreator);
// 
// }
// dumpNodes = Collections.unmodifiableMap(builder);// builder.build();
// } catch (IOException e) {
// throw new RuntimeException(e);
// }
// 
// }
public static List<ColumnMetaData> reflectTableCols(String sql) {
    if (StringUtils.isEmpty(sql)) {
        throw new IllegalArgumentException("param sql can not be null");
    }
    List<ColumnMetaData> result = Lists.newArrayList();
    Query query = parseQuery(sql);
    StreamTransformVisitor v = new StreamTransformVisitor(null);
    query.accept(v, new StackableAstVisitorContext<>(1));
    ColRef colsRef = v.getColsRef();
    int index = 0;
    for (Map.Entry<ColName, IDataTupleCreator> /* colName */
    entry : colsRef.getColRefMap().entrySet()) {
        // int index, String key, int type, boolean pk
        result.add(new ColumnMetaData(index++, StringUtils.lowerCase(entry.getKey().getName()), new DataType(-1), /**
         * 暂时无法取到类型，先用-1占一下位置
         */
        false));
    }
    return result;
}