Search in sources :

Example 6 with DependencyNode

use of com.qlangtech.tis.sql.parser.meta.DependencyNode in project tis by qlangtech.

the class DataFlowAppSource method getDependencyTables.

/**
 * 取得依赖的db->table映射关系
 *
 * @return
 */
// @Override
public Map<DBNode, List<String>> getDependencyTables(IDBTableNamesGetter dbTableNamesGetter) {
    try {
        SqlTaskNodeMeta.SqlDataFlowTopology topology = SqlTaskNodeMeta.getSqlDataFlowTopology(dataflowName);
        Map<DBNode, List<String>> /* tables */
        dbNameMap = Maps.newHashMap();
        List<String> tables = null;
        DBNode dbNode = null;
        for (DependencyNode node : topology.getDumpNodes()) {
            dbNode = new DBNode(node.getDbName(), Integer.parseInt(node.getDbid()));
            node.parseEntityName();
            tables = dbNameMap.get(dbNode);
            if (tables == null) {
                // DB 下的全部table
                tables = Lists.newArrayList();
                dbNameMap.put(dbNode, tables);
            }
            tables.add(node.getName());
        }
        for (Map.Entry<DBNode, List<String>> /* tables */
        entry : dbNameMap.entrySet()) {
            entry.setValue(dbTableNamesGetter.getTableNames(entry.getKey().getDbId(), entry.getValue()));
        }
        return dbNameMap;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
Also used : DBNode(com.qlangtech.tis.sql.parser.DBNode) DependencyNode(com.qlangtech.tis.sql.parser.meta.DependencyNode) SqlTaskNodeMeta(com.qlangtech.tis.sql.parser.SqlTaskNodeMeta)

Example 7 with DependencyNode

use of com.qlangtech.tis.sql.parser.meta.DependencyNode in project tis by qlangtech.

the class TestSqlTaskNodeMeta method testDescrialize.

public void testDescrialize() throws Exception {
    File f = new File(parent, "card_expense_relative.yaml");
    SqlTaskNodeMeta sqlNodeMeta = SqlTaskNodeMeta.deserializeTaskNode(f);
    Assert.assertNotNull(sqlNodeMeta);
    Position pos = sqlNodeMeta.getPosition();
    Assert.assertNotNull(pos);
    Assert.assertTrue(pos.getX() > 0);
    Assert.assertTrue(pos.getY() > 0);
    Assert.assertEquals("card_expense_relative", sqlNodeMeta.getExportName());
    String sqlContent = sqlNodeMeta.getSql();
    System.out.println(sqlContent);
    Assert.assertNotNull(sqlContent);
    List<DependencyNode> required = sqlNodeMeta.getDependencies();
    Assert.assertEquals(2, required.size());
    Assert.assertEquals(1059, sqlNodeMeta.getPosition().getX());
    Assert.assertEquals(264, sqlNodeMeta.getPosition().getY());
    Assert.assertEquals(NodeType.JOINER_SQL, sqlNodeMeta.getNodeType());
    Assert.assertEquals("6e7b9c50-0fba-8a19-f029-d973e5a833c7", sqlNodeMeta.getId());
}
Also used : Position(com.qlangtech.tis.sql.parser.meta.Position) DependencyNode(com.qlangtech.tis.sql.parser.meta.DependencyNode) File(java.io.File)

Example 8 with DependencyNode

use of com.qlangtech.tis.sql.parser.meta.DependencyNode in project tis by qlangtech.

the class CollectionAction method createTopology.

private SqlTaskNodeMeta.SqlDataFlowTopology createTopology(String topologyName, OfflineManager.ProcessedTable dsTable, TargetColumnMeta targetColMetas) throws Exception {
    SqlTaskNodeMeta.SqlDataFlowTopology topology = new SqlTaskNodeMeta.SqlDataFlowTopology();
    SqlTaskNodeMeta.TopologyProfile profile = new SqlTaskNodeMeta.TopologyProfile();
    profile.setName(topologyName);
    profile.setTimestamp(System.currentTimeMillis());
    topology.setProfile(profile);
    DependencyNode dNode = createDumpNode(dsTable);
    topology.addDumpTab(dNode);
    SqlTaskNodeMeta joinNodeMeta = new SqlTaskNodeMeta();
    joinNodeMeta.setId(String.valueOf(UUID.randomUUID()));
    joinNodeMeta.addDependency(dNode);
    joinNodeMeta.setExportName(topologyName);
    joinNodeMeta.setType(NodeType.JOINER_SQL.getType());
    joinNodeMeta.setPosition(DEFAULT_SINGLE_JOINER_POSITION);
    joinNodeMeta.setSql(ColumnMetaData.buildExtractSQL(dsTable.getName(), true, targetColMetas.targetColMetas).toString());
    topology.addNodeMeta(joinNodeMeta);
    ERRules.createErRule(topologyName, createDumpNode(dsTable), targetColMetas.getPKMeta());
    return topology;
}
Also used : DependencyNode(com.qlangtech.tis.sql.parser.meta.DependencyNode) SqlTaskNodeMeta(com.qlangtech.tis.sql.parser.SqlTaskNodeMeta)

Example 9 with DependencyNode

use of com.qlangtech.tis.sql.parser.meta.DependencyNode in project tis by qlangtech.

the class CollectionAction method createDumpNode.

private DependencyNode createDumpNode(OfflineManager.ProcessedTable dsTable) {
    DependencyNode dNode = new DependencyNode();
    dNode.setId(String.valueOf(UUID.randomUUID()));
    dNode.setDbName(dsTable.getDBName());
    dNode.setName(dsTable.getName());
    dNode.setDbid(String.valueOf(dsTable.getDbId()));
    dNode.setTabid(String.valueOf(dsTable.getId()));
    dNode.setExtraSql(dsTable.getExtraSql());
    dNode.setPosition(DEFAULT_SINGLE_TABLE_POSITION);
    dNode.setType(NodeType.DUMP.getType());
    return dNode;
}
Also used : DependencyNode(com.qlangtech.tis.sql.parser.meta.DependencyNode)

Example 10 with DependencyNode

use of com.qlangtech.tis.sql.parser.meta.DependencyNode in project tis by qlangtech.

the class FlatTableRelation method isLinkable.

/**
 * 是否能通过主外键相连
 *
 * @param next
 * @return
 */
public boolean isLinkable(FlatTableRelation next) {
    DependencyNode currForeig = this.getTailer();
    DependencyNode nextPrimary = next.getHeader();
    EntityName currEntityName = currForeig.parseEntityName();
    EntityName nextEntityName = nextPrimary.parseEntityName();
    if (!currEntityName.equals(nextEntityName)) {
        // return false;
        throw new IllegalStateException("curr foreign:" + currEntityName + " next primary:" + nextEntityName + "  must be equal");
    }
    return CollectionUtils.isEqualCollection(this.getTailerKeys().stream().map((r) -> r.getHeadLinkKey()).collect(Collectors.toList()), next.getHeaderKeys().stream().map((r) -> r.getHeadLinkKey()).collect(Collectors.toList()));
}
Also used : EntityName(com.qlangtech.tis.sql.parser.tuple.creator.EntityName) DependencyNode(com.qlangtech.tis.sql.parser.meta.DependencyNode)

Aggregations

DependencyNode (com.qlangtech.tis.sql.parser.meta.DependencyNode)16 SqlTaskNodeMeta (com.qlangtech.tis.sql.parser.SqlTaskNodeMeta)5 EntityName (com.qlangtech.tis.sql.parser.tuple.creator.EntityName)5 TabExtraMeta (com.qlangtech.tis.sql.parser.meta.TabExtraMeta)3 Lists (com.google.common.collect.Lists)2 IDumpTable (com.qlangtech.tis.fullbuild.indexbuild.IDumpTable)2 DumpPhaseStatus (com.qlangtech.tis.fullbuild.phasestatus.impl.DumpPhaseStatus)2 DataflowTask (com.qlangtech.tis.fullbuild.taskflow.DataflowTask)2 File (java.io.File)2 List (java.util.List)2 Map (java.util.Map)2 Objects (java.util.Objects)2 Collectors (java.util.stream.Collectors)2 JSONField (com.alibaba.fastjson.annotation.JSONField)1 Joiner (com.google.common.base.Joiner)1 TIS (com.qlangtech.tis.TIS)1 FullbuildPhase (com.qlangtech.tis.assemble.FullbuildPhase)1 HiveDBUtils (com.qlangtech.tis.dump.hive.HiveDBUtils)1 HiveRemoveHistoryDataTask (com.qlangtech.tis.dump.hive.HiveRemoveHistoryDataTask)1 ExecuteResult (com.qlangtech.tis.exec.ExecuteResult)1