Examples with ERRules com.qlangtech.tis.sql.parser.er.ERRules used on opensource projects

Example 6 with ERRules

use of com.qlangtech.tis.sql.parser.er.ERRules in project tis by qlangtech.

the class TestSqlTaskNodeMeta method testGetRewriteSql.

public void testGetRewriteSql() throws Exception {
    SqlTaskNodeMeta taskNodeMeta = new SqlTaskNodeMeta();
    SqlDataFlowTopology topology = SqlTaskNodeMeta.getSqlDataFlowTopology(TestSupplyGoodsParse.topologyName);
    assertNotNull(topology);
    SqlTaskNodeMeta finalNode = topology.getFinalNode();
    assertNotNull(finalNode);
    taskNodeMeta.setSql(finalNode.getSql());
    Map<IDumpTable, ITabPartition> dumpPartition = Maps.newHashMap();
    String pt = "20200703113848";
    dumpPartition.put(EntityName.parse("scmdb.warehouse_goods"), () -> pt);
    dumpPartition.put(EntityName.parse("tis.stock_info_collapse"), () -> pt);
    dumpPartition.put(EntityName.parse("scmdb.supplier_goods"), () -> pt);
    dumpPartition.put(EntityName.parse("tis.warehouse_collapse"), () -> pt);
    dumpPartition.put(EntityName.parse("tis.supplier_collapse"), () -> pt);
    dumpPartition.put(EntityName.parse("scmdb.goods"), () -> pt);
    dumpPartition.put(EntityName.parse("scmdb.stock_info"), () -> pt);
    dumpPartition.put(EntityName.parse("scmdb.category"), () -> pt);
    dumpPartition.put(EntityName.parse("scmdb.goods_sync_shop"), () -> pt);
    ITemplateContext tplContext = EasyMock.createMock("templateContext", ITemplateContext.class);
    IJoinTaskContext joinTaskContext = EasyMock.createMock("joinTaskContext", IJoinTaskContext.class);
    EasyMock.expect(tplContext.getExecContext()).andReturn(joinTaskContext);
    EasyMock.expect(joinTaskContext.getExecutePhaseRange()).andReturn(ExecutePhaseRange.fullRange()).times(2);
    EasyMock.expect(joinTaskContext.getIndexShardCount()).andReturn(1).times(1);
    Optional<ERRules> erRule = ERRules.getErRule(TestSupplyGoodsParse.topologyName);
    assertTrue(erRule.isPresent());
    EasyMock.replay(tplContext, joinTaskContext);
    ISqlTask.RewriteSql rewriteSql = taskNodeMeta.getRewriteSql("supply_goods", new TabPartitions(dumpPartition), erRule.get(), tplContext, true);
    assertNotNull(rewriteSql);
    assertEquals(TestSqlRewriter.getScriptContent("supply_goods_rewrite_result.txt"), rewriteSql.sqlContent);
    System.out.println(rewriteSql.sqlContent);
    EasyMock.verify(tplContext, joinTaskContext);
}

Example 7 with ERRules

use of com.qlangtech.tis.sql.parser.er.ERRules in project tis by qlangtech.

the class StreamComponentCodeGeneratorFlink method getERRule.

// @Override
private IERRules getERRule() {
    ERRules erRules = new ERRules() {

        public List<PrimaryTableMeta> getPrimaryTabs() {
            TabExtraMeta tabMeta = new TabExtraMeta();
            PrimaryTableMeta ptab = new PrimaryTableMeta("tabName", tabMeta);
            return Collections.singletonList(ptab);
        }
    };
    return erRules;
}

Example 8 with ERRules

use of com.qlangtech.tis.sql.parser.er.ERRules in project tis by qlangtech.

the class TestSqlRewriter method createMockErRules.

public static ERRules createMockErRules(TableMeta totalpayMeta) {
    ERRules erRules = EasyMock.createMock("erRules", ERRules.class);
    EasyMock.expect(erRules.getTabFieldProcessorMap()).andReturn(Collections.emptyMap());
    EasyMock.expect(erRules.getPrimaryTab(totalpayinfo)).andReturn(Optional.ofNullable(totalpayMeta)).times(1);
    return erRules;
}

Example 9 with ERRules

use of com.qlangtech.tis.sql.parser.er.ERRules in project tis by qlangtech.

the class TestFlatTableRelation method testGetFinalLinkKey.

public void testGetFinalLinkKey() {
    ERRules totalpayErRules = TestERRules.getTotalpayErRules();
    Optional<TableRelation> orderdetailParentRelation = totalpayErRules.getFirstParent("orderdetail");
    assertTrue(orderdetailParentRelation.isPresent());
    Optional<TableRelation> instancedetailParentRelation = totalpayErRules.getFirstParent("instancedetail");
    assertTrue(instancedetailParentRelation.isPresent());
    // orderdetailParentRelation.get().
    String primaryKey = "totalpay_id";
    String entityId = "entity_id";
    TableRelation.FinalLinkKey finalLinkKey = FlatTableRelation.getFinalLinkKey(primaryKey, orderdetailParentRelation.get().getCurrentTableRelation(true));
    // for (TableRelation r : totalpayErRules.getRelationList()) {
    // System.out.println(r);
    // }
    assertNotNull(finalLinkKey);
    assertEquals(primaryKey, finalLinkKey.linkKeyName);
    Optional<TableRelation> servicebillinfoParentRelation = totalpayErRules.getFirstParent("servicebillinfo");
    assertTrue(servicebillinfoParentRelation.isPresent());
    finalLinkKey = FlatTableRelation.getFinalLinkKey(primaryKey, servicebillinfoParentRelation.get().getCurrentTableRelation(true));
    assertNotNull(finalLinkKey);
    assertEquals("servicebill_id", finalLinkKey.linkKeyName);
    finalLinkKey = FlatTableRelation.getFinalLinkKey(entityId, servicebillinfoParentRelation.get().getCurrentTableRelation(true));
    assertNotNull(finalLinkKey);
    assertEquals(entityId, finalLinkKey.linkKeyName);
}

Example 10 with ERRules

use of com.qlangtech.tis.sql.parser.er.ERRules in project tis by qlangtech.

the class OfflineDatasourceAction method doSaveErRule.

/**
 * 保存ER关系
 *
 * @param context
 * @throws Exception
 */
@Func(value = PermissionConstant.DATAFLOW_UPDATE)
public void doSaveErRule(Context context) throws Exception {
    com.alibaba.fastjson.JSONObject j = this.parseJsonPost();
    ERRules erRules = new ERRules();
    final String topology = j.getString("topologyName");
    if (StringUtils.isEmpty(topology)) {
        throw new IllegalArgumentException("param 'topology' can not be empty");
    }
    SqlDataFlowTopology df = SqlTaskNodeMeta.getSqlDataFlowTopology(topology);
    com.alibaba.fastjson.JSONArray edges = j.getJSONArray("edges");
    com.alibaba.fastjson.JSONArray nodes = j.getJSONArray("nodes");
    com.alibaba.fastjson.JSONObject edge = null;
    com.alibaba.fastjson.JSONObject sourceNode = null;
    com.alibaba.fastjson.JSONObject targetNode = null;
    com.alibaba.fastjson.JSONObject linkrule = null;
    com.alibaba.fastjson.JSONArray linkKeyList = null;
    com.alibaba.fastjson.JSONObject link = null;
    com.alibaba.fastjson.JSONObject nodeTuple = null;
    com.alibaba.fastjson.JSONObject node = null;
    com.alibaba.fastjson.JSONObject ermeta = null;
    TableRelation erRelation = null;
    for (int i = 0; i < edges.size(); i++) {
        edge = edges.getJSONObject(i);
        sourceNode = edge.getJSONObject("sourceNode");
        // getTableName(sourceNode);
        targetNode = edge.getJSONObject("targetNode");
        // getTableName(targetNode);
        linkrule = edge.getJSONObject("linkrule");
        if (linkrule == null) {
            throw new IllegalStateException("linkrule can not be null");
        }
        erRelation = $(edge.getString("id"), df, getTableName(targetNode), getTableName(sourceNode), TabCardinality.parse(linkrule.getString("cardinality")));
        linkKeyList = linkrule.getJSONArray("linkKeyList");
        for (int jj = 0; jj < linkKeyList.size(); jj++) {
            link = linkKeyList.getJSONObject(jj);
            erRelation.addJoinerKey(link.getString("parentKey"), link.getString("childKey"));
        }
        erRules.addRelation(erRelation);
    }
    com.alibaba.fastjson.JSONObject nodeMeta = null;
    com.alibaba.fastjson.JSONObject colTransfer = null;
    DependencyNode dumpNode = null;
    com.alibaba.fastjson.JSONArray columnTransferList = null;
    TabExtraMeta tabMeta = null;
    String sharedKey = null;
    for (int index = 0; index < nodes.size(); index++) {
        nodeTuple = nodes.getJSONObject(index);
        node = nodeTuple.getJSONObject("node");
        ermeta = node.getJSONObject("extraMeta");
        dumpNode = new DependencyNode();
        if (ermeta != null) {
            tabMeta = new TabExtraMeta();
            tabMeta.setPrimaryIndexTab(ermeta.getBoolean("primaryIndexTab"));
            if (tabMeta.isPrimaryIndexTab()) {
                sharedKey = ermeta.getString("sharedKey");
                tabMeta.setSharedKey(sharedKey);
                com.alibaba.fastjson.JSONArray primaryIndexColumnNames = ermeta.getJSONArray("primaryIndexColumnNames");
                com.alibaba.fastjson.JSONObject primaryIndex = null;
                List<PrimaryLinkKey> names = Lists.newArrayList();
                PrimaryLinkKey plinkKey = null;
                for (int i = 0; i < primaryIndexColumnNames.size(); i++) {
                    primaryIndex = primaryIndexColumnNames.getJSONObject(i);
                    plinkKey = new PrimaryLinkKey();
                    plinkKey.setName(primaryIndex.getString("name"));
                    plinkKey.setPk(primaryIndex.getBoolean("pk"));
                    names.add(plinkKey);
                }
                tabMeta.setPrimaryIndexColumnNames(names);
            }
            tabMeta.setMonitorTrigger(ermeta.getBoolean("monitorTrigger"));
            tabMeta.setTimeVerColName(null);
            if (tabMeta.isMonitorTrigger()) {
                tabMeta.setTimeVerColName(ermeta.getString("timeVerColName"));
            }
            columnTransferList = ermeta.getJSONArray("columnTransferList");
            for (int i = 0; i < columnTransferList.size(); i++) {
                colTransfer = columnTransferList.getJSONObject(i);
                tabMeta.addColumnTransfer(new ColumnTransfer(colTransfer.getString("colKey"), colTransfer.getString("transfer"), colTransfer.getString("param")));
            }
            dumpNode.setExtraMeta(tabMeta);
        }
        dumpNode.setId(node.getString("id"));
        nodeMeta = node.getJSONObject("nodeMeta");
        dumpNode.setTabid(nodeMeta.getString("tabid"));
        dumpNode.setDbid(nodeMeta.getString("dbid"));
        Position pos = new Position();
        pos.setX(node.getIntValue("x"));
        pos.setY(node.getIntValue("y"));
        dumpNode.setPosition(pos);
        dumpNode.setName(nodeMeta.getString("tabname"));
        // dumpNode.setExtraSql(nodeMeta.getString("sqlcontent"));
        erRules.addDumpNode(dumpNode);
    }
    List<PrimaryTableMeta> primaryTabs = erRules.getPrimaryTabs();
    if (primaryTabs.size() < 1) {
        this.addErrorMessage(context, "还没有定义ER主索引表");
        return;
    }
    List<PrimaryLinkKey> pkNames = null;
    for (PrimaryTableMeta meta : primaryTabs) {
        if (StringUtils.isEmpty(meta.getSharedKey())) {
            this.addErrorMessage(context, "主索引表:" + meta.getTabName() + " 还未定义分区键");
        }
        pkNames = meta.getPrimaryKeyNames();
        if (pkNames.size() < 1) {
            this.addErrorMessage(context, "主索引表:" + meta.getTabName() + " 还未定义主键");
        }
    }
    if (this.hasErrors(context)) {
        return;
    }
    // File parent = new File(SqlTaskNode.parent, topology);
    // FileUtils.forceMkdir(parent);
    // FileUtils.write(new File(parent, ERRules.ER_RULES_FILE_NAME), ERRules.serialize(erRules), getEncode(), false);
    ERRules.write(topology, erRules);
    // System.out.println(j.toJSONString());
    long wfId = df.getProfile().getDataflowId();
    if (wfId < 1) {
        throw new IllegalStateException("topology '" + topology + "' relevant wfid can not be null");
    }
    WorkFlow wf = new WorkFlow();
    wf.setOpTime(new Date());
    WorkFlowCriteria wfCriteria = new WorkFlowCriteria();
    wfCriteria.createCriteria().andIdEqualTo((int) wfId);
    this.getWorkflowDAOFacade().getWorkFlowDAO().updateByExampleSelective(wf, wfCriteria);
}