Examples with SqlDataFlowTopology com.qlangtech.tis.sql.parser.SqlTaskNodeMeta.SqlDataFlowTopology used on opensource projects

Example 6 with SqlDataFlowTopology

use of com.qlangtech.tis.sql.parser.SqlTaskNodeMeta.SqlDataFlowTopology in project tis by qlangtech.

the class TestSqlDataFlowTopology method testDeserializeConsoleHttpRemote.

// view-source:http://10.1.21.134:8080/config/config.ajax?action=fullbuild_workflow_action&event_submit_do_get_workflow_detail=true&workflow_id=45&current_time=1510899645000
public void testDeserializeConsoleHttpRemote() throws Exception {
    AjaxResult<SqlDataFlowTopology> result = null;
    SqlDataFlowTopology topology = null;
    try (InputStream input = this.getClass().getResourceAsStream("SqlDataFlowTopologyHttpResponse.json")) {
        String jContent = IOUtils.toString(input, "utf8");
        JSONObject jObject = JSON.parseObject(jContent);
        SqlDataFlowTopology topology2 = SqlDataFlowTopology.deserialize(jObject.getString(IAjaxResult.KEY_BIZRESULT));
        Assert.assertNotNull(topology2);
        Assert.assertEquals("totalpay", topology2.getName());
        Assert.assertTrue(topology2.getDumpNodes().size() > 0);
        Assert.assertTrue(topology2.getNodeMetas().size() > 0);
    }
    HttpUtils.addMockApply(-1, "mock", "/com/qlangtech/tis/sql/parser/SqlDataFlowTopologyHttpResponse.json");
    // try (InputStream input = this.getClass().getResourceAsStream("SqlDataFlowTopologyHttpResponse.json")) {
    // HttpUtils.mockConnMaker = new MockConnectionMaker() {
    // 
    // @Override
    // public MockHttpURLConnection create(URL url, List<Header> heads, HTTPMethod method, byte[] content) {
    // return new MockHttpURLConnection(input);
    // }
    // };
    // }
    result = HttpUtils.soapRemote("http://mock", SqlDataFlowTopology.class);
    Assert.assertNotNull(result);
    topology = result.getBizresult();
    Assert.assertNotNull(topology);
    Assert.assertEquals("totalpay", topology.getName());
}

Example 7 with SqlDataFlowTopology

use of com.qlangtech.tis.sql.parser.SqlTaskNodeMeta.SqlDataFlowTopology in project tis by qlangtech.

the class TestSqlDataFlowTopology method testDeserialize.

public void testDeserialize() throws Exception {
    // 通过yaml的反序列化方式
    SqlDataFlowTopology topology = SqlTaskNodeMeta.getSqlDataFlowTopology("totalpay");
    // topology.getFinalNode()
    String jsonContent = com.alibaba.fastjson.JSON.toJSONString(topology, SerializerFeature.DisableCircularReferenceDetect, SerializerFeature.PrettyFormat);
    // Assert.assertTrue(topology.getTimestamp() > 0);
    Assert.assertTrue(topology.getDumpNodes().size() > 0);
    Assert.assertTrue(topology.getNodeMetas().size() > 0);
    Assert.assertTrue(StringUtils.isNotBlank(topology.getName()));
    Assert.assertNotNull(jsonContent);
    // System.out.println(jsonContent);
    SqlDataFlowTopology topology2 = SqlDataFlowTopology.deserialize(jsonContent);
    Assert.assertNotNull(topology2);
    Collection<DependencyNode> dumpNodes = topology2.getDumpNodes();
    Assert.assertEquals(topology.getDumpNodes().size(), dumpNodes.size());
    Assert.assertEquals(topology.getName(), topology2.getName());
    List<SqlTaskNodeMeta> sqlNode = topology2.getNodeMetas();
    for (SqlTaskNodeMeta n : sqlNode) {
        Assert.assertTrue(n.getDependencies().size() > 0);
        Assert.assertNotNull(n.getSql());
    }
    Assert.assertEquals(topology.getNodeMetas().size(), sqlNode.size());
    Assert.assertEquals(topology.getTimestamp(), topology2.getTimestamp());
}

Example 8 with SqlDataFlowTopology

use of com.qlangtech.tis.sql.parser.SqlTaskNodeMeta.SqlDataFlowTopology in project tis by qlangtech.

the class OfflineDatasourceAction method doSaveErRule.

/**
 * 保存ER关系
 *
 * @param context
 * @throws Exception
 */
@Func(value = PermissionConstant.DATAFLOW_UPDATE)
public void doSaveErRule(Context context) throws Exception {
    com.alibaba.fastjson.JSONObject j = this.parseJsonPost();
    ERRules erRules = new ERRules();
    final String topology = j.getString("topologyName");
    if (StringUtils.isEmpty(topology)) {
        throw new IllegalArgumentException("param 'topology' can not be empty");
    }
    SqlDataFlowTopology df = SqlTaskNodeMeta.getSqlDataFlowTopology(topology);
    com.alibaba.fastjson.JSONArray edges = j.getJSONArray("edges");
    com.alibaba.fastjson.JSONArray nodes = j.getJSONArray("nodes");
    com.alibaba.fastjson.JSONObject edge = null;
    com.alibaba.fastjson.JSONObject sourceNode = null;
    com.alibaba.fastjson.JSONObject targetNode = null;
    com.alibaba.fastjson.JSONObject linkrule = null;
    com.alibaba.fastjson.JSONArray linkKeyList = null;
    com.alibaba.fastjson.JSONObject link = null;
    com.alibaba.fastjson.JSONObject nodeTuple = null;
    com.alibaba.fastjson.JSONObject node = null;
    com.alibaba.fastjson.JSONObject ermeta = null;
    TableRelation erRelation = null;
    for (int i = 0; i < edges.size(); i++) {
        edge = edges.getJSONObject(i);
        sourceNode = edge.getJSONObject("sourceNode");
        // getTableName(sourceNode);
        targetNode = edge.getJSONObject("targetNode");
        // getTableName(targetNode);
        linkrule = edge.getJSONObject("linkrule");
        if (linkrule == null) {
            throw new IllegalStateException("linkrule can not be null");
        }
        erRelation = $(edge.getString("id"), df, getTableName(targetNode), getTableName(sourceNode), TabCardinality.parse(linkrule.getString("cardinality")));
        linkKeyList = linkrule.getJSONArray("linkKeyList");
        for (int jj = 0; jj < linkKeyList.size(); jj++) {
            link = linkKeyList.getJSONObject(jj);
            erRelation.addJoinerKey(link.getString("parentKey"), link.getString("childKey"));
        }
        erRules.addRelation(erRelation);
    }
    com.alibaba.fastjson.JSONObject nodeMeta = null;
    com.alibaba.fastjson.JSONObject colTransfer = null;
    DependencyNode dumpNode = null;
    com.alibaba.fastjson.JSONArray columnTransferList = null;
    TabExtraMeta tabMeta = null;
    String sharedKey = null;
    for (int index = 0; index < nodes.size(); index++) {
        nodeTuple = nodes.getJSONObject(index);
        node = nodeTuple.getJSONObject("node");
        ermeta = node.getJSONObject("extraMeta");
        dumpNode = new DependencyNode();
        if (ermeta != null) {
            tabMeta = new TabExtraMeta();
            tabMeta.setPrimaryIndexTab(ermeta.getBoolean("primaryIndexTab"));
            if (tabMeta.isPrimaryIndexTab()) {
                sharedKey = ermeta.getString("sharedKey");
                tabMeta.setSharedKey(sharedKey);
                com.alibaba.fastjson.JSONArray primaryIndexColumnNames = ermeta.getJSONArray("primaryIndexColumnNames");
                com.alibaba.fastjson.JSONObject primaryIndex = null;
                List<PrimaryLinkKey> names = Lists.newArrayList();
                PrimaryLinkKey plinkKey = null;
                for (int i = 0; i < primaryIndexColumnNames.size(); i++) {
                    primaryIndex = primaryIndexColumnNames.getJSONObject(i);
                    plinkKey = new PrimaryLinkKey();
                    plinkKey.setName(primaryIndex.getString("name"));
                    plinkKey.setPk(primaryIndex.getBoolean("pk"));
                    names.add(plinkKey);
                }
                tabMeta.setPrimaryIndexColumnNames(names);
            }
            tabMeta.setMonitorTrigger(ermeta.getBoolean("monitorTrigger"));
            tabMeta.setTimeVerColName(null);
            if (tabMeta.isMonitorTrigger()) {
                tabMeta.setTimeVerColName(ermeta.getString("timeVerColName"));
            }
            columnTransferList = ermeta.getJSONArray("columnTransferList");
            for (int i = 0; i < columnTransferList.size(); i++) {
                colTransfer = columnTransferList.getJSONObject(i);
                tabMeta.addColumnTransfer(new ColumnTransfer(colTransfer.getString("colKey"), colTransfer.getString("transfer"), colTransfer.getString("param")));
            }
            dumpNode.setExtraMeta(tabMeta);
        }
        dumpNode.setId(node.getString("id"));
        nodeMeta = node.getJSONObject("nodeMeta");
        dumpNode.setTabid(nodeMeta.getString("tabid"));
        dumpNode.setDbid(nodeMeta.getString("dbid"));
        Position pos = new Position();
        pos.setX(node.getIntValue("x"));
        pos.setY(node.getIntValue("y"));
        dumpNode.setPosition(pos);
        dumpNode.setName(nodeMeta.getString("tabname"));
        // dumpNode.setExtraSql(nodeMeta.getString("sqlcontent"));
        erRules.addDumpNode(dumpNode);
    }
    List<PrimaryTableMeta> primaryTabs = erRules.getPrimaryTabs();
    if (primaryTabs.size() < 1) {
        this.addErrorMessage(context, "还没有定义ER主索引表");
        return;
    }
    List<PrimaryLinkKey> pkNames = null;
    for (PrimaryTableMeta meta : primaryTabs) {
        if (StringUtils.isEmpty(meta.getSharedKey())) {
            this.addErrorMessage(context, "主索引表:" + meta.getTabName() + " 还未定义分区键");
        }
        pkNames = meta.getPrimaryKeyNames();
        if (pkNames.size() < 1) {
            this.addErrorMessage(context, "主索引表:" + meta.getTabName() + " 还未定义主键");
        }
    }
    if (this.hasErrors(context)) {
        return;
    }
    // File parent = new File(SqlTaskNode.parent, topology);
    // FileUtils.forceMkdir(parent);
    // FileUtils.write(new File(parent, ERRules.ER_RULES_FILE_NAME), ERRules.serialize(erRules), getEncode(), false);
    ERRules.write(topology, erRules);
    // System.out.println(j.toJSONString());
    long wfId = df.getProfile().getDataflowId();
    if (wfId < 1) {
        throw new IllegalStateException("topology '" + topology + "' relevant wfid can not be null");
    }
    WorkFlow wf = new WorkFlow();
    wf.setOpTime(new Date());
    WorkFlowCriteria wfCriteria = new WorkFlowCriteria();
    wfCriteria.createCriteria().andIdEqualTo((int) wfId);
    this.getWorkflowDAOFacade().getWorkFlowDAO().updateByExampleSelective(wf, wfCriteria);
}

Example 9 with SqlDataFlowTopology

use of com.qlangtech.tis.sql.parser.SqlTaskNodeMeta.SqlDataFlowTopology in project tis by qlangtech.

the class OfflineDatasourceAction method doGetWorkflowTopologyERRule.

/**
 * 取得已经保存的erRules
 *
 * @param context
 * @throws Exception
 */
public void doGetWorkflowTopologyERRule(Context context) throws Exception {
    final String topology = this.getString("topology");
    if (StringUtils.isEmpty(topology)) {
        throw new IllegalStateException("please set param topology");
    }
    SqlDataFlowTopology wfTopology = SqlTaskNodeMeta.getSqlDataFlowTopology(topology);
    this.setBizResult(context, wfTopology.getDumpNodes());
}