Examples with ERRules com.qlangtech.tis.sql.parser.er.ERRules used on opensource projects

Example 1 with ERRules

use of com.qlangtech.tis.sql.parser.er.ERRules in project plugins by qlangtech.

the class TestHiveFlatTableBuilder method testTotalpaySummary.

public void testTotalpaySummary() throws Exception {
    HiveFlatTableBuilder flatTableBuilder = (HiveFlatTableBuilder) flatTableBuilderStore.getPlugin();
    // ISqlTask nodeMeta, boolean isFinalNode
    // , ITemplateContext tplContext, ITaskContext taskContext, IFs2Table fs2Table, IJoinTaskStatus joinTaskStatus
    IFs2Table fs2Table = new MockFs2Table();
    IJoinTaskStatus joinTaskStatus = EasyMock.createMock("joinTaskStatus", IJoinTaskStatus.class);
    joinTaskStatus.setComplete(true);
    joinTaskStatus.createJobStatus(EasyMock.anyInt());
    JobLog jobLog = new JobLog();
    EasyMock.expect(joinTaskStatus.getJoblog(EasyMock.anyInt())).andReturn(jobLog).anyTimes();
    joinTaskStatus.setStart();
    IJoinTaskContext joinTaskContext = EasyMock.createMock("joinTaskContext", IJoinTaskContext.class);
    Map<IDumpTable, ITabPartition> dateParams = Maps.newHashMap();
    EasyMock.expect(joinTaskContext.getAttribute(ExecChainContextUtils.PARTITION_DATA_PARAMS)).andReturn(dateParams).anyTimes();
    Map<String, Boolean> taskWorkStatus = Maps.newHashMap();
    EasyMock.expect(joinTaskContext.getAttribute(AdapterTask.KEY_TASK_WORK_STATUS)).andReturn(taskWorkStatus);
    ERRules erRules = EasyMock.createMock("erRules", ERRules.class);
    EasyMock.expect(joinTaskContext.getAttribute("er_rules")).andReturn(erRules);
    EasyMock.replay(joinTaskStatus, joinTaskContext, erRules);
    MockTemplateContext tplContext = new MockTemplateContext(joinTaskContext);
    flatTableBuilder.startTask((context) -> {
        try (InputStream input = TestHiveFlatTableBuilder.class.getResourceAsStream("groupby_totalpay.sql")) {
            // try (InputStream input = TestHiveFlatTableBuilder.class.getResourceAsStream("totalpay_summary.sql")) {
            ISqlTask sqlTask = new DefaultSqlTask(IOUtils.toString(input, TisUTF8.get()));
            // ISqlTask nodeMeta, boolean isFinalNode
            // , ITemplateContext tplContext, ITaskContext taskContext, //
            // IJoinTaskStatus joinTaskStatus
            DataflowTask joinTask = flatTableBuilder.createTask(sqlTask, true, tplContext, context, joinTaskStatus);
            joinTask.run();
        }
    });
}

Example 2 with ERRules

use of com.qlangtech.tis.sql.parser.er.ERRules in project tis by qlangtech.

the class OfflineDatasourceAction method doUpdateTopology.

private void doUpdateTopology(Context context, TopologyUpdateCallback dbSaver) throws Exception {
    final String content = IOUtils.toString(this.getRequest().getInputStream(), getEncode());
    JSONTokener tokener = new JSONTokener(content);
    JSONObject topology = new JSONObject(tokener);
    final String topologyName = topology.getString("topologyName");
    if (StringUtils.isEmpty(topologyName)) {
        this.addErrorMessage(context, "请填写数据流名称");
        return;
    }
    File parent = new File(SqlTaskNode.parent, topologyName);
    FileUtils.forceMkdir(parent);
    JSONArray nodes = topology.getJSONArray("nodes");
    // 这部分信息暂时先不需要了，已经包含在‘nodes’中了
    JSONArray edges = topology.getJSONArray("edges");
    JSONObject o = null;
    JSONObject nodeMeta = null;
    JSONObject nestNodeMeta = null;
    JSONArray joinDependencies = null;
    JSONObject dep = null;
    NodeType nodetype = null;
    DependencyNode dnode = null;
    SqlTaskNodeMeta pnode = null;
    Position pos = null;
    final SqlDataFlowTopology topologyPojo = new SqlDataFlowTopology();
    SqlTaskNodeMeta.TopologyProfile profile = new SqlTaskNodeMeta.TopologyProfile();
    profile.setName(topologyName);
    profile.setTimestamp(System.currentTimeMillis());
    // profile.setDataflowId(this.getWorkflowId(topologyName));
    topologyPojo.setProfile(profile);
    int x, y;
    int tableid;
    Tab tab;
    for (int i = 0; i < nodes.length(); i++) {
        o = nodes.getJSONObject(i);
        x = o.getInt("x");
        if (x < 0) {
            // 如果在边界外的图形需要跳过`
            continue;
        }
        y = o.getInt("y");
        pos = new Position();
        pos.setX(x);
        pos.setY(y);
        nodeMeta = o.getJSONObject("nodeMeta");
        nestNodeMeta = nodeMeta.getJSONObject("nodeMeta");
        nodetype = NodeType.parse(nestNodeMeta.getString("type"));
        if (nodetype == NodeType.DUMP) {
            dnode = new DependencyNode();
            dnode.setExtraSql(SqlTaskNodeMeta.processBigContent(nodeMeta.getString("sqlcontent")));
            dnode.setId(o.getString("id"));
            tableid = nodeMeta.getInt("tabid");
            Map<Integer, com.qlangtech.tis.workflow.pojo.DatasourceDb> dbMap = Maps.newHashMap();
            tab = getDatabase(this.offlineDAOFacade, dbMap, tableid);
            dnode.setDbName(tab.db.getName());
            dnode.setName(tab.tab.getName());
            dnode.setTabid(String.valueOf(tableid));
            dnode.setDbid(String.valueOf(nodeMeta.get("dbid")));
            dnode.setPosition(pos);
            dnode.setType(NodeType.DUMP.getType());
            topologyPojo.addDumpTab(dnode);
        } else if (nodetype == NodeType.JOINER_SQL) {
            pnode = new SqlTaskNodeMeta();
            pnode.setId(o.getString("id"));
            pnode.setPosition(pos);
            pnode.setSql(SqlTaskNodeMeta.processBigContent(nodeMeta.getString("sql")));
            pnode.setExportName(nodeMeta.getString("exportName"));
            // pnode.setId(String.valueOf(nodeMeta.get("id")));
            // pnode.setId(o.getString("id"));
            pnode.setType(NodeType.JOINER_SQL.getType());
            joinDependencies = nodeMeta.getJSONArray("dependencies");
            for (int k = 0; k < joinDependencies.length(); k++) {
                dep = joinDependencies.getJSONObject(k);
                dnode = new DependencyNode();
                dnode.setId(dep.getString("value"));
                dnode.setName(dep.getString("label"));
                dnode.setType(NodeType.DUMP.getType());
                pnode.addDependency(dnode);
            }
            topologyPojo.addNodeMeta(pnode);
        } else {
            throw new IllegalStateException("nodetype:" + nodetype + " is illegal");
        }
    }
    // 校验一下是否只有一个最终输出节点
    List<SqlTaskNodeMeta> finalNodes = topologyPojo.getFinalNodes();
    if (finalNodes.size() > 1) {
        this.addErrorMessage(context, "最终输出节点(" + finalNodes.stream().map((r) -> r.getExportName()).collect(Collectors.joining(",")) + ")不能多于一个");
        return;
    }
    if (finalNodes.size() < 1) {
    // 这种情况为单表导入，不需要spark和hive的支持
    // this.addErrorMessage(context, "请定义数据处理节点");
    // return;
    }
    Optional<ERRules> erRule = ERRules.getErRule(topologyPojo.getName());
    this.setBizResult(context, new ERRulesStatus(erRule));
    dbSaver.execute(topologyName, topologyPojo);
    // 保存一个时间戳
    SqlTaskNodeMeta.persistence(topologyPojo, parent);
    // 备份之用
    FileUtils.write(new File(parent, topologyName + "_content.json"), content, getEncode(), false);
    this.addActionMessage(context, "'" + topologyName + "'保存成功");
}

Example 3 with ERRules

use of com.qlangtech.tis.sql.parser.er.ERRules in project tis by qlangtech.

the class TestStreamComponentCodeGenerator method testSingleTableCodeGenerator.

/**
 * 测试单表增量脚本生成
 *
 * @throws Exception
 */
public void testSingleTableCodeGenerator() throws Exception {
    // CoreAction.create
    String topologyName = "employees4local";
    String collectionName = "search4employee4local";
    Optional<ERRules> erRule = ERRules.getErRule(topologyName);
    IAppSource appSource = IAppSource.load(null, collectionName);
    assertTrue(appSource instanceof SingleTableAppSource);
    // 测试针对单表的的topology增量脚本生成
    long timestamp = 20191111115959l;
    SqlTaskNodeMeta.SqlDataFlowTopology topology = SqlTaskNodeMeta.getSqlDataFlowTopology(topologyName);
    assertNotNull(topology);
    if (!erRule.isPresent()) {
        ERRules.createDefaultErRule(topology);
    }
    List<FacadeContext> facadeList = Lists.newArrayList();
    StreamComponentCodeGenerator streamCodeGenerator = new StreamComponentCodeGenerator(collectionName, timestamp, facadeList, (IStreamIncrGenerateStrategy) appSource, true);
    // EasyMock.replay(streamIncrGenerateStrategy);
    streamCodeGenerator.build();
    assertGenerateContentEqual(timestamp, collectionName, "S4employee4localListener.scala");
// EasyMock.verify(streamIncrGenerateStrategy);
}

Example 4 with ERRules

use of com.qlangtech.tis.sql.parser.er.ERRules in project tis by qlangtech.

the class TestSqlRewriter method testTotalpaySummaryRewriteByWithoutDefinePrimaryTableShareKey.

/**
 * 执行索引全量构建过程中，测试ERRule没有定义主表的<b>shareKey</b>，会导致final表的分区函数无法正常创建，需要主动抛出一个异常
 */
public void testTotalpaySummaryRewriteByWithoutDefinePrimaryTableShareKey() {
    TestJoinTaskContext taskContext = new TestJoinTaskContext(ExecutePhaseRange.fullRange());
    SqlTaskNodeMeta nodeMeta = getSqlTaskNodeMeta(totalpaySummary);
    ERRules erRules = createMockErRules(new TableMeta(totalpayinfo.getTableName(), null));
    EasyMock.replay(erRules);
    Exception occurException = null;
    try {
        assertFull(totalpaySummary, true, nodeMeta, taskContext, erRules, nodeMeta.getSql());
        fail("shall throw new exception");
    } catch (Exception e) {
        occurException = e;
    }
    assertNotNull("must throw an error", occurException);
    assertEquals(SqlRewriter.ERROR_WithoutDefinePrimaryTableShareKey, ExceptionUtils.getRootCause(occurException).getMessage());
    EasyMock.verify(erRules);
}

Example 5 with ERRules

use of com.qlangtech.tis.sql.parser.er.ERRules in project tis by qlangtech.

the class TestSqlRewriter method testTotalpaySummaryRewriteByWithoutDefinePrimaryTable.

/**
 * 执行索引全量构建过程中，测试ERRule没有定义主表，会导致final表的分区函数无法正常创建，需要主动抛出一个异常
 */
public void testTotalpaySummaryRewriteByWithoutDefinePrimaryTable() {
    TestJoinTaskContext taskContext = new TestJoinTaskContext(ExecutePhaseRange.fullRange());
    SqlTaskNodeMeta nodeMeta = getSqlTaskNodeMeta(totalpaySummary);
    // ERRules erRules = EasyMock.createMock("erRules", ERRules.class);
    // EasyMock.expect(erRules.getTabFieldProcessorMap()).andReturn(Collections.emptyMap());
    // EasyMock.expect(erRules.getPrimaryTab(totalpayinfo))
    // .andReturn(Optional.empty()/** 没有定义主表*/).anyTimes();
    ERRules erRules = createMockErRules(null);
    EasyMock.replay(erRules);
    Exception occurException = null;
    try {
        assertFull(totalpaySummary, true, nodeMeta, taskContext, erRules, nodeMeta.getSql());
        fail("shall throw new exception");
    } catch (Exception e) {
        occurException = e;
    }
    assertNotNull("must throw an error", occurException);
    assertEquals(SqlRewriter.ERROR_WithoutDefinePrimaryTableShareKey, ExceptionUtils.getRootCause(occurException).getMessage());
    EasyMock.verify(erRules);
}