Examples with TableMeta com.qlangtech.tis.sql.parser.er.TableMeta used on opensource projects

Example 1 with TableMeta

use of com.qlangtech.tis.sql.parser.er.TableMeta in project tis by qlangtech.

the class TestCoreAction method testTriggerFullbuildTaskByWithoutDefinePrimaryTable.

/**
 * 执行索引全量构建过程中，测试ERRule没有定义主表，会导致final表的分区函数无法正常创建，需要主动抛出一个异常
 */
public void testTriggerFullbuildTaskByWithoutDefinePrimaryTable() throws Exception {
    TableMeta tableMeta = new TableMeta(totalpayinfo.getTableName(), null);
    HttpUtils.addMockApply(-1, "tis-assemble/trigger", "assemble.trigger.result.faild.json", TestCoreAction.class);
    ITISCoordinator zkCoordinator = MockZKUtils.createZkMock();
    MockZooKeeperGetter.mockCoordinator = zkCoordinator;
    // this.createCoordinatorMock(false, (zk) -> {
    // });
    triggerFullbuildTask(tableMeta, (aResult) -> {
        assertFalse(aResult.isSuccess());
        List<String> errorMsgs = aResult.getErrorMsgs();
        assertNotNull(errorMsgs);
        assertTrue(errorMsgs.size() > 0);
    });
}

Example 2 with TableMeta

use of com.qlangtech.tis.sql.parser.er.TableMeta in project tis by qlangtech.

the class TestCoreAction method testTriggerFullbuildTask.

/**
 * 触发全量构建
 */
public void testTriggerFullbuildTask() throws Exception {
    TableMeta tableMeta = new TableMeta(totalpayinfo.getTableName(), "entity_id");
    ITISCoordinator zkCoordinator = MockZKUtils.createZkMock();
    MockZooKeeperGetter.mockCoordinator = zkCoordinator;
    HttpUtils.addMockApply(-1, "tis-assemble/trigger", "assemble.trigger.result.success.json", TestCoreAction.class);
    // this.createCoordinatorMock(false, (zk) -> {
    // TestCollectionAction.createAssembleLogCollectPathMock(zk);
    // });
    triggerFullbuildTask(tableMeta, (aResult) -> {
        assertTrue(aResult.isSuccess());
        org.json.JSONObject biz = (org.json.JSONObject) aResult.getBizResult();
        assertEquals(1234, biz.getInt(IParamContext.KEY_TASK_ID));
    });
}

Example 3 with TableMeta

use of com.qlangtech.tis.sql.parser.er.TableMeta in project tis by qlangtech.

the class TestSqlRewriter method testTotalpaySummaryRewriteByWithoutDefinePrimaryTableShareKey.

/**
 * 执行索引全量构建过程中，测试ERRule没有定义主表的<b>shareKey</b>，会导致final表的分区函数无法正常创建，需要主动抛出一个异常
 */
public void testTotalpaySummaryRewriteByWithoutDefinePrimaryTableShareKey() {
    TestJoinTaskContext taskContext = new TestJoinTaskContext(ExecutePhaseRange.fullRange());
    SqlTaskNodeMeta nodeMeta = getSqlTaskNodeMeta(totalpaySummary);
    ERRules erRules = createMockErRules(new TableMeta(totalpayinfo.getTableName(), null));
    EasyMock.replay(erRules);
    Exception occurException = null;
    try {
        assertFull(totalpaySummary, true, nodeMeta, taskContext, erRules, nodeMeta.getSql());
        fail("shall throw new exception");
    } catch (Exception e) {
        occurException = e;
    }
    assertNotNull("must throw an error", occurException);
    assertEquals(SqlRewriter.ERROR_WithoutDefinePrimaryTableShareKey, ExceptionUtils.getRootCause(occurException).getMessage());
    EasyMock.verify(erRules);
}

Example 4 with TableMeta

use of com.qlangtech.tis.sql.parser.er.TableMeta in project tis by qlangtech.

the class SqlRewriter method createPtPmodCols.

@Override
protected String createPtPmodCols(AliasTable a) {
    if (isFinal) {
        Optional<TableMeta> ptab = this.erRules.getPrimaryTab(a.getTable());
        StringBuffer result = new StringBuffer(a.getAlias() + "." + IDumpTable.PARTITION_PT + ",");
        // 如果当前是索引构建的场景下，需要校验是否已经设置分区键，这个判断非常重要2021/2/7，这个校验在最开始的点击触发按钮的时候也要校验
        if (joinContext.getExecutePhaseRange().contains(FullbuildPhase.BUILD) && !TableMeta.hasValidPrimayTableSharedKey(ptab)) {
            throw new IllegalStateException(ERROR_WithoutDefinePrimaryTableShareKey);
        }
        if (ptab.isPresent()) {
            TableMeta tabMeta = ptab.get();
            String shardKey = tabMeta.getSharedKey();
            if (StringUtils.isEmpty(shardKey)) {
                throw new IllegalStateException(tabMeta.toString() + " has not set 'shardKey' ");
            }
            try {
                Integer shardCount = joinContext.getIndexShardCount();
                result.append("abs( hash( cast( ").append(a.getAlias()).append(".").append(shardKey).append(" as string)) % ").append(shardCount).append(" ) AS ").append(IDumpTable.PARTITION_PMOD);
            } catch (Exception e) {
                throw new RuntimeException(tabMeta.toString(), e);
            }
        // 说明是主表,这个函数是Hive专用的
        // return a.getAlias() + "." + IDumpTable.PARTITION_PT + "," + "abs(pmod( hash( cast( " + a.getAlias() + "." + shardKey + " as string) ) , " + shardCount + ")) AS " + IDumpTable.PARTITION_PMOD;
        } else {
            // return a.getAlias() + "." + IDumpTable.PARTITION_PT + "," + a.getAlias() + "." + IDumpTable.PARTITION_PMOD;
            result.append(a.getAlias()).append(".").append(IDumpTable.PARTITION_PMOD);
        }
        return result.toString();
    } else {
        return super.createPtPmodCols(a);
    }
}

Example 5 with TableMeta

use of com.qlangtech.tis.sql.parser.er.TableMeta in project tis by qlangtech.

the class SqlTaskNodeMeta method validateSql.

/**
 * 对sql进行粗略的校验
 *
 * @param sql
 * @param dependencyNodes
 * @return
 */
public static Optional<TisSqlFormatException> validateSql(String sql, List<String> dependencyNodes) {
    // Optional<TisSqlFormatException> result  = Optional.empty();
    SqlTaskNodeMeta taskNodeMeta = new SqlTaskNodeMeta();
    // 这个sql语句有错误，需要校验成错误，抛异常
    taskNodeMeta.setSql(sql);
    final IJoinTaskContext tskContext = new DftJoinTaskContext(ExecutePhaseRange.fullRange());
    final ITemplateContext tplContext = new ITemplateContext() {

        @Override
        public <T> T getContextValue(String key) {
            return null;
        }

        @Override
        public void putContextValue(String key, Object v) {
        }

        @Override
        public IJoinTaskContext getExecContext() {
            return tskContext;
        }
    };
    try {
        String pt = "20200703113848";
        ITabPartition p = () -> pt;
        Map<IDumpTable, ITabPartition> tabPartition = dependencyNodes.stream().collect(Collectors.toMap((r) -> EntityName.parse(r), (r) -> p));
        taskNodeMeta.getRewriteSql("testTaskName", new MockDumpPartition(tabPartition), new IPrimaryTabFinder() {

            @Override
            public Optional<TableMeta> getPrimaryTab(IDumpTable entityName) {
                return Optional.empty();
            }

            @Override
            public Map<EntityName, TabFieldProcessor> getTabFieldProcessorMap() {
                return Collections.emptyMap();
            }
        }, tplContext, false);
        return Optional.empty();
    } catch (Throwable e) {
        int indexOf;
        if ((indexOf = ExceptionUtils.indexOfType(e, TisSqlFormatException.class)) > -1) {
            TisSqlFormatException ex = (TisSqlFormatException) ExceptionUtils.getThrowables(e)[indexOf];
            // System.out.println(ex.summary());
            return Optional.of(ex);
        // assertEquals("base ref:gg can not find relevant table entity in map,mapSize:1,exist:[g:tis.commodity_goods],位置，行:1,列:44", ex.summary());
        } else {
            throw e;
        }
    }
}