Search in sources :

Example 1 with TableMeta

use of com.qlangtech.tis.sql.parser.er.TableMeta in project tis by qlangtech.

the class TestCoreAction method testTriggerFullbuildTaskByWithoutDefinePrimaryTable.

/**
 * 执行索引全量构建过程中,测试ERRule没有定义主表,会导致final表的分区函数无法正常创建,需要主动抛出一个异常
 */
public void testTriggerFullbuildTaskByWithoutDefinePrimaryTable() throws Exception {
    TableMeta tableMeta = new TableMeta(totalpayinfo.getTableName(), null);
    HttpUtils.addMockApply(-1, "tis-assemble/trigger", "assemble.trigger.result.faild.json", TestCoreAction.class);
    ITISCoordinator zkCoordinator = MockZKUtils.createZkMock();
    MockZooKeeperGetter.mockCoordinator = zkCoordinator;
    // this.createCoordinatorMock(false, (zk) -> {
    // });
    triggerFullbuildTask(tableMeta, (aResult) -> {
        assertFalse(aResult.isSuccess());
        List<String> errorMsgs = aResult.getErrorMsgs();
        assertNotNull(errorMsgs);
        assertTrue(errorMsgs.size() > 0);
    });
}
Also used : ITISCoordinator(com.qlangtech.tis.cloud.ITISCoordinator) TableMeta(com.qlangtech.tis.sql.parser.er.TableMeta) PrimaryTableMeta(com.qlangtech.tis.sql.parser.er.PrimaryTableMeta)

Example 2 with TableMeta

use of com.qlangtech.tis.sql.parser.er.TableMeta in project tis by qlangtech.

the class TestCoreAction method testTriggerFullbuildTask.

/**
 * 触发全量构建
 */
public void testTriggerFullbuildTask() throws Exception {
    TableMeta tableMeta = new TableMeta(totalpayinfo.getTableName(), "entity_id");
    ITISCoordinator zkCoordinator = MockZKUtils.createZkMock();
    MockZooKeeperGetter.mockCoordinator = zkCoordinator;
    HttpUtils.addMockApply(-1, "tis-assemble/trigger", "assemble.trigger.result.success.json", TestCoreAction.class);
    // this.createCoordinatorMock(false, (zk) -> {
    // TestCollectionAction.createAssembleLogCollectPathMock(zk);
    // });
    triggerFullbuildTask(tableMeta, (aResult) -> {
        assertTrue(aResult.isSuccess());
        org.json.JSONObject biz = (org.json.JSONObject) aResult.getBizResult();
        assertEquals(1234, biz.getInt(IParamContext.KEY_TASK_ID));
    });
}
Also used : ITISCoordinator(com.qlangtech.tis.cloud.ITISCoordinator) TableMeta(com.qlangtech.tis.sql.parser.er.TableMeta) PrimaryTableMeta(com.qlangtech.tis.sql.parser.er.PrimaryTableMeta)

Example 3 with TableMeta

use of com.qlangtech.tis.sql.parser.er.TableMeta in project tis by qlangtech.

the class TestSqlRewriter method testTotalpaySummaryRewriteByWithoutDefinePrimaryTableShareKey.

/**
 * 执行索引全量构建过程中,测试ERRule没有定义主表的<b>shareKey</b>,会导致final表的分区函数无法正常创建,需要主动抛出一个异常
 */
public void testTotalpaySummaryRewriteByWithoutDefinePrimaryTableShareKey() {
    TestJoinTaskContext taskContext = new TestJoinTaskContext(ExecutePhaseRange.fullRange());
    SqlTaskNodeMeta nodeMeta = getSqlTaskNodeMeta(totalpaySummary);
    ERRules erRules = createMockErRules(new TableMeta(totalpayinfo.getTableName(), null));
    EasyMock.replay(erRules);
    Exception occurException = null;
    try {
        assertFull(totalpaySummary, true, nodeMeta, taskContext, erRules, nodeMeta.getSql());
        fail("shall throw new exception");
    } catch (Exception e) {
        occurException = e;
    }
    assertNotNull("must throw an error", occurException);
    assertEquals(SqlRewriter.ERROR_WithoutDefinePrimaryTableShareKey, ExceptionUtils.getRootCause(occurException).getMessage());
    EasyMock.verify(erRules);
}
Also used : TableMeta(com.qlangtech.tis.sql.parser.er.TableMeta) TestJoinTaskContext(com.qlangtech.tis.order.center.TestJoinTaskContext) ERRules(com.qlangtech.tis.sql.parser.er.ERRules)

Example 4 with TableMeta

use of com.qlangtech.tis.sql.parser.er.TableMeta in project tis by qlangtech.

the class SqlRewriter method createPtPmodCols.

@Override
protected String createPtPmodCols(AliasTable a) {
    if (isFinal) {
        Optional<TableMeta> ptab = this.erRules.getPrimaryTab(a.getTable());
        StringBuffer result = new StringBuffer(a.getAlias() + "." + IDumpTable.PARTITION_PT + ",");
        // 如果当前是索引构建的场景下,需要校验是否已经设置分区键,这个判断非常重要2021/2/7,这个校验在最开始的点击触发按钮的时候也要校验
        if (joinContext.getExecutePhaseRange().contains(FullbuildPhase.BUILD) && !TableMeta.hasValidPrimayTableSharedKey(ptab)) {
            throw new IllegalStateException(ERROR_WithoutDefinePrimaryTableShareKey);
        }
        if (ptab.isPresent()) {
            TableMeta tabMeta = ptab.get();
            String shardKey = tabMeta.getSharedKey();
            if (StringUtils.isEmpty(shardKey)) {
                throw new IllegalStateException(tabMeta.toString() + " has not set 'shardKey' ");
            }
            try {
                Integer shardCount = joinContext.getIndexShardCount();
                result.append("abs( hash( cast( ").append(a.getAlias()).append(".").append(shardKey).append(" as string)) % ").append(shardCount).append(" ) AS ").append(IDumpTable.PARTITION_PMOD);
            } catch (Exception e) {
                throw new RuntimeException(tabMeta.toString(), e);
            }
        // 说明是主表,这个函数是Hive专用的
        // return a.getAlias() + "." + IDumpTable.PARTITION_PT + "," + "abs(pmod( hash( cast( " + a.getAlias() + "." + shardKey + " as string) ) , " + shardCount + ")) AS " + IDumpTable.PARTITION_PMOD;
        } else {
            // return a.getAlias() + "." + IDumpTable.PARTITION_PT + "," + a.getAlias() + "." + IDumpTable.PARTITION_PMOD;
            result.append(a.getAlias()).append(".").append(IDumpTable.PARTITION_PMOD);
        }
        return result.toString();
    } else {
        return super.createPtPmodCols(a);
    }
}
Also used : TableMeta(com.qlangtech.tis.sql.parser.er.TableMeta) TisSqlFormatException(com.qlangtech.tis.sql.parser.exception.TisSqlFormatException)

Example 5 with TableMeta

use of com.qlangtech.tis.sql.parser.er.TableMeta in project tis by qlangtech.

the class SqlTaskNodeMeta method validateSql.

/**
 * 对sql进行粗略的校验
 *
 * @param sql
 * @param dependencyNodes
 * @return
 */
public static Optional<TisSqlFormatException> validateSql(String sql, List<String> dependencyNodes) {
    // Optional<TisSqlFormatException> result  = Optional.empty();
    SqlTaskNodeMeta taskNodeMeta = new SqlTaskNodeMeta();
    // 这个sql语句有错误,需要校验成错误,抛异常
    taskNodeMeta.setSql(sql);
    final IJoinTaskContext tskContext = new DftJoinTaskContext(ExecutePhaseRange.fullRange());
    final ITemplateContext tplContext = new ITemplateContext() {

        @Override
        public <T> T getContextValue(String key) {
            return null;
        }

        @Override
        public void putContextValue(String key, Object v) {
        }

        @Override
        public IJoinTaskContext getExecContext() {
            return tskContext;
        }
    };
    try {
        String pt = "20200703113848";
        ITabPartition p = () -> pt;
        Map<IDumpTable, ITabPartition> tabPartition = dependencyNodes.stream().collect(Collectors.toMap((r) -> EntityName.parse(r), (r) -> p));
        taskNodeMeta.getRewriteSql("testTaskName", new MockDumpPartition(tabPartition), new IPrimaryTabFinder() {

            @Override
            public Optional<TableMeta> getPrimaryTab(IDumpTable entityName) {
                return Optional.empty();
            }

            @Override
            public Map<EntityName, TabFieldProcessor> getTabFieldProcessorMap() {
                return Collections.emptyMap();
            }
        }, tplContext, false);
        return Optional.empty();
    } catch (Throwable e) {
        int indexOf;
        if ((indexOf = ExceptionUtils.indexOfType(e, TisSqlFormatException.class)) > -1) {
            TisSqlFormatException ex = (TisSqlFormatException) ExceptionUtils.getThrowables(e)[indexOf];
            // System.out.println(ex.summary());
            return Optional.of(ex);
        // assertEquals("base ref:gg can not find relevant table entity in map,mapSize:1,exist:[g:tis.commodity_goods],位置,行:1,列:44", ex.summary());
        } else {
            throw e;
        }
    }
}
Also used : LineIterator(org.apache.commons.io.LineIterator) IDumpTable(com.qlangtech.tis.fullbuild.indexbuild.IDumpTable) TableMeta(com.qlangtech.tis.sql.parser.er.TableMeta) StringUtils(org.apache.commons.lang.StringUtils) ColumnTransfer(com.qlangtech.tis.sql.parser.meta.ColumnTransfer) TIS(com.qlangtech.tis.TIS) DefaultDumpNodeMapContext(com.qlangtech.tis.sql.parser.utils.DefaultDumpNodeMapContext) Node(org.yaml.snakeyaml.nodes.Node) ExecutePhaseRange(com.qlangtech.tis.exec.ExecutePhaseRange) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ScalarStyle(org.yaml.snakeyaml.DumperOptions.ScalarStyle) Method(java.lang.reflect.Method) ITabPartition(com.qlangtech.tis.fullbuild.indexbuild.ITabPartition) DependencyNode(com.qlangtech.tis.sql.parser.meta.DependencyNode) EntityName(com.qlangtech.tis.sql.parser.tuple.creator.EntityName) IJoinTaskContext(com.qlangtech.tis.order.center.IJoinTaskContext) Property(org.yaml.snakeyaml.introspector.Property) Collectors(java.util.stream.Collectors) SqlParser(com.facebook.presto.sql.parser.SqlParser) IOUtils(org.apache.commons.io.IOUtils) LineBreak(org.yaml.snakeyaml.DumperOptions.LineBreak) JSONObject(com.alibaba.fastjson.JSONObject) Pattern(java.util.regex.Pattern) ParsingOptions(com.facebook.presto.sql.parser.ParsingOptions) Joiner(com.google.common.base.Joiner) Types(java.sql.Types) java.util(java.util) JSONField(com.alibaba.fastjson.annotation.JSONField) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) TableTupleCreator(com.qlangtech.tis.sql.parser.tuple.creator.impl.TableTupleCreator) Yaml(org.yaml.snakeyaml.Yaml) DumperOptions(org.yaml.snakeyaml.DumperOptions) IAppSourcePipelineController(com.qlangtech.tis.order.center.IAppSourcePipelineController) Lists(com.google.common.collect.Lists) TabFieldProcessor(com.qlangtech.tis.sql.parser.er.TabFieldProcessor) TisUTF8(com.qlangtech.tis.manage.common.TisUTF8) Position(com.qlangtech.tis.sql.parser.meta.Position) Constructor(org.yaml.snakeyaml.constructor.Constructor) NodeType(com.qlangtech.tis.sql.parser.meta.NodeType) ExceptionUtils(org.apache.commons.lang.exception.ExceptionUtils) com.qlangtech.tis.plugin.ds(com.qlangtech.tis.plugin.ds) FileUtils(org.apache.commons.io.FileUtils) Tag(org.yaml.snakeyaml.nodes.Tag) Maps(com.google.common.collect.Maps) TisSqlFormatException(com.qlangtech.tis.sql.parser.exception.TisSqlFormatException) NodeTuple(org.yaml.snakeyaml.nodes.NodeTuple) FlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle) JSON(com.alibaba.fastjson.JSON) ERRules(com.qlangtech.tis.sql.parser.er.ERRules) Expression(com.facebook.presto.sql.tree.Expression) ITemplateContext(com.qlangtech.tis.fullbuild.taskflow.ITemplateContext) java.io(java.io) IPrimaryTabFinder(com.qlangtech.tis.sql.parser.er.IPrimaryTabFinder) Representer(org.yaml.snakeyaml.representer.Representer) TypeDescription(org.yaml.snakeyaml.TypeDescription) Statement(com.facebook.presto.sql.tree.Statement) ITabPartition(com.qlangtech.tis.fullbuild.indexbuild.ITabPartition) IDumpTable(com.qlangtech.tis.fullbuild.indexbuild.IDumpTable) TisSqlFormatException(com.qlangtech.tis.sql.parser.exception.TisSqlFormatException) IPrimaryTabFinder(com.qlangtech.tis.sql.parser.er.IPrimaryTabFinder) JSONObject(com.alibaba.fastjson.JSONObject) IJoinTaskContext(com.qlangtech.tis.order.center.IJoinTaskContext) ITemplateContext(com.qlangtech.tis.fullbuild.taskflow.ITemplateContext)

Aggregations

TableMeta (com.qlangtech.tis.sql.parser.er.TableMeta)5 ITISCoordinator (com.qlangtech.tis.cloud.ITISCoordinator)2 ERRules (com.qlangtech.tis.sql.parser.er.ERRules)2 PrimaryTableMeta (com.qlangtech.tis.sql.parser.er.PrimaryTableMeta)2 TisSqlFormatException (com.qlangtech.tis.sql.parser.exception.TisSqlFormatException)2 JSON (com.alibaba.fastjson.JSON)1 JSONObject (com.alibaba.fastjson.JSONObject)1 JSONField (com.alibaba.fastjson.annotation.JSONField)1 ParsingOptions (com.facebook.presto.sql.parser.ParsingOptions)1 SqlParser (com.facebook.presto.sql.parser.SqlParser)1 Expression (com.facebook.presto.sql.tree.Expression)1 Statement (com.facebook.presto.sql.tree.Statement)1 Joiner (com.google.common.base.Joiner)1 Lists (com.google.common.collect.Lists)1 Maps (com.google.common.collect.Maps)1 TIS (com.qlangtech.tis.TIS)1 ExecutePhaseRange (com.qlangtech.tis.exec.ExecutePhaseRange)1 IDumpTable (com.qlangtech.tis.fullbuild.indexbuild.IDumpTable)1 ITabPartition (com.qlangtech.tis.fullbuild.indexbuild.ITabPartition)1 ITemplateContext (com.qlangtech.tis.fullbuild.taskflow.ITemplateContext)1