Examples with ITabPartition com.qlangtech.tis.fullbuild.indexbuild.ITabPartition used on opensource projects

Example 1 with ITabPartition

use of com.qlangtech.tis.fullbuild.indexbuild.ITabPartition in project plugins by qlangtech.

the class TestHiveFlatTableBuilder method testTotalpaySummary.

public void testTotalpaySummary() throws Exception {
    HiveFlatTableBuilder flatTableBuilder = (HiveFlatTableBuilder) flatTableBuilderStore.getPlugin();
    // ISqlTask nodeMeta, boolean isFinalNode
    // , ITemplateContext tplContext, ITaskContext taskContext, IFs2Table fs2Table, IJoinTaskStatus joinTaskStatus
    IFs2Table fs2Table = new MockFs2Table();
    IJoinTaskStatus joinTaskStatus = EasyMock.createMock("joinTaskStatus", IJoinTaskStatus.class);
    joinTaskStatus.setComplete(true);
    joinTaskStatus.createJobStatus(EasyMock.anyInt());
    JobLog jobLog = new JobLog();
    EasyMock.expect(joinTaskStatus.getJoblog(EasyMock.anyInt())).andReturn(jobLog).anyTimes();
    joinTaskStatus.setStart();
    IJoinTaskContext joinTaskContext = EasyMock.createMock("joinTaskContext", IJoinTaskContext.class);
    Map<IDumpTable, ITabPartition> dateParams = Maps.newHashMap();
    EasyMock.expect(joinTaskContext.getAttribute(ExecChainContextUtils.PARTITION_DATA_PARAMS)).andReturn(dateParams).anyTimes();
    Map<String, Boolean> taskWorkStatus = Maps.newHashMap();
    EasyMock.expect(joinTaskContext.getAttribute(AdapterTask.KEY_TASK_WORK_STATUS)).andReturn(taskWorkStatus);
    ERRules erRules = EasyMock.createMock("erRules", ERRules.class);
    EasyMock.expect(joinTaskContext.getAttribute("er_rules")).andReturn(erRules);
    EasyMock.replay(joinTaskStatus, joinTaskContext, erRules);
    MockTemplateContext tplContext = new MockTemplateContext(joinTaskContext);
    flatTableBuilder.startTask((context) -> {
        try (InputStream input = TestHiveFlatTableBuilder.class.getResourceAsStream("groupby_totalpay.sql")) {
            // try (InputStream input = TestHiveFlatTableBuilder.class.getResourceAsStream("totalpay_summary.sql")) {
            ISqlTask sqlTask = new DefaultSqlTask(IOUtils.toString(input, TisUTF8.get()));
            // ISqlTask nodeMeta, boolean isFinalNode
            // , ITemplateContext tplContext, ITaskContext taskContext, //
            // IJoinTaskStatus joinTaskStatus
            DataflowTask joinTask = flatTableBuilder.createTask(sqlTask, true, tplContext, context, joinTaskStatus);
            joinTask.run();
        }
    });
}

Example 2 with ITabPartition

use of com.qlangtech.tis.fullbuild.indexbuild.ITabPartition in project tis by qlangtech.

the class SqlRewriter method visitAliasedRelation.

@Override
protected Void visitAliasedRelation(AliasedRelation node, Integer indent) {
    if (node.getRelation() instanceof Table) {
        Table tab = (Table) node.getRelation();
        Map.Entry<IDumpTable, ITabPartition> dumpTable = parseDumpTable(tab.getName());
        waitProcessAliasTabsSet.add(new AliasTable(node.getAlias().getValue(), dumpTable.getKey(), dumpTable.getValue()));
        processTable(dumpTable);
        builder.append(' ').append(formatExpression(node.getAlias(), parameters));
        SqlFormatter.appendAliasColumns(builder, node.getColumnNames());
        return null;
    } else if (node.getRelation() instanceof TableSubquery) {
        SqlRewriter w = new SqlRewriter(new SqlStringBuilder(), this.tabPartition, this.erRules, this.parameters, false, joinContext);
        w.process(node.getRelation(), 0);
        Optional<AliasTable> subTable = w.getWaitProcessAliasTabsSet().stream().findFirst();
        if (!subTable.isPresent()) {
            throw new IllegalStateException("subtable:" + node.getAlias().getValue() + " can not find subtable");
        }
        // 为了重写select部分的pt,pmod部分，需要把该别名记录下来
        waitProcessAliasTabsSet.add(new AliasTable(node.getAlias().getValue(), subTable.get()));
        return super.visitAliasedRelation(node, indent);
    } else {
        throw new UnsupportedOperationException();
    }
// process(node.getRelation(), indent);
// builder.append(' ').append(formatExpression(node.getAlias(), parameters));
// SqlFormatter.appendAliasColumns(builder, node.getColumnNames());
// return null;//
// return super.visitAliasedRelation(node, indent);
}

Example 3 with ITabPartition

use of com.qlangtech.tis.fullbuild.indexbuild.ITabPartition in project tis by qlangtech.

the class TestSqlTaskNodeMeta method testGetRewriteSql.

public void testGetRewriteSql() throws Exception {
    SqlTaskNodeMeta taskNodeMeta = new SqlTaskNodeMeta();
    SqlDataFlowTopology topology = SqlTaskNodeMeta.getSqlDataFlowTopology(TestSupplyGoodsParse.topologyName);
    assertNotNull(topology);
    SqlTaskNodeMeta finalNode = topology.getFinalNode();
    assertNotNull(finalNode);
    taskNodeMeta.setSql(finalNode.getSql());
    Map<IDumpTable, ITabPartition> dumpPartition = Maps.newHashMap();
    String pt = "20200703113848";
    dumpPartition.put(EntityName.parse("scmdb.warehouse_goods"), () -> pt);
    dumpPartition.put(EntityName.parse("tis.stock_info_collapse"), () -> pt);
    dumpPartition.put(EntityName.parse("scmdb.supplier_goods"), () -> pt);
    dumpPartition.put(EntityName.parse("tis.warehouse_collapse"), () -> pt);
    dumpPartition.put(EntityName.parse("tis.supplier_collapse"), () -> pt);
    dumpPartition.put(EntityName.parse("scmdb.goods"), () -> pt);
    dumpPartition.put(EntityName.parse("scmdb.stock_info"), () -> pt);
    dumpPartition.put(EntityName.parse("scmdb.category"), () -> pt);
    dumpPartition.put(EntityName.parse("scmdb.goods_sync_shop"), () -> pt);
    ITemplateContext tplContext = EasyMock.createMock("templateContext", ITemplateContext.class);
    IJoinTaskContext joinTaskContext = EasyMock.createMock("joinTaskContext", IJoinTaskContext.class);
    EasyMock.expect(tplContext.getExecContext()).andReturn(joinTaskContext);
    EasyMock.expect(joinTaskContext.getExecutePhaseRange()).andReturn(ExecutePhaseRange.fullRange()).times(2);
    EasyMock.expect(joinTaskContext.getIndexShardCount()).andReturn(1).times(1);
    Optional<ERRules> erRule = ERRules.getErRule(TestSupplyGoodsParse.topologyName);
    assertTrue(erRule.isPresent());
    EasyMock.replay(tplContext, joinTaskContext);
    ISqlTask.RewriteSql rewriteSql = taskNodeMeta.getRewriteSql("supply_goods", new TabPartitions(dumpPartition), erRule.get(), tplContext, true);
    assertNotNull(rewriteSql);
    assertEquals(TestSqlRewriter.getScriptContent("supply_goods_rewrite_result.txt"), rewriteSql.sqlContent);
    System.out.println(rewriteSql.sqlContent);
    EasyMock.verify(tplContext, joinTaskContext);
}

Example 4 with ITabPartition

use of com.qlangtech.tis.fullbuild.indexbuild.ITabPartition in project tis by qlangtech.

the class TestIndexSwapTaskflowLauncher method createRangeChainContext.

// 
public static DefaultChainContext createRangeChainContext(String collectionName, FullbuildPhase start, FullbuildPhase end, String... pts) throws Exception {
    TestParamContext params = new TestParamContext();
    params.set(IFullBuildContext.KEY_APP_SHARD_COUNT, String.valueOf(shardCount));
    params.set(IFullBuildContext.KEY_APP_NAME, collectionName);
    params.set(IFullBuildContext.KEY_WORKFLOW_NAME, "totalpay");
    params.set(IFullBuildContext.KEY_WORKFLOW_ID, WF_ID);
    params.set(IExecChainContext.COMPONENT_START, start.getName());
    params.set(IExecChainContext.COMPONENT_END, end.getName());
    final DefaultChainContext chainContext = new DefaultChainContext(params);
    ExecutePhaseRange range = chainContext.getExecutePhaseRange();
    Assert.assertEquals(start, range.getStart());
    Assert.assertEquals(end, range.getEnd());
    Map<IDumpTable, ITabPartition> dateParams = Maps.newHashMap();
    chainContext.setAttribute(ExecChainContextUtils.PARTITION_DATA_PARAMS, new TabPartitions(dateParams));
    chainContext.setAttribute(IExecChainContext.KEY_TASK_ID, TASK_ID);
    // final PluginStore<IndexBuilderTriggerFactory> buildTriggerFactory = TIS.getPluginStore(IndexBuilderTriggerFactory.class);
    // assertNotNull(buildTriggerFactory.getPlugin());
    // if (pts.length > 0) {
    // chainContext.setPs(pts[0]);
    // } else {
    // SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHHmmss");
    // String pt = format.format(new Date());
    // chainContext.setPs(pt);
    // }
    chainContext.setMdcParamContext(() -> {
    });
    return chainContext;
}

Example 5 with ITabPartition

use of com.qlangtech.tis.fullbuild.indexbuild.ITabPartition in project tis by qlangtech.

the class ExecChainContextUtils method getDependencyTablesMINPartition.

public static ITabPartition getDependencyTablesMINPartition(IJoinTaskContext context) {
    TabPartitions dateParams = getDependencyTablesPartitions(context);
    // dateParams.values().stream().min(Comparator.comparing((r) -> Long.parseLong(r.getPt())));
    Optional<ITabPartition> min = dateParams.getMinTablePartition();
    if (!min.isPresent()) {
        return () -> context.getPartitionTimestamp();
    }
    return min.get();
}