Examples with TabPartitions com.qlangtech.tis.sql.parser.TabPartitions used on opensource projects

Example 1 with TabPartitions

use of com.qlangtech.tis.sql.parser.TabPartitions in project plugins by qlangtech.

the class HiveTask method executeSql.

@Override
protected void executeSql(String taskname, String sql) {
    this.validateDependenciesNode(taskname);
    final Connection conn = this.getTaskContext().getObj();
    final EntityName newCreateTab = EntityName.parse(this.nodeMeta.getExportName());
    // final String newCreatePt = primaryTable.getTabPartition();
    this.getContent();
    List<String> allpts = null;
    try {
        logger.info("\n execute hive task:{} \n{}", taskname, sql);
        HiveDBUtils.execute(conn, sql, joinTaskStatus);
        // 将当前的join task的partition设置到当前上下文中
        TabPartitions dumpPartition = this.getDumpPartition();
        dumpPartition.putPt(newCreateTab, this.rewriteSql.primaryTable);
        allpts = HiveRemoveHistoryDataTask.getHistoryPts(conn, newCreateTab);
    } catch (Exception e) {
        // TODO 一旦有异常要将整个链路执行都停下来
        throw new RuntimeException("taskname:" + taskname, e);
    }
    IAliasTable child = null;
    // 校验最新的Partition 是否已经生成
    if (!allpts.contains(this.rewriteSql.primaryTable.getPt())) {
        StringBuffer errInfo = new StringBuffer();
        errInfo.append("\ntable:" + newCreateTab + "," + IDumpTable.PARTITION_PT + ":" + this.rewriteSql.primaryTable + " is not exist in exist partition set [" + Joiner.on(",").join(allpts) + "]");
        child = this.rewriteSql.primaryTable.getChild();
        if (child != null && !child.isSubQueryTable()) {
            try {
                allpts = HiveRemoveHistoryDataTask.getHistoryPts(conn, child.getTable());
            } catch (Exception e) {
                throw new RuntimeException(child.getTable().getFullName(), e);
            }
            errInfo.append("\n\t child table:").append(child.getTable()).append(",").append(IDumpTable.PARTITION_PT).append(":").append(this.rewriteSql.primaryTable).append(" is not exist in exist partition set [").append(Joiner.on(",").join(allpts)).append("]");
        }
        throw new IllegalStateException(errInfo.toString());
    }
}

Example 2 with TabPartitions

use of com.qlangtech.tis.sql.parser.TabPartitions in project tis by qlangtech.

the class TestIndexSwapTaskflowLauncher method createRangeChainContext.

// 
public static DefaultChainContext createRangeChainContext(String collectionName, FullbuildPhase start, FullbuildPhase end, String... pts) throws Exception {
    TestParamContext params = new TestParamContext();
    params.set(IFullBuildContext.KEY_APP_SHARD_COUNT, String.valueOf(shardCount));
    params.set(IFullBuildContext.KEY_APP_NAME, collectionName);
    params.set(IFullBuildContext.KEY_WORKFLOW_NAME, "totalpay");
    params.set(IFullBuildContext.KEY_WORKFLOW_ID, WF_ID);
    params.set(IExecChainContext.COMPONENT_START, start.getName());
    params.set(IExecChainContext.COMPONENT_END, end.getName());
    final DefaultChainContext chainContext = new DefaultChainContext(params);
    ExecutePhaseRange range = chainContext.getExecutePhaseRange();
    Assert.assertEquals(start, range.getStart());
    Assert.assertEquals(end, range.getEnd());
    Map<IDumpTable, ITabPartition> dateParams = Maps.newHashMap();
    chainContext.setAttribute(ExecChainContextUtils.PARTITION_DATA_PARAMS, new TabPartitions(dateParams));
    chainContext.setAttribute(IExecChainContext.KEY_TASK_ID, TASK_ID);
    // final PluginStore<IndexBuilderTriggerFactory> buildTriggerFactory = TIS.getPluginStore(IndexBuilderTriggerFactory.class);
    // assertNotNull(buildTriggerFactory.getPlugin());
    // if (pts.length > 0) {
    // chainContext.setPs(pts[0]);
    // } else {
    // SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHHmmss");
    // String pt = format.format(new Date());
    // chainContext.setPs(pt);
    // }
    chainContext.setMdcParamContext(() -> {
    });
    return chainContext;
}

Example 3 with TabPartitions

use of com.qlangtech.tis.sql.parser.TabPartitions in project tis by qlangtech.

the class ExecChainContextUtils method getDependencyTablesMINPartition.

public static ITabPartition getDependencyTablesMINPartition(IJoinTaskContext context) {
    TabPartitions dateParams = getDependencyTablesPartitions(context);
    // dateParams.values().stream().min(Comparator.comparing((r) -> Long.parseLong(r.getPt())));
    Optional<ITabPartition> min = dateParams.getMinTablePartition();
    if (!min.isPresent()) {
        return () -> context.getPartitionTimestamp();
    }
    return min.get();
}