Examples with DownloadSession com.aliyun.odps.tunnel.TableTunnel.DownloadSession used on opensource projects

Example 1 with DownloadSession

use of com.aliyun.odps.tunnel.TableTunnel.DownloadSession in project DataX by alibaba.

the class OdpsSplitUtil method splitForNonPartitionedTable.

private static List<Configuration> splitForNonPartitionedTable(Odps odps, int adviceNum, Configuration sliceConfig) {
    List<Configuration> params = new ArrayList<Configuration>();
    String tunnelServer = sliceConfig.getString(Key.TUNNEL_SERVER);
    String tableName = sliceConfig.getString(Key.TABLE);
    String projectName = sliceConfig.getString(Key.PROJECT);
    DownloadSession session = OdpsUtil.createMasterSessionForNonPartitionedTable(odps, tunnelServer, projectName, tableName);
    String id = session.getId();
    long count = session.getRecordCount();
    List<Pair<Long, Long>> splitResult = splitRecordCount(count, adviceNum);
    for (Pair<Long, Long> pair : splitResult) {
        Configuration iParam = sliceConfig.clone();
        iParam.set(Constant.SESSION_ID, id);
        iParam.set(Constant.START_INDEX, pair.getLeft().longValue());
        iParam.set(Constant.STEP_COUNT, pair.getRight().longValue());
        params.add(iParam);
    }
    return params;
}

Example 2 with DownloadSession

use of com.aliyun.odps.tunnel.TableTunnel.DownloadSession in project DataX by alibaba.

the class OdpsSplitUtil method splitOnePartition.

private static List<Configuration> splitOnePartition(Odps odps, String onePartition, int adviceNum, Configuration sliceConfig) {
    List<Configuration> params = new ArrayList<Configuration>();
    String tunnelServer = sliceConfig.getString(Key.TUNNEL_SERVER);
    String tableName = sliceConfig.getString(Key.TABLE);
    String projectName = sliceConfig.getString(Key.PROJECT);
    DownloadSession session = OdpsUtil.createMasterSessionForPartitionedTable(odps, tunnelServer, projectName, tableName, onePartition);
    String id = session.getId();
    long count = session.getRecordCount();
    List<Pair<Long, Long>> splitResult = splitRecordCount(count, adviceNum);
    for (Pair<Long, Long> pair : splitResult) {
        Configuration iParam = sliceConfig.clone();
        iParam.set(Key.PARTITION, onePartition);
        iParam.set(Constant.SESSION_ID, id);
        iParam.set(Constant.START_INDEX, pair.getLeft().longValue());
        iParam.set(Constant.STEP_COUNT, pair.getRight().longValue());
        params.add(iParam);
    }
    return params;
}