Examples with Configuration com.alibaba.datax.common.util.Configuration used on opensource projects

Example 11 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class ColumnCastTest method test_date.

@Test
public void test_date() throws IOException {
    Assert.assertTrue(DateCast.asString(new DateColumn(System.currentTimeMillis())).startsWith("201"));
    Configuration configuration = this.produce();
    configuration.set("common.column.datetimeFormat", "MM/dd/yyyy HH:mm:ss");
    DateCast.init(configuration);
    System.out.println(DateCast.asString(new DateColumn(System.currentTimeMillis())));
    Assert.assertTrue(!DateCast.asString(new DateColumn(System.currentTimeMillis())).startsWith("2014"));
    DateColumn dateColumn = new DateColumn(new Time(0L));
    System.out.println(dateColumn.asString());
    Assert.assertTrue(dateColumn.asString().equals("08:00:00"));
    configuration.set("common.column.timeZone", "GMT");
    DateCast.init(configuration);
    System.err.println(DateCast.asString(dateColumn));
    Assert.assertTrue(dateColumn.asString().equals("00:00:00"));
    configuration.set("common.column.timeZone", "GMT+8");
    DateCast.init(configuration);
    System.out.println(dateColumn.asString());
    Assert.assertTrue(dateColumn.asString().equals("08:00:00"));
    dateColumn = new DateColumn(new Date(0L));
    System.out.println(dateColumn.asString());
    Assert.assertTrue(dateColumn.asString().equals("1970-01-01"));
    dateColumn = new DateColumn(new java.util.Date(0L));
    System.out.println(dateColumn.asString());
    Assert.assertTrue(dateColumn.asString().equals("01/01/1970 08:00:00"));
}

Example 12 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class TaskGroupContainerTest method initConfiguration.

private void initConfiguration(Configuration configuration) {
    int channelNumber = 5;
    taskNumber = channelNumber + 3;
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_JOB_ID, 0);
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 1);
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_SLEEPINTERVAL, 200);
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_REPORTINTERVAL, 1000);
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_CHANNEL, channelNumber);
    Configuration jobContent = configuration.getListConfiguration(CoreConstant.DATAX_JOB_CONTENT).get(0);
    List<Configuration> jobContents = new ArrayList<Configuration>();
    for (int i = 0; i < this.taskNumber; i++) {
        Configuration newJobContent = jobContent.clone();
        newJobContent.set(CoreConstant.TASK_ID, i);
        jobContents.add(newJobContent);
    }
    configuration.set(CoreConstant.DATAX_JOB_CONTENT, jobContents);
    LocalTGCommunicationManager.clear();
    LocalTGCommunicationManager.registerTaskGroupCommunication(1, new Communication());
}

Example 13 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class StandAloneSchedulerTest method testSchedule.

@Test
public void testSchedule() throws NoSuchFieldException, IllegalAccessException {
    int taskNumber = 10;
    List<Configuration> jobList = new ArrayList<Configuration>();
    List<Configuration> internal = new ArrayList<Configuration>();
    int randomSize = 20;
    int length = RandomUtils.nextInt(0, randomSize) + 1;
    for (int i = 0; i < length; i++) {
        internal.add(Configuration.newDefault());
    }
    LocalTGCommunicationManager.clear();
    for (int i = 0; i < taskNumber; i++) {
        Configuration configuration = Configuration.newDefault();
        configuration.set(CoreConstant.DATAX_CORE_CONTAINER_JOB_REPORTINTERVAL, 11);
        configuration.set(CoreConstant.DATAX_CORE_CONTAINER_JOB_ID, 0);
        configuration.set(CoreConstant.DATAX_JOB_CONTENT, internal);
        configuration.set(CoreConstant.DATAX_CORE_CONTAINER_JOB_MODE, ExecuteMode.STANDALONE.getValue());
        configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, i);
        jobList.add(configuration);
        LocalTGCommunicationManager.registerTaskGroupCommunication(i, new Communication());
    }
    StandAloneJobContainerCommunicator standAloneJobContainerCommunicator = PowerMockito.mock(StandAloneJobContainerCommunicator.class);
    ProcessInnerScheduler scheduler = PowerMockito.spy(new StandAloneScheduler(standAloneJobContainerCommunicator));
    PowerMockito.doNothing().when(scheduler).startAllTaskGroup(anyListOf(Configuration.class));
    Communication communication = new Communication();
    communication.setState(State.SUCCEEDED);
    PowerMockito.when(standAloneJobContainerCommunicator.collect()).thenReturn(communication);
    PowerMockito.doNothing().when(standAloneJobContainerCommunicator).report(communication);
    scheduler.schedule(jobList);
}

Example 14 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class Hbase094xHelper method doSplit.

private static List<Configuration> doSplit(Configuration config, byte[] startRowkeyByte, byte[] endRowkeyByte, Pair<byte[][], byte[][]> regionRanges) {
    List<Configuration> configurations = new ArrayList<Configuration>();
    for (int i = 0; i < regionRanges.getFirst().length; i++) {
        byte[] regionStartKey = regionRanges.getFirst()[i];
        byte[] regionEndKey = regionRanges.getSecond()[i];
        // 注意如果用户指定userEndKey为"",则此判断应该不成立。userEndKey为""表示取得最大的region
        if (Bytes.compareTo(regionEndKey, HConstants.EMPTY_BYTE_ARRAY) == 0 && (endRowkeyByte.length != 0 && (Bytes.compareTo(regionStartKey, endRowkeyByte) > 0))) {
            continue;
        }
        // 用户配置的userStartKey大于等于region的endkey,则这个region不应该含在内
        if ((Bytes.compareTo(regionEndKey, HConstants.EMPTY_BYTE_ARRAY) != 0) && (Bytes.compareTo(startRowkeyByte, regionEndKey) >= 0)) {
            continue;
        }
        // 注意如果用户指定的userEndKey为"",则次判断应该不成立。userEndKey为""表示取得最大的region
        if (endRowkeyByte.length != 0 && (Bytes.compareTo(endRowkeyByte, regionStartKey) <= 0)) {
            continue;
        }
        Configuration p = config.clone();
        String thisStartKey = getStartKey(startRowkeyByte, regionStartKey);
        String thisEndKey = getEndKey(endRowkeyByte, regionEndKey);
        p.set(Key.START_ROWKEY, thisStartKey);
        p.set(Key.END_ROWKEY, thisEndKey);
        LOG.debug("startRowkey:[{}], endRowkey:[{}] .", thisStartKey, thisEndKey);
        configurations.add(p);
    }
    return configurations;
}

Example 15 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class Hbase094xHelper method validateRowkeyColumn.

public static void validateRowkeyColumn(com.alibaba.datax.common.util.Configuration originalConfig) {
    List<Configuration> rowkeyColumn = originalConfig.getListConfiguration(Key.ROWKEY_COLUMN);
    if (rowkeyColumn == null || rowkeyColumn.isEmpty()) {
        throw DataXException.asDataXException(Hbase094xWriterErrorCode.REQUIRED_VALUE, "rowkeyColumn为必填项，其形式为：rowkeyColumn:[{\"index\": 0,\"type\": \"string\"},{\"index\": -1,\"type\": \"string\",\"value\": \"_\"}]");
    }
    int rowkeyColumnSize = rowkeyColumn.size();
    //包含{"index":0,"type":"string"} 或者 {"index":-1,"type":"string","value":"_"}
    for (Configuration aRowkeyColumn : rowkeyColumn) {
        Integer index = aRowkeyColumn.getInt(Key.INDEX);
        String type = aRowkeyColumn.getNecessaryValue(Key.TYPE, Hbase094xWriterErrorCode.REQUIRED_VALUE);
        ColumnType.getByTypeName(type);
        if (index == null) {
            throw DataXException.asDataXException(Hbase094xWriterErrorCode.REQUIRED_VALUE, "rowkeyColumn配置项中index为必填项");
        }
        //不能只有-1列,即rowkey连接串
        if (rowkeyColumnSize == 1 && index == -1) {
            throw DataXException.asDataXException(Hbase094xWriterErrorCode.ILLEGAL_VALUE, "rowkeyColumn配置项不能全为常量列,至少指定一个rowkey列");
        }
        if (index == -1) {
            aRowkeyColumn.getNecessaryValue(Key.VALUE, Hbase094xWriterErrorCode.REQUIRED_VALUE);
        }
    }
}