Examples with TaskConfig org.apache.helix.task.TaskConfig used on opensource projects

Example 11 with TaskConfig

use of org.apache.helix.task.TaskConfig in project helix by apache.

the class TestIndependentTaskRebalancer method testDifferentTasks.

@Test
public void testDifferentTasks() throws Exception {
    // Create a job with two different tasks
    String jobName = TestHelper.getTestMethodName();
    Workflow.Builder workflowBuilder = new Workflow.Builder(jobName);
    List<TaskConfig> taskConfigs = Lists.newArrayListWithCapacity(2);
    TaskConfig taskConfig1 = new TaskConfig("TaskOne", null);
    TaskConfig taskConfig2 = new TaskConfig("TaskTwo", null);
    taskConfigs.add(taskConfig1);
    taskConfigs.add(taskConfig2);
    Map<String, String> jobCommandMap = Maps.newHashMap();
    jobCommandMap.put("Timeout", "1000");
    JobConfig.Builder jobBuilder = new JobConfig.Builder().setCommand("DummyCommand").addTaskConfigs(taskConfigs).setJobCommandConfigMap(jobCommandMap);
    workflowBuilder.addJob(jobName, jobBuilder);
    _driver.start(workflowBuilder.build());
    // Ensure the job completes
    _driver.pollForWorkflowState(jobName, TaskState.COMPLETED);
    // Ensure that each class was invoked
    Assert.assertTrue(_invokedClasses.contains(TaskOne.class.getName()));
    Assert.assertTrue(_invokedClasses.contains(TaskTwo.class.getName()));
}

Example 12 with TaskConfig

use of org.apache.helix.task.TaskConfig in project ambry by linkedin.

the class HelixClusterWideAggregationTool method main.

/**
 * @param args takes in three mandatory arguments: the ZK layout, the cluster name, the workflow name. Optional
 *             argument to create the workflow as a recurrent workflow and specifies the recurrent time interval.
 *             The ZK layout has to be of the following form:
 *             {
 *               "zkInfo" : [
 *                 {
 *                   "datacenter":"dc1",
 *                    "id" : "1",
 *                   "zkConnectStr":"abc.example.com:2199",
 *                 },
 *                 {
 *                   "datacenter":"dc2",
 *                   "id" : "2",
 *                   "zkConnectStr":"def.example.com:2300",
 *                 }
 *               ]
 *             }
 * @throws Exception
 */
public static void main(String[] args) throws Exception {
    OptionParser parser = new OptionParser();
    ArgumentAcceptingOptionSpec<String> zkLayoutPathOpt = parser.accepts("zkLayoutPath", "The path to the json file containing zookeeper connect info. This should be of the following form: \n{\n" + "  \"zkInfo\" : [\n" + "     {\n" + "       \"datacenter\":\"dc1\",\n" + "       \"id\":\"1\",\n" + "       \"zkConnectStr\":\"abc.example.com:2199\",\n" + "     },\n" + "     {\n" + "       \"datacenter\":\"dc2\",\n" + "       \"id\":\"2\",\n" + "       \"zkConnectStr\":\"def.example.com:2300\",\n" + "     },\n" + "     {\n" + "       \"datacenter\":\"dc3\",\n" + "       \"id\":\"3\",\n" + "       \"zkConnectStr\":\"ghi.example.com:2400\",\n" + "     }\n" + "  ]\n" + "}").withRequiredArg().describedAs("zk_connect_info_path").ofType(String.class);
    ArgumentAcceptingOptionSpec<String> clusterNameOpt = parser.accepts("clusterName", "The cluster name in helix").withRequiredArg().describedAs("cluster_name").ofType(String.class);
    ArgumentAcceptingOptionSpec<String> workflowNameOpt = parser.accepts("workflowName", "The name of the one-time workflow").withRequiredArg().describedAs("workflow_name").ofType(String.class);
    ArgumentAcceptingOptionSpec<Long> recurrentIntervalInMinutesOpt = parser.accepts("recurrentIntervalInMinutes", "The frequency for the recurrent workflow").withOptionalArg().describedAs("recurrent_interval_in_minutes").ofType(Long.class).defaultsTo(Utils.Infinite_Time);
    parser.accepts("delete", "Flag to remove the given workflow from the cluster(s) instead of creating one");
    OptionSet options = parser.parse(args);
    Boolean isDelete = options.has("delete");
    String zkLayoutPath = options.valueOf(zkLayoutPathOpt);
    String clusterName = options.valueOf(clusterNameOpt);
    String workflowName = options.valueOf(workflowNameOpt);
    Long recurrentIntervalInMinutes = options.valueOf(recurrentIntervalInMinutesOpt);
    Map<String, ClusterMapUtils.DcZkInfo> dataCenterToZKAddress = ClusterMapUtils.parseDcJsonAndPopulateDcInfo(Utils.readStringFromFile(zkLayoutPath));
    for (ClusterMapUtils.DcZkInfo zkInfo : dataCenterToZKAddress.values()) {
        String zkAddress = zkInfo.getZkConnectStr();
        ZkClient zkClient = new ZkClient(zkAddress, SESSION_TIMEOUT, CONNECTION_TIMEOUT, new ZNRecordSerializer());
        TaskDriver taskDriver = new TaskDriver(zkClient, clusterName);
        if (isDelete) {
            try {
                taskDriver.stop(workflowName);
                taskDriver.delete(workflowName);
            } catch (Exception | Error e) {
                System.out.println(String.format("Failed to delete %s. Workflow not found in cluster %s at %s", workflowName, clusterName, zkAddress));
            }
        } else {
            try {
                Workflow.Builder workflowBuilder = new Workflow.Builder(workflowName);
                String jobId = ONE_TIME_JOB_ID;
                if (recurrentIntervalInMinutes != Utils.Infinite_Time) {
                    jobId = RECURRENT_JOB_ID;
                    workflowBuilder.setScheduleConfig(ScheduleConfig.recurringFromNow(TimeUnit.MINUTES, recurrentIntervalInMinutes));
                    workflowBuilder.setExpiry(TimeUnit.MINUTES.toMillis(recurrentIntervalInMinutes));
                }
                JobConfig.Builder jobConfigBuilder = new JobConfig.Builder();
                List<TaskConfig> taskConfigs = new ArrayList<>();
                taskConfigs.add(new TaskConfig.Builder().setTaskId(TASK_ID).setCommand(String.format("%s_%s", HelixHealthReportAggregatorTask.TASK_COMMAND_PREFIX, REPORT_NAME)).build());
                jobConfigBuilder.addTaskConfigs(taskConfigs);
                workflowBuilder.addJob(jobId, jobConfigBuilder);
                Workflow workflow = workflowBuilder.build();
                taskDriver.start(workflow);
                System.out.println(String.format("%s_%s started successfully", workflowName, jobId));
            } catch (Exception | Error e) {
                System.out.println(String.format("Failed to start %s in cluster %s at %s", workflowName, clusterName, zkAddress));
            }
        }
    }
}

Example 13 with TaskConfig

use of org.apache.helix.task.TaskConfig in project incubator-gobblin by apache.

the class GobblinHelixJobLauncher method createJob.

/**
 * Create a job from a given batch of {@link WorkUnit}s.
 */
private JobConfig.Builder createJob(List<WorkUnit> workUnits) throws IOException {
    Map<String, TaskConfig> taskConfigMap = Maps.newHashMap();
    try (ParallelRunner stateSerDeRunner = new ParallelRunner(this.stateSerDeRunnerThreads, this.fs)) {
        int multiTaskIdSequence = 0;
        for (WorkUnit workUnit : workUnits) {
            if (workUnit instanceof MultiWorkUnit) {
                workUnit.setId(JobLauncherUtils.newMultiTaskId(this.jobContext.getJobId(), multiTaskIdSequence++));
            }
            addWorkUnit(workUnit, stateSerDeRunner, taskConfigMap);
        }
        Path jobStateFilePath;
        // write the job.state using the state store if present, otherwise serialize directly to the file
        if (this.stateStores.haveJobStateStore()) {
            jobStateFilePath = GobblinClusterUtils.getJobStateFilePath(true, this.appWorkDir, this.jobContext.getJobId());
            this.stateStores.getJobStateStore().put(jobStateFilePath.getParent().getName(), jobStateFilePath.getName(), this.jobContext.getJobState());
        } else {
            jobStateFilePath = GobblinClusterUtils.getJobStateFilePath(false, this.appWorkDir, this.jobContext.getJobId());
            SerializationUtils.serializeState(this.fs, jobStateFilePath, this.jobContext.getJobState());
        }
        LOGGER.debug("GobblinHelixJobLauncher.createJob: jobStateFilePath {}, jobState {} jobProperties {}", jobStateFilePath, this.jobContext.getJobState().toString(), this.jobContext.getJobState().getProperties());
    }
    JobConfig.Builder jobConfigBuilder = new JobConfig.Builder();
    jobConfigBuilder.setMaxAttemptsPerTask(this.jobContext.getJobState().getPropAsInt(ConfigurationKeys.MAX_TASK_RETRIES_KEY, ConfigurationKeys.DEFAULT_MAX_TASK_RETRIES));
    jobConfigBuilder.setFailureThreshold(workUnits.size());
    jobConfigBuilder.addTaskConfigMap(taskConfigMap).setCommand(GobblinTaskRunner.GOBBLIN_TASK_FACTORY_NAME);
    jobConfigBuilder.setNumConcurrentTasksPerInstance(ConfigUtils.getInt(jobConfig, GobblinClusterConfigurationKeys.HELIX_CLUSTER_TASK_CONCURRENCY, GobblinClusterConfigurationKeys.HELIX_CLUSTER_TASK_CONCURRENCY_DEFAULT));
    if (Task.getExecutionModel(ConfigUtils.configToState(jobConfig)).equals(ExecutionModel.STREAMING)) {
        jobConfigBuilder.setRebalanceRunningTask(true);
    }
    return jobConfigBuilder;
}

Example 14 with TaskConfig

use of org.apache.helix.task.TaskConfig in project helix by apache.

the class TestTaskRebalancerParallel method testWhenAllowOverlapJobAssignment.

/**
 * This test starts 4 jobs in job queue, the job all stuck, and verify that
 * (1) the number of running job does not exceed configured max allowed parallel jobs
 * (2) one instance can be assigned to multiple jobs in the workflow when allow overlap assignment
 */
@Test(dependsOnMethods = { "testWhenDisallowOverlapJobAssignment" })
public void testWhenAllowOverlapJobAssignment() throws Exception {
    // Disable all participants except one to enforce all assignment to be on one host
    for (int i = 1; i < _numNodes; i++) {
        _participants[i].syncStop();
    }
    ClusterLiveNodesVerifier verifier = new ClusterLiveNodesVerifier(_gZkClient, CLUSTER_NAME, Collections.singletonList(_participants[0].getInstanceName()));
    Assert.assertTrue(verifier.verify());
    String queueName = TestHelper.getTestMethodName();
    WorkflowConfig.Builder cfgBuilder = new WorkflowConfig.Builder(queueName);
    cfgBuilder.setParallelJobs(PARALLEL_COUNT);
    cfgBuilder.setAllowOverlapJobAssignment(true);
    JobQueue.Builder queueBuild = new JobQueue.Builder(queueName).setWorkflowConfig(cfgBuilder.build());
    JobQueue queue = queueBuild.build();
    _driver.createQueue(queue);
    // Create jobs that can be assigned to any instances
    List<JobConfig.Builder> jobConfigBuilders = new ArrayList<JobConfig.Builder>();
    for (int i = 0; i < PARALLEL_COUNT; i++) {
        List<TaskConfig> taskConfigs = new ArrayList<TaskConfig>();
        for (int j = 0; j < TASK_COUNT; j++) {
            taskConfigs.add(new TaskConfig.Builder().setTaskId("task_" + j).setCommand(MockTask.TASK_COMMAND).build());
        }
        jobConfigBuilders.add(new JobConfig.Builder().addTaskConfigs(taskConfigs));
    }
    _driver.stop(queueName);
    for (int i = 0; i < jobConfigBuilders.size(); ++i) {
        _driver.enqueueJob(queueName, "job_" + (i + 1), jobConfigBuilders.get(i));
    }
    _driver.resume(queueName);
    Thread.sleep(2000);
    Assert.assertTrue(TaskTestUtil.pollForWorkflowParallelState(_driver, queueName));
    for (int i = 1; i < _numNodes; i++) {
        _participants[i].syncStart();
    }
}

Example 15 with TaskConfig

use of org.apache.helix.task.TaskConfig in project helix by apache.

the class TestGenericTaskAssignmentCalculator method beforeClass.

@BeforeClass
public void beforeClass() throws Exception {
    _participants = new MockParticipantManager[_numNodes];
    String namespace = "/" + CLUSTER_NAME;
    if (_gZkClient.exists(namespace)) {
        _gZkClient.deleteRecursively(namespace);
    }
    // Setup cluster and instances
    ClusterSetup setupTool = new ClusterSetup(ZK_ADDR);
    setupTool.addCluster(CLUSTER_NAME, true);
    for (int i = 0; i < _numNodes; i++) {
        String storageNodeName = PARTICIPANT_PREFIX + "_" + (_startPort + i);
        setupTool.addInstanceToCluster(CLUSTER_NAME, storageNodeName);
    }
    // start dummy participants
    for (int i = 0; i < _numNodes; i++) {
        final String instanceName = PARTICIPANT_PREFIX + "_" + (_startPort + i);
        // Set task callbacks
        Map<String, TaskFactory> taskFactoryReg = new HashMap<String, TaskFactory>();
        taskFactoryReg.put("TaskOne", new TaskFactory() {

            @Override
            public Task createNewTask(TaskCallbackContext context) {
                return new TaskOne(context, instanceName);
            }
        });
        _participants[i] = new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, instanceName);
        // Register a Task state model factory.
        StateMachineEngine stateMachine = _participants[i].getStateMachineEngine();
        stateMachine.registerStateModelFactory("Task", new TaskStateModelFactory(_participants[i], taskFactoryReg));
        _participants[i].syncStart();
    }
    // Start controller
    String controllerName = CONTROLLER_PREFIX + "_0";
    _controller = new ClusterControllerManager(ZK_ADDR, CLUSTER_NAME, controllerName);
    _controller.syncStart();
    // Start an admin connection
    _manager = HelixManagerFactory.getZKHelixManager(CLUSTER_NAME, "Admin", InstanceType.ADMINISTRATOR, ZK_ADDR);
    _manager.connect();
    _driver = new TaskDriver(_manager);
    Map<String, String> taskConfigMap = Maps.newHashMap();
    _taskConfig = new TaskConfig("TaskOne", taskConfigMap);
    _jobCommandMap = Maps.newHashMap();
}