Search in sources :

Example 11 with TaskRunning

use of org.apache.kafka.trogdor.rest.TaskRunning in project kafka by apache.

the class CoordinatorTest method testAgentFailureAndTaskExpiry.

/**
 * If an agent fails in the middle of a task and comes back up when the task is considered expired,
 * we want the task to be marked as DONE and not re-sent should a second failure happen.
 */
@Test
public void testAgentFailureAndTaskExpiry() throws Exception {
    MockTime time = new MockTime(0, 0, 0);
    Scheduler scheduler = new MockScheduler(time);
    try (MiniTrogdorCluster cluster = new MiniTrogdorCluster.Builder().addCoordinator("node01").addAgent("node02").scheduler(scheduler).build()) {
        CoordinatorClient coordinatorClient = cluster.coordinatorClient();
        NoOpTaskSpec fooSpec = new NoOpTaskSpec(1, 500);
        coordinatorClient.createTask(new CreateTaskRequest("foo", fooSpec));
        TaskState expectedState = new ExpectedTaskBuilder("foo").taskState(new TaskPending(fooSpec)).build().taskState();
        TaskState resp = coordinatorClient.task(new TaskRequest("foo"));
        assertEquals(expectedState, resp);
        time.sleep(2);
        new ExpectedTasks().addTask(new ExpectedTaskBuilder("foo").taskState(new TaskRunning(fooSpec, 2, new TextNode("active"))).workerState(new WorkerRunning("foo", fooSpec, 2, new TextNode("active"))).build()).waitFor(coordinatorClient).waitFor(cluster.agentClient("node02"));
        cluster.restartAgent("node02");
        time.sleep(550);
        // coordinator heartbeat sees that the agent is back up, re-schedules the task but the agent expires it
        new ExpectedTasks().addTask(new ExpectedTaskBuilder("foo").taskState(new TaskDone(fooSpec, 2, 552, "worker expired", false, null)).workerState(new WorkerDone("foo", fooSpec, 552, 552, null, "worker expired")).build()).waitFor(coordinatorClient).waitFor(cluster.agentClient("node02"));
        cluster.restartAgent("node02");
        // coordinator heartbeat sees that the agent is back up but does not re-schedule the task as it is DONE
        new ExpectedTasks().addTask(new ExpectedTaskBuilder("foo").taskState(new TaskDone(fooSpec, 2, 552, "worker expired", false, null)).build()).waitFor(coordinatorClient).waitFor(cluster.agentClient("node02"));
    }
}
Also used : MockScheduler(org.apache.kafka.common.utils.MockScheduler) WorkerDone(org.apache.kafka.trogdor.rest.WorkerDone) WorkerRunning(org.apache.kafka.trogdor.rest.WorkerRunning) MockScheduler(org.apache.kafka.common.utils.MockScheduler) Scheduler(org.apache.kafka.common.utils.Scheduler) ExpectedTaskBuilder(org.apache.kafka.trogdor.common.ExpectedTasks.ExpectedTaskBuilder) TaskDone(org.apache.kafka.trogdor.rest.TaskDone) TextNode(com.fasterxml.jackson.databind.node.TextNode) DestroyTaskRequest(org.apache.kafka.trogdor.rest.DestroyTaskRequest) StopTaskRequest(org.apache.kafka.trogdor.rest.StopTaskRequest) CreateTaskRequest(org.apache.kafka.trogdor.rest.CreateTaskRequest) TaskRequest(org.apache.kafka.trogdor.rest.TaskRequest) ExpectedTaskBuilder(org.apache.kafka.trogdor.common.ExpectedTasks.ExpectedTaskBuilder) NoOpTaskSpec(org.apache.kafka.trogdor.task.NoOpTaskSpec) TaskPending(org.apache.kafka.trogdor.rest.TaskPending) ExpectedTasks(org.apache.kafka.trogdor.common.ExpectedTasks) CreateTaskRequest(org.apache.kafka.trogdor.rest.CreateTaskRequest) MiniTrogdorCluster(org.apache.kafka.trogdor.common.MiniTrogdorCluster) TaskRunning(org.apache.kafka.trogdor.rest.TaskRunning) TaskState(org.apache.kafka.trogdor.rest.TaskState) MockTime(org.apache.kafka.common.utils.MockTime) Test(org.junit.jupiter.api.Test)

Example 12 with TaskRunning

use of org.apache.kafka.trogdor.rest.TaskRunning in project kafka by apache.

the class CoordinatorTest method testTaskDistribution.

@Test
public void testTaskDistribution() throws Exception {
    MockTime time = new MockTime(0, 0, 0);
    Scheduler scheduler = new MockScheduler(time);
    try (MiniTrogdorCluster cluster = new MiniTrogdorCluster.Builder().addCoordinator("node01").addAgent("node01").addAgent("node02").scheduler(scheduler).build()) {
        CoordinatorClient coordinatorClient = cluster.coordinatorClient();
        AgentClient agentClient1 = cluster.agentClient("node01");
        AgentClient agentClient2 = cluster.agentClient("node02");
        new ExpectedTasks().waitFor(coordinatorClient).waitFor(agentClient1).waitFor(agentClient2);
        NoOpTaskSpec fooSpec = new NoOpTaskSpec(5, 7);
        coordinatorClient.createTask(new CreateTaskRequest("foo", fooSpec));
        new ExpectedTasks().addTask(new ExpectedTaskBuilder("foo").taskState(new TaskPending(fooSpec)).build()).waitFor(coordinatorClient).waitFor(agentClient1).waitFor(agentClient2);
        time.sleep(11);
        ObjectNode status1 = new ObjectNode(JsonNodeFactory.instance);
        status1.set("node01", new TextNode("active"));
        status1.set("node02", new TextNode("active"));
        new ExpectedTasks().addTask(new ExpectedTaskBuilder("foo").taskState(new TaskRunning(fooSpec, 11, status1)).workerState(new WorkerRunning("foo", fooSpec, 11, new TextNode("active"))).build()).waitFor(coordinatorClient).waitFor(agentClient1).waitFor(agentClient2);
        time.sleep(7);
        ObjectNode status2 = new ObjectNode(JsonNodeFactory.instance);
        status2.set("node01", new TextNode("done"));
        status2.set("node02", new TextNode("done"));
        new ExpectedTasks().addTask(new ExpectedTaskBuilder("foo").taskState(new TaskDone(fooSpec, 11, 18, "", false, status2)).workerState(new WorkerDone("foo", fooSpec, 11, 18, new TextNode("done"), "")).build()).waitFor(coordinatorClient).waitFor(agentClient1).waitFor(agentClient2);
    }
}
Also used : MockScheduler(org.apache.kafka.common.utils.MockScheduler) WorkerDone(org.apache.kafka.trogdor.rest.WorkerDone) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) WorkerRunning(org.apache.kafka.trogdor.rest.WorkerRunning) MockScheduler(org.apache.kafka.common.utils.MockScheduler) Scheduler(org.apache.kafka.common.utils.Scheduler) TaskDone(org.apache.kafka.trogdor.rest.TaskDone) TextNode(com.fasterxml.jackson.databind.node.TextNode) ExpectedTaskBuilder(org.apache.kafka.trogdor.common.ExpectedTasks.ExpectedTaskBuilder) NoOpTaskSpec(org.apache.kafka.trogdor.task.NoOpTaskSpec) TaskPending(org.apache.kafka.trogdor.rest.TaskPending) ExpectedTasks(org.apache.kafka.trogdor.common.ExpectedTasks) CreateTaskRequest(org.apache.kafka.trogdor.rest.CreateTaskRequest) MiniTrogdorCluster(org.apache.kafka.trogdor.common.MiniTrogdorCluster) AgentClient(org.apache.kafka.trogdor.agent.AgentClient) TaskRunning(org.apache.kafka.trogdor.rest.TaskRunning) MockTime(org.apache.kafka.common.utils.MockTime) Test(org.junit.jupiter.api.Test)

Example 13 with TaskRunning

use of org.apache.kafka.trogdor.rest.TaskRunning in project kafka by apache.

the class CoordinatorClientTest method testPrettyPrintTaskInfo.

@Test
public void testPrettyPrintTaskInfo() {
    assertEquals("Will start at 2019-01-08T07:05:59.85Z", CoordinatorClient.prettyPrintTaskInfo(new TaskPending(new NoOpTaskSpec(1546931159850L, 9000)), ZoneOffset.UTC));
    assertEquals("Started 2009-07-07T01:45:59.85Z; will stop after 9s", CoordinatorClient.prettyPrintTaskInfo(new TaskRunning(new NoOpTaskSpec(1146931159850L, 9000), 1246931159850L, JsonNodeFactory.instance.objectNode()), ZoneOffset.UTC));
    assertEquals("Started 2009-07-07T01:45:59.85Z", CoordinatorClient.prettyPrintTaskInfo(new TaskStopping(new NoOpTaskSpec(1146931159850L, 9000), 1246931159850L, JsonNodeFactory.instance.objectNode()), ZoneOffset.UTC));
    assertEquals("FINISHED at 2019-01-08T20:59:29.85Z after 10s", CoordinatorClient.prettyPrintTaskInfo(new TaskDone(new NoOpTaskSpec(0, 1000), 1546981159850L, 1546981169850L, "", false, JsonNodeFactory.instance.objectNode()), ZoneOffset.UTC));
    assertEquals("CANCELLED at 2019-01-08T20:59:29.85Z after 10s", CoordinatorClient.prettyPrintTaskInfo(new TaskDone(new NoOpTaskSpec(0, 1000), 1546981159850L, 1546981169850L, "", true, JsonNodeFactory.instance.objectNode()), ZoneOffset.UTC));
    assertEquals("FAILED at 2019-01-08T20:59:29.85Z after 10s", CoordinatorClient.prettyPrintTaskInfo(new TaskDone(new NoOpTaskSpec(0, 1000), 1546981159850L, 1546981169850L, "foobar", true, JsonNodeFactory.instance.objectNode()), ZoneOffset.UTC));
}
Also used : TaskPending(org.apache.kafka.trogdor.rest.TaskPending) TaskDone(org.apache.kafka.trogdor.rest.TaskDone) TaskRunning(org.apache.kafka.trogdor.rest.TaskRunning) NoOpTaskSpec(org.apache.kafka.trogdor.task.NoOpTaskSpec) TaskStopping(org.apache.kafka.trogdor.rest.TaskStopping) Test(org.junit.jupiter.api.Test)

Aggregations

TaskRunning (org.apache.kafka.trogdor.rest.TaskRunning)13 MockScheduler (org.apache.kafka.common.utils.MockScheduler)12 MockTime (org.apache.kafka.common.utils.MockTime)12 Scheduler (org.apache.kafka.common.utils.Scheduler)12 ExpectedTaskBuilder (org.apache.kafka.trogdor.common.ExpectedTasks.ExpectedTaskBuilder)12 MiniTrogdorCluster (org.apache.kafka.trogdor.common.MiniTrogdorCluster)12 CreateTaskRequest (org.apache.kafka.trogdor.rest.CreateTaskRequest)12 TaskPending (org.apache.kafka.trogdor.rest.TaskPending)12 NoOpTaskSpec (org.apache.kafka.trogdor.task.NoOpTaskSpec)12 ExpectedTasks (org.apache.kafka.trogdor.common.ExpectedTasks)11 WorkerRunning (org.apache.kafka.trogdor.rest.WorkerRunning)10 Test (org.junit.jupiter.api.Test)10 TextNode (com.fasterxml.jackson.databind.node.TextNode)9 TaskDone (org.apache.kafka.trogdor.rest.TaskDone)9 WorkerDone (org.apache.kafka.trogdor.rest.WorkerDone)6 AgentClient (org.apache.kafka.trogdor.agent.AgentClient)5 DestroyTaskRequest (org.apache.kafka.trogdor.rest.DestroyTaskRequest)5 StopTaskRequest (org.apache.kafka.trogdor.rest.StopTaskRequest)5 ObjectNode (com.fasterxml.jackson.databind.node.ObjectNode)4 TaskRequest (org.apache.kafka.trogdor.rest.TaskRequest)3