Examples with ResubmitWorkerRequest io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ResubmitWorkerRequest used on opensource projects

Example 1 with ResubmitWorkerRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ResubmitWorkerRequest in project mantis by Netflix.

the class JobRoute method getJobRoutes.

private Route getJobRoutes() {
    return route(path(STATUS_ENDPOINT, () -> post(() -> decodeRequest(() -> entity(Unmarshaller.entityToString(), req -> {
        if (logger.isDebugEnabled()) {
            logger.debug("/api/postjobstatus called {}", req);
        }
        try {
            workerHeartbeatStatusPOST.increment();
            PostJobStatusRequest postJobStatusRequest = Jackson.fromJSON(req, PostJobStatusRequest.class);
            WorkerEvent workerStatusRequest = createWorkerStatusRequest(postJobStatusRequest);
            if (workerStatusRequest instanceof WorkerHeartbeat) {
                if (!ConfigurationProvider.getConfig().isHeartbeatProcessingEnabled()) {
                    // skip heartbeat processing
                    if (logger.isTraceEnabled()) {
                        logger.trace("skipped heartbeat event {}", workerStatusRequest);
                    }
                    workerHeartbeatSkipped.increment();
                    return complete(StatusCodes.OK);
                }
            }
            return completeWithFuture(jobRouteHandler.workerStatus(workerStatusRequest).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error handling job status {}", req, e);
            return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"invalid JSON payload to post job status\"}");
        }
    })))), pathPrefix(API_JOBS, () -> route(post(() -> route(path(KILL_ENDPOINT, () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), req -> {
        logger.debug("/api/jobs/kill called {}", req);
        try {
            final KillJobRequest killJobRequest = Jackson.fromJSON(req, KillJobRequest.class);
            return completeWithFuture(jobRouteHandler.kill(killJobRequest).thenApply(resp -> {
                if (resp.responseCode == BaseResponse.ResponseCode.SUCCESS) {
                    return new JobClusterManagerProto.KillJobResponse(resp.requestId, resp.responseCode, resp.getState(), "[\"" + resp.getJobId().getId() + " Killed\"]", resp.getJobId(), resp.getUser());
                } else if (resp.responseCode == BaseResponse.ResponseCode.CLIENT_ERROR) {
                    // for backwards compatibility with old master
                    return new JobClusterManagerProto.KillJobResponse(resp.requestId, BaseResponse.ResponseCode.SUCCESS, resp.getState(), "[\"" + resp.message + " \"]", resp.getJobId(), resp.getUser());
                }
                return resp;
            }).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error on job kill {}", req, e);
            return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"invalid json payload to kill job\"}");
        }
    }))), path(RESUBMIT_WORKER_ENDPOINT, () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), req -> {
        logger.debug("/api/jobs/resubmitWorker called {}", req);
        try {
            final ResubmitWorkerRequest resubmitWorkerRequest = Jackson.fromJSON(req, ResubmitWorkerRequest.class);
            return completeWithFuture(jobRouteHandler.resubmitWorker(resubmitWorkerRequest).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error on worker resubmit {}", req, e);
            return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"invalid json payload to resubmit worker\"}");
        }
    }))), path(SCALE_STAGE_ENDPOINT, () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), req -> {
        logger.debug("/api/jobs/scaleStage called {}", req);
        try {
            ScaleStageRequest scaleStageRequest = Jackson.fromJSON(req, ScaleStageRequest.class);
            int numWorkers = scaleStageRequest.getNumWorkers();
            int maxWorkersPerStage = ConfigurationProvider.getConfig().getMaxWorkersPerStage();
            if (numWorkers > maxWorkersPerStage) {
                logger.warn("rejecting ScaleStageRequest {} with invalid num workers", scaleStageRequest);
                return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"num workers must be less than " + maxWorkersPerStage + "\"}");
            }
            return completeWithFuture(jobRouteHandler.scaleStage(scaleStageRequest).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error scaling stage {}", req, e);
            return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"invalid json payload to scale stage " + e.getMessage() + "\"}");
        }
    }))))), get(() -> route(// - optional labels.op query param - default value is 'or' if not specified (other possible value is 'and'
    path(segment("list"), () -> {
        jobListGET.increment();
        return jobListRoute(Optional.empty());
    }), path(segment("list").slash("matchinglabels"), () -> {
        jobListLabelMatchGET.increment();
        return jobListRoute(Optional.empty());
    }), path(segment("list").slash(PathMatchers.segment()), (jobId) -> {
        logger.debug("/api/jobs/list/{} called", jobId);
        jobListJobIdGET.increment();
        return completeAsync(jobRouteHandler.getJobDetails(new JobClusterManagerProto.GetJobDetailsRequest("masterAPI", jobId)), resp -> {
            Optional<MantisJobMetadataView> mantisJobMetadataView = resp.getJobMetadata().map(metaData -> new MantisJobMetadataView(metaData, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), false));
            return completeOK(mantisJobMetadataView, Jackson.marshaller());
        });
    }), path(segment("list").slash("matching").slash(PathMatchers.segment()), (regex) -> {
        jobListRegexGET.increment();
        return jobListRoute(Optional.ofNullable(regex).filter(r -> !r.isEmpty()));
    }), path(segment("archived").slash(PathMatchers.segment()), (jobId) -> parameterOptional(StringUnmarshallers.INTEGER, "limit", (limit) -> {
        jobArchivedWorkersGET.increment();
        Optional<JobId> jobIdO = JobId.fromId(jobId);
        if (jobIdO.isPresent()) {
            ListArchivedWorkersRequest req = new ListArchivedWorkersRequest(jobIdO.get(), limit.orElse(DEFAULT_LIST_ARCHIVED_WORKERS_LIMIT));
            return alwaysCache(cache, requestUriKeyer, () -> extractUri(uri -> completeAsync(jobRouteHandler.listArchivedWorkers(req), resp -> {
                List<MantisWorkerMetadataWritable> workers = resp.getWorkerMetadata().stream().map(wm -> DataFormatAdapter.convertMantisWorkerMetadataToMantisWorkerMetadataWritable(wm)).collect(Collectors.toList());
                return completeOK(workers, Jackson.marshaller());
            })));
        } else {
            return complete(StatusCodes.BAD_REQUEST, "error: 'archived/<jobId>' request must include a valid jobId");
        }
    })), path(segment("archived"), () -> {
        jobArchivedWorkersGETInvalid.increment();
        return complete(StatusCodes.BAD_REQUEST, "error: 'archived' Request must include jobId");
    }))))));
}

Example 2 with ResubmitWorkerRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ResubmitWorkerRequest in project mantis by Netflix.

the class JobClusterTest method testResubmitWorker.

@Test
public void testResubmitWorker() {
    TestKit probe = new TestKit(system);
    String clusterName = "testResubmitWorker";
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    MantisJobStore jobStoreMock = mock(MantisJobStore.class);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName);
    ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreMock, schedulerMock, eventPublisher));
    jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
    JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
    assertEquals(SUCCESS, createResp.responseCode);
    try {
        final JobDefinition jobDefn = createJob(clusterName, 1, MantisJobDurationType.Transient);
        String jobId = clusterName + "-1";
        JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, jobDefn, jobId);
        JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId, SUCCESS, JobState.Accepted);
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobClusterActor, jobId, 1, new WorkerId(jobId, 0, 1));
        JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId, SUCCESS, JobState.Launched);
        jobClusterActor.tell(new ResubmitWorkerRequest(jobId, 1, user, of("justbecause")), probe.getRef());
        ResubmitWorkerResponse resp = probe.expectMsgClass(ResubmitWorkerResponse.class);
        assertTrue(BaseResponse.ResponseCode.SUCCESS.equals(resp.responseCode));
        jobClusterActor.tell(new GetJobDetailsRequest("nj", JobId.fromId(jobId).get()), probe.getRef());
        GetJobDetailsResponse detailsResp = probe.expectMsgClass(GetJobDetailsResponse.class);
        IMantisWorkerMetadata workerMetadata = detailsResp.getJobMetadata().get().getWorkerByIndex(1, 0).get().getMetadata();
        assertEquals(2, workerMetadata.getWorkerNumber());
        assertEquals(1, workerMetadata.getResubmitOf());
        assertEquals(1, workerMetadata.getTotalResubmitCount());
        JobTestHelper.killJobAndVerify(probe, clusterName, new JobId(clusterName, 1), jobClusterActor);
        verify(jobStoreMock, times(1)).createJobCluster(any());
        verify(jobStoreMock, times(1)).updateJobCluster(any());
        verify(jobStoreMock, times(1)).replaceTerminatedWorker(any(), any());
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        fail();
    }
}

Example 3 with ResubmitWorkerRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ResubmitWorkerRequest in project mantis by Netflix.

the class JobClusterTest method testListArchivedWorkers.

@Test
public void testListArchivedWorkers() {
    TestKit probe = new TestKit(system);
    String clusterName = "testListArchivedWorkers";
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName);
    ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStore, schedulerMock, eventPublisher));
    jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
    JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
    assertEquals(SUCCESS, createResp.responseCode);
    String jobId = clusterName + "-1";
    try {
        final JobDefinition jobDefn = createJob(clusterName, 1, MantisJobDurationType.Transient);
        JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, jobDefn, jobId);
        JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId, SUCCESS, JobState.Accepted);
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobClusterActor, jobId, 1, new WorkerId(jobId, 0, 1));
        JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId, SUCCESS, JobState.Launched);
        jobClusterActor.tell(new ResubmitWorkerRequest(jobId, 1, user, of("justbecause")), probe.getRef());
        ResubmitWorkerResponse resp = probe.expectMsgClass(ResubmitWorkerResponse.class);
        assertTrue(BaseResponse.ResponseCode.SUCCESS.equals(resp.responseCode));
        jobClusterActor.tell(new ListArchivedWorkersRequest(new JobId(clusterName, 1)), probe.getRef());
        ListArchivedWorkersResponse archivedWorkersResponse = probe.expectMsgClass(ListArchivedWorkersResponse.class);
        assertEquals(SUCCESS, archivedWorkersResponse.responseCode);
        assertEquals(1, archivedWorkersResponse.getWorkerMetadata().size());
        IMantisWorkerMetadata archivedWorker = archivedWorkersResponse.getWorkerMetadata().get(0);
        assertEquals(1, archivedWorker.getWorkerNumber());
        assertEquals(0, archivedWorker.getWorkerIndex());
        assertEquals(0, archivedWorker.getResubmitOf());
        JobTestHelper.killJobAndVerify(probe, clusterName, new JobId(clusterName, 1), jobClusterActor);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        fail();
    }
}