use of io.mantisrx.server.master.store.MantisWorkerMetadataWritable in project mantis by Netflix.
the class JobRouteTest method testJobClusterGetJobsList.
@Test(dependsOnMethods = { "testJobClusterGetJobIds" })
public void testJobClusterGetJobsList() throws InterruptedException {
final CountDownLatch latch = new CountDownLatch(1);
final CompletionStage<HttpResponse> responseFuture = http.singleRequest(HttpRequest.GET(jobAPIEndpoint("list")));
responseFuture.thenCompose(r -> processRespFut(r, Optional.of(200))).whenComplete((msg, t) -> {
String responseMessage = getResponseMessage(msg, t);
logger.info("got response---> {}", responseMessage);
List<MantisJobMetadataView> jobInfos = Collections.emptyList();
try {
jobInfos = Jackson.fromJSON(responseMessage, new TypeReference<List<MantisJobMetadataView>>() {
});
} catch (IOException e) {
logger.error("failed to deser json {}", responseMessage, e);
fail("job list deser failed");
}
logger.info("jobInfos---> {}", jobInfos);
assertEquals(1, jobInfos.size());
MantisJobMetadataView mjm = jobInfos.get(0);
assertEquals(mjm.getJobMetadata().getJobId(), "sine-function-1");
assertEquals(mjm.getJobMetadata().getName(), "sine-function");
assertTrue(mjm.getStageMetadataList().size() > 0);
MantisStageMetadataWritable msm = mjm.getStageMetadataList().get(0);
assertEquals(1, msm.getNumWorkers());
assertTrue(mjm.getWorkerMetadataList().size() > 0);
MantisWorkerMetadataWritable mwm = mjm.getWorkerMetadataList().get(0);
assertEquals("sine-function-1", mwm.getJobId());
assertEquals(false, mwm.getCluster().isPresent());
latch.countDown();
});
assertTrue(latch.await(2, TimeUnit.SECONDS));
}
use of io.mantisrx.server.master.store.MantisWorkerMetadataWritable in project mantis by Netflix.
the class JacksonTest method testDeser4.
@Test
public void testDeser4() throws IOException {
final ObjectMapper objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
List<MantisJobMetadataView> jobIdInfos = Jackson.fromJSON(objectMapper, "[{\"jobMetadata\":{\"jobId\":\"sine-function-1\",\"name\":\"sine-function\"," + "\"user\":\"nmahilani\",\"submittedAt\":1527703650220,\"jarUrl\":\"https://mantis.staging.us-east-1.prod.netflix.net/mantis-artifacts/mantis-examples-sine-function-0.2.9.zip\"," + "\"numStages\":2,\"sla\":{\"runtimeLimitSecs\":0,\"minRuntimeSecs\":0,\"slaType\":\"Lossy\",\"durationType\":\"Perpetual\",\"userProvidedType\":\"\"}," + "\"state\":\"Accepted\",\"subscriptionTimeoutSecs\":0,\"parameters\":[{\"name\":\"useRandom\",\"value\":\"True\"}],\"nextWorkerNumberToUse\":11," + "\"migrationConfig\":{\"strategy\":\"PERCENTAGE\",\"configString\":\"{\\\"percentToMove\\\":25,\\\"intervalMs\\\":60000}\"}," + "\"labels\":[{\"name\":\"_mantis.user\",\"value\":\"nmahilani\"},{\"name\":\"_mantis.ownerEmail\",\"value\":\"nmahilani@netflix.com\"}," + "{\"name\":\"_mantis.jobType\",\"value\":\"other\"},{\"name\":\"_mantis.criticality\",\"value\":\"low\"},{\"name\":\"_mantis.artifact.version\",\"value\":\"0.2.9\"}]}," + "\"stageMetadataList\":[{\"jobId\":\"sine-function-1\",\"stageNum\":0,\"numStages\":2,\"machineDefinition\":{\"cpuCores\":1.0,\"memoryMB\":200.0,\"networkMbps\":128.0,\"diskMB\":1024.0,\"numPorts\":1}," + "\"numWorkers\":1,\"hardConstraints\":null,\"softConstraints\":null,\"scalingPolicy\":null,\"scalable\":false}," + "{\"jobId\":\"sine-function-1\",\"stageNum\":1,\"numStages\":2,\"machineDefinition\":{\"cpuCores\":1.0,\"memoryMB\":200.0,\"networkMbps\":128.0,\"diskMB\":1024.0,\"numPorts\":1},\"numWorkers\":1,\"hardConstraints\":[],\"softConstraints\":[\"M4Cluster\"]," + "\"scalingPolicy\":{\"stage\":1,\"min\":1,\"max\":10,\"increment\":2,\"decrement\":1,\"coolDownSecs\":600," + "\"strategies\":{\"CPU\":{\"reason\":\"CPU\",\"scaleDownBelowPct\":15.0,\"scaleUpAbovePct\":75.0,\"rollingCount\":{\"count\":12,\"of\":20}}},\"enabled\":true},\"scalable\":true}]," + "\"workerMetadataList\":[{\"workerIndex\":0,\"workerNumber\":2,\"jobId\":\"sine-function-1\",\"stageNum\":0,\"numberOfPorts\":4,\"metricsPort\":0,\"consolePort\":0," + "\"debugPort\":-1,\"ports\":[],\"state\":\"Accepted\",\"slave\":null,\"slaveID\":null,\"cluster\":{\"present\":false},\"acceptedAt\":1527703650231,\"launchedAt\":0,\"startingAt\":0,\"startedAt\":0," + "\"completedAt\":0,\"reason\":null,\"resubmitOf\":-1,\"totalResubmitCount\":0},{\"workerIndex\":0,\"workerNumber\":3,\"jobId\":\"sine-function-1\",\"stageNum\":1,\"numberOfPorts\":4,\"metricsPort\":0,\"consolePort\":0,\"debugPort\":-1,\"ports\":[],\"state\":\"Accepted\"," + "\"slave\":null,\"slaveID\":null,\"cluster\":{\"present\":false},\"acceptedAt\":1527703650232,\"launchedAt\":0,\"startingAt\":0,\"startedAt\":0,\"completedAt\":0," + "\"reason\":null,\"resubmitOf\":-1,\"totalResubmitCount\":0}]}]", new TypeReference<List<MantisJobMetadataView>>() {
});
System.out.println(jobIdInfos);
MantisWorkerMetadataWritable mwm = jobIdInfos.get(0).getWorkerMetadataList().get(0);
mwm.setCluster(Optional.ofNullable("test"));
System.out.println(objectMapper.writer(Jackson.DEFAULT_FILTER_PROVIDER).writeValueAsString(mwm));
}
use of io.mantisrx.server.master.store.MantisWorkerMetadataWritable in project mantis by Netflix.
the class NoOpMantisJobOperations method convertMantisWorkerMetadataToMantisWorkerMetadataWritable.
public static MantisWorkerMetadataWritable convertMantisWorkerMetadataToMantisWorkerMetadataWritable(IMantisWorkerMetadata workerMeta) {
MantisWorkerMetadataWritable writable = new MantisWorkerMetadataWritable(workerMeta.getWorkerIndex(), workerMeta.getWorkerNumber(), workerMeta.getJobId(), workerMeta.getStageNum(), workerMeta.getNumberOfPorts());
setWorkerMetadataWritable(writable, workerMeta);
return writable;
}
use of io.mantisrx.server.master.store.MantisWorkerMetadataWritable in project mantis by Netflix.
the class JobRoute method getJobRoutes.
private Route getJobRoutes() {
return route(path(STATUS_ENDPOINT, () -> post(() -> decodeRequest(() -> entity(Unmarshaller.entityToString(), req -> {
if (logger.isDebugEnabled()) {
logger.debug("/api/postjobstatus called {}", req);
}
try {
workerHeartbeatStatusPOST.increment();
PostJobStatusRequest postJobStatusRequest = Jackson.fromJSON(req, PostJobStatusRequest.class);
WorkerEvent workerStatusRequest = createWorkerStatusRequest(postJobStatusRequest);
if (workerStatusRequest instanceof WorkerHeartbeat) {
if (!ConfigurationProvider.getConfig().isHeartbeatProcessingEnabled()) {
// skip heartbeat processing
if (logger.isTraceEnabled()) {
logger.trace("skipped heartbeat event {}", workerStatusRequest);
}
workerHeartbeatSkipped.increment();
return complete(StatusCodes.OK);
}
}
return completeWithFuture(jobRouteHandler.workerStatus(workerStatusRequest).thenApply(this::toHttpResponse));
} catch (IOException e) {
logger.warn("Error handling job status {}", req, e);
return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"invalid JSON payload to post job status\"}");
}
})))), pathPrefix(API_JOBS, () -> route(post(() -> route(path(KILL_ENDPOINT, () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), req -> {
logger.debug("/api/jobs/kill called {}", req);
try {
final KillJobRequest killJobRequest = Jackson.fromJSON(req, KillJobRequest.class);
return completeWithFuture(jobRouteHandler.kill(killJobRequest).thenApply(resp -> {
if (resp.responseCode == BaseResponse.ResponseCode.SUCCESS) {
return new JobClusterManagerProto.KillJobResponse(resp.requestId, resp.responseCode, resp.getState(), "[\"" + resp.getJobId().getId() + " Killed\"]", resp.getJobId(), resp.getUser());
} else if (resp.responseCode == BaseResponse.ResponseCode.CLIENT_ERROR) {
// for backwards compatibility with old master
return new JobClusterManagerProto.KillJobResponse(resp.requestId, BaseResponse.ResponseCode.SUCCESS, resp.getState(), "[\"" + resp.message + " \"]", resp.getJobId(), resp.getUser());
}
return resp;
}).thenApply(this::toHttpResponse));
} catch (IOException e) {
logger.warn("Error on job kill {}", req, e);
return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"invalid json payload to kill job\"}");
}
}))), path(RESUBMIT_WORKER_ENDPOINT, () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), req -> {
logger.debug("/api/jobs/resubmitWorker called {}", req);
try {
final ResubmitWorkerRequest resubmitWorkerRequest = Jackson.fromJSON(req, ResubmitWorkerRequest.class);
return completeWithFuture(jobRouteHandler.resubmitWorker(resubmitWorkerRequest).thenApply(this::toHttpResponse));
} catch (IOException e) {
logger.warn("Error on worker resubmit {}", req, e);
return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"invalid json payload to resubmit worker\"}");
}
}))), path(SCALE_STAGE_ENDPOINT, () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), req -> {
logger.debug("/api/jobs/scaleStage called {}", req);
try {
ScaleStageRequest scaleStageRequest = Jackson.fromJSON(req, ScaleStageRequest.class);
int numWorkers = scaleStageRequest.getNumWorkers();
int maxWorkersPerStage = ConfigurationProvider.getConfig().getMaxWorkersPerStage();
if (numWorkers > maxWorkersPerStage) {
logger.warn("rejecting ScaleStageRequest {} with invalid num workers", scaleStageRequest);
return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"num workers must be less than " + maxWorkersPerStage + "\"}");
}
return completeWithFuture(jobRouteHandler.scaleStage(scaleStageRequest).thenApply(this::toHttpResponse));
} catch (IOException e) {
logger.warn("Error scaling stage {}", req, e);
return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"invalid json payload to scale stage " + e.getMessage() + "\"}");
}
}))))), get(() -> route(// - optional labels.op query param - default value is 'or' if not specified (other possible value is 'and'
path(segment("list"), () -> {
jobListGET.increment();
return jobListRoute(Optional.empty());
}), path(segment("list").slash("matchinglabels"), () -> {
jobListLabelMatchGET.increment();
return jobListRoute(Optional.empty());
}), path(segment("list").slash(PathMatchers.segment()), (jobId) -> {
logger.debug("/api/jobs/list/{} called", jobId);
jobListJobIdGET.increment();
return completeAsync(jobRouteHandler.getJobDetails(new JobClusterManagerProto.GetJobDetailsRequest("masterAPI", jobId)), resp -> {
Optional<MantisJobMetadataView> mantisJobMetadataView = resp.getJobMetadata().map(metaData -> new MantisJobMetadataView(metaData, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), false));
return completeOK(mantisJobMetadataView, Jackson.marshaller());
});
}), path(segment("list").slash("matching").slash(PathMatchers.segment()), (regex) -> {
jobListRegexGET.increment();
return jobListRoute(Optional.ofNullable(regex).filter(r -> !r.isEmpty()));
}), path(segment("archived").slash(PathMatchers.segment()), (jobId) -> parameterOptional(StringUnmarshallers.INTEGER, "limit", (limit) -> {
jobArchivedWorkersGET.increment();
Optional<JobId> jobIdO = JobId.fromId(jobId);
if (jobIdO.isPresent()) {
ListArchivedWorkersRequest req = new ListArchivedWorkersRequest(jobIdO.get(), limit.orElse(DEFAULT_LIST_ARCHIVED_WORKERS_LIMIT));
return alwaysCache(cache, requestUriKeyer, () -> extractUri(uri -> completeAsync(jobRouteHandler.listArchivedWorkers(req), resp -> {
List<MantisWorkerMetadataWritable> workers = resp.getWorkerMetadata().stream().map(wm -> DataFormatAdapter.convertMantisWorkerMetadataToMantisWorkerMetadataWritable(wm)).collect(Collectors.toList());
return completeOK(workers, Jackson.marshaller());
})));
} else {
return complete(StatusCodes.BAD_REQUEST, "error: 'archived/<jobId>' request must include a valid jobId");
}
})), path(segment("archived"), () -> {
jobArchivedWorkersGETInvalid.increment();
return complete(StatusCodes.BAD_REQUEST, "error: 'archived' Request must include jobId");
}))))));
}
use of io.mantisrx.server.master.store.MantisWorkerMetadataWritable in project mantis by Netflix.
the class JobRouteTest method testJobClusterGetJobDetail.
@Test(dependsOnMethods = { "testJobClusterGetJobsList" })
public void testJobClusterGetJobDetail() throws InterruptedException {
final CountDownLatch latch = new CountDownLatch(1);
final CompletionStage<HttpResponse> responseFuture = http.singleRequest(HttpRequest.GET(jobAPIEndpoint("list/sine-function-1")));
responseFuture.thenCompose(r -> processRespFut(r, Optional.of(200))).whenComplete((msg, t) -> {
String responseMessage = getResponseMessage(msg, t);
logger.info("got response---> {}", responseMessage);
MantisJobMetadataView mjm = null;
try {
mjm = Jackson.fromJSON(responseMessage, MantisJobMetadataView.class);
} catch (IOException e) {
logger.error("failed to deser json {}", responseMessage, e);
fail("job info deser failed");
}
logger.info("jobInfo---> {}", mjm);
assertNotNull(mjm);
assertEquals(mjm.getJobMetadata().getJobId(), "sine-function-1");
assertEquals(mjm.getJobMetadata().getName(), "sine-function");
assertTrue(mjm.getStageMetadataList().size() > 0);
MantisStageMetadataWritable msm = mjm.getStageMetadataList().get(0);
assertEquals(1, msm.getNumWorkers());
assertTrue(mjm.getWorkerMetadataList().size() > 0);
MantisWorkerMetadataWritable mwm = mjm.getWorkerMetadataList().get(0);
assertEquals("sine-function-1", mwm.getJobId());
assertEquals(false, mwm.getCluster().isPresent());
latch.countDown();
});
assertTrue(latch.await(2, TimeUnit.SECONDS));
}
Aggregations