use of io.mantisrx.master.jobcluster.job.MantisJobMetadataView in project mantis by Netflix.
the class JobRouteTest method testJobClusterGetJobsList.
@Test(dependsOnMethods = { "testJobClusterGetJobIds" })
public void testJobClusterGetJobsList() throws InterruptedException {
final CountDownLatch latch = new CountDownLatch(1);
final CompletionStage<HttpResponse> responseFuture = http.singleRequest(HttpRequest.GET(jobAPIEndpoint("list")));
responseFuture.thenCompose(r -> processRespFut(r, Optional.of(200))).whenComplete((msg, t) -> {
String responseMessage = getResponseMessage(msg, t);
logger.info("got response---> {}", responseMessage);
List<MantisJobMetadataView> jobInfos = Collections.emptyList();
try {
jobInfos = Jackson.fromJSON(responseMessage, new TypeReference<List<MantisJobMetadataView>>() {
});
} catch (IOException e) {
logger.error("failed to deser json {}", responseMessage, e);
fail("job list deser failed");
}
logger.info("jobInfos---> {}", jobInfos);
assertEquals(1, jobInfos.size());
MantisJobMetadataView mjm = jobInfos.get(0);
assertEquals(mjm.getJobMetadata().getJobId(), "sine-function-1");
assertEquals(mjm.getJobMetadata().getName(), "sine-function");
assertTrue(mjm.getStageMetadataList().size() > 0);
MantisStageMetadataWritable msm = mjm.getStageMetadataList().get(0);
assertEquals(1, msm.getNumWorkers());
assertTrue(mjm.getWorkerMetadataList().size() > 0);
MantisWorkerMetadataWritable mwm = mjm.getWorkerMetadataList().get(0);
assertEquals("sine-function-1", mwm.getJobId());
assertEquals(false, mwm.getCluster().isPresent());
latch.countDown();
});
assertTrue(latch.await(2, TimeUnit.SECONDS));
}
use of io.mantisrx.master.jobcluster.job.MantisJobMetadataView in project mantis by Netflix.
the class JacksonTest method testDeser4.
@Test
public void testDeser4() throws IOException {
final ObjectMapper objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
List<MantisJobMetadataView> jobIdInfos = Jackson.fromJSON(objectMapper, "[{\"jobMetadata\":{\"jobId\":\"sine-function-1\",\"name\":\"sine-function\"," + "\"user\":\"nmahilani\",\"submittedAt\":1527703650220,\"jarUrl\":\"https://mantis.staging.us-east-1.prod.netflix.net/mantis-artifacts/mantis-examples-sine-function-0.2.9.zip\"," + "\"numStages\":2,\"sla\":{\"runtimeLimitSecs\":0,\"minRuntimeSecs\":0,\"slaType\":\"Lossy\",\"durationType\":\"Perpetual\",\"userProvidedType\":\"\"}," + "\"state\":\"Accepted\",\"subscriptionTimeoutSecs\":0,\"parameters\":[{\"name\":\"useRandom\",\"value\":\"True\"}],\"nextWorkerNumberToUse\":11," + "\"migrationConfig\":{\"strategy\":\"PERCENTAGE\",\"configString\":\"{\\\"percentToMove\\\":25,\\\"intervalMs\\\":60000}\"}," + "\"labels\":[{\"name\":\"_mantis.user\",\"value\":\"nmahilani\"},{\"name\":\"_mantis.ownerEmail\",\"value\":\"nmahilani@netflix.com\"}," + "{\"name\":\"_mantis.jobType\",\"value\":\"other\"},{\"name\":\"_mantis.criticality\",\"value\":\"low\"},{\"name\":\"_mantis.artifact.version\",\"value\":\"0.2.9\"}]}," + "\"stageMetadataList\":[{\"jobId\":\"sine-function-1\",\"stageNum\":0,\"numStages\":2,\"machineDefinition\":{\"cpuCores\":1.0,\"memoryMB\":200.0,\"networkMbps\":128.0,\"diskMB\":1024.0,\"numPorts\":1}," + "\"numWorkers\":1,\"hardConstraints\":null,\"softConstraints\":null,\"scalingPolicy\":null,\"scalable\":false}," + "{\"jobId\":\"sine-function-1\",\"stageNum\":1,\"numStages\":2,\"machineDefinition\":{\"cpuCores\":1.0,\"memoryMB\":200.0,\"networkMbps\":128.0,\"diskMB\":1024.0,\"numPorts\":1},\"numWorkers\":1,\"hardConstraints\":[],\"softConstraints\":[\"M4Cluster\"]," + "\"scalingPolicy\":{\"stage\":1,\"min\":1,\"max\":10,\"increment\":2,\"decrement\":1,\"coolDownSecs\":600," + "\"strategies\":{\"CPU\":{\"reason\":\"CPU\",\"scaleDownBelowPct\":15.0,\"scaleUpAbovePct\":75.0,\"rollingCount\":{\"count\":12,\"of\":20}}},\"enabled\":true},\"scalable\":true}]," + "\"workerMetadataList\":[{\"workerIndex\":0,\"workerNumber\":2,\"jobId\":\"sine-function-1\",\"stageNum\":0,\"numberOfPorts\":4,\"metricsPort\":0,\"consolePort\":0," + "\"debugPort\":-1,\"ports\":[],\"state\":\"Accepted\",\"slave\":null,\"slaveID\":null,\"cluster\":{\"present\":false},\"acceptedAt\":1527703650231,\"launchedAt\":0,\"startingAt\":0,\"startedAt\":0," + "\"completedAt\":0,\"reason\":null,\"resubmitOf\":-1,\"totalResubmitCount\":0},{\"workerIndex\":0,\"workerNumber\":3,\"jobId\":\"sine-function-1\",\"stageNum\":1,\"numberOfPorts\":4,\"metricsPort\":0,\"consolePort\":0,\"debugPort\":-1,\"ports\":[],\"state\":\"Accepted\"," + "\"slave\":null,\"slaveID\":null,\"cluster\":{\"present\":false},\"acceptedAt\":1527703650232,\"launchedAt\":0,\"startingAt\":0,\"startedAt\":0,\"completedAt\":0," + "\"reason\":null,\"resubmitOf\":-1,\"totalResubmitCount\":0}]}]", new TypeReference<List<MantisJobMetadataView>>() {
});
System.out.println(jobIdInfos);
MantisWorkerMetadataWritable mwm = jobIdInfos.get(0).getWorkerMetadataList().get(0);
mwm.setCluster(Optional.ofNullable("test"));
System.out.println(objectMapper.writer(Jackson.DEFAULT_FILTER_PROVIDER).writeValueAsString(mwm));
}
use of io.mantisrx.master.jobcluster.job.MantisJobMetadataView in project mantis by Netflix.
the class JobClusterActor method onJobList.
@Override
public void onJobList(final ListJobsRequest request) {
if (logger.isDebugEnabled()) {
logger.info("Entering JCA:onJobList");
}
final ActorRef sender = getSender();
final ActorRef self = getSelf();
Set<JobId> jobIdsFilteredByLabelsSet = new HashSet<>();
// If labels criterion is given prefilter by labels
if (!request.getCriteria().getMatchingLabels().isEmpty()) {
jobIdsFilteredByLabelsSet = jobManager.getJobsMatchingLabels(request.getCriteria().getMatchingLabels(), request.getCriteria().getLabelsOperand());
// Found no jobs matching labels exit
if (jobIdsFilteredByLabelsSet.isEmpty()) {
if (logger.isTraceEnabled()) {
logger.trace("Exit JCA:onJobList {}", jobIdsFilteredByLabelsSet.size());
}
sender.tell(new ListJobsResponse(request.requestId, SUCCESS, "", new ArrayList<>()), self);
return;
}
}
// Found jobs matching labels or no labels criterion given.
// Apply additional criterion to both active and completed jobs
getFilteredNonTerminalJobList(request.getCriteria(), jobIdsFilteredByLabelsSet).mergeWith(getFilteredTerminalJobList(request.getCriteria(), jobIdsFilteredByLabelsSet)).collect(() -> Lists.<MantisJobMetadataView>newArrayList(), List::add).doOnNext(resultList -> {
if (logger.isTraceEnabled()) {
logger.trace("Exit JCA:onJobList {}", resultList.size());
}
sender.tell(new ListJobsResponse(request.requestId, SUCCESS, "", resultList), self);
}).subscribe();
}
use of io.mantisrx.master.jobcluster.job.MantisJobMetadataView in project mantis by Netflix.
the class JobClusterActor method getFilteredTerminalJobList.
/**
* JobState ActiveOnly Execute?
* None None Y
* None TRUE N
* None FALSE Y
* Active None N
* Active TRUE N
* Active FALSE N
* Terminal None Y
* Terminal TRUE Y
* Terminal FALSE Y
* @param request
* @return
*/
private Observable<MantisJobMetadataView> getFilteredTerminalJobList(ListJobCriteria request, Set<JobId> jobIdSet) {
if (logger.isTraceEnabled()) {
logger.trace("JobClusterActor:getFilteredTerminalJobList");
}
if ((request.getJobState().isPresent() && !request.getJobState().get().equals(JobState.MetaState.Terminal))) {
if (logger.isTraceEnabled()) {
logger.trace("Exit JobClusterActor:getFilteredTerminalJobList with empty");
}
return Observable.empty();
} else if (!request.getJobState().isPresent() && (request.getActiveOnly().isPresent() && request.getActiveOnly().get())) {
if (logger.isTraceEnabled()) {
logger.trace("Exit JobClusterActor:getFilteredTerminalJobList with empty");
}
return Observable.empty();
}
List<CompletedJob> jobInfoList;
if (!jobIdSet.isEmpty()) {
jobInfoList = jobIdSet.stream().map((jId) -> jobManager.getCompletedJob(jId)).filter((compJobOp) -> compJobOp.isPresent()).map((compJobOp) -> compJobOp.get()).collect(Collectors.toList());
} else {
jobInfoList = jobManager.getCompletedJobsList();
}
List<CompletedJob> shortenedList = jobInfoList.subList(0, Math.min(jobInfoList.size(), request.getLimit().orElse(DEFAULT_LIMIT)));
return Observable.from(shortenedList).flatMap((cJob) -> {
try {
if (logger.isDebugEnabled()) {
logger.debug("Fetching details for completed job {}", cJob);
}
Optional<IMantisJobMetadata> metaOp = jobManager.getJobDataForCompletedJob(cJob.getJobId());
if (metaOp.isPresent()) {
if (logger.isDebugEnabled()) {
logger.debug("Fetched details for completed job {} -> {}", cJob, metaOp.get());
}
return Observable.just(new MantisJobMetadataView(metaOp.get(), cJob.getTerminatedAt(), request.getStageNumberList(), request.getWorkerIndexList(), request.getWorkerNumberList(), request.getWorkerStateList(), false));
}
} catch (Exception e) {
logger.error("caught exception", e);
return Observable.empty();
}
return Observable.empty();
});
}
use of io.mantisrx.master.jobcluster.job.MantisJobMetadataView in project mantis by Netflix.
the class JobClusterActor method getFilteredNonTerminalJobList.
private Observable<MantisJobMetadataView> getFilteredNonTerminalJobList(ListJobCriteria request, Set<JobId> prefilteredJobIdSet) {
if (logger.isTraceEnabled()) {
logger.trace("Entering JobClusterActor:getFilteredNonTerminalJobList");
}
Duration timeout = Duration.ofMillis(500);
if ((request.getJobState().isPresent() && request.getJobState().get().equals(JobState.MetaState.Terminal))) {
if (logger.isTraceEnabled()) {
logger.trace("Exit JobClusterActor:getFilteredNonTerminalJobList with empty");
}
return Observable.empty();
}
List<JobInfo> jobInfoList;
//
if (!prefilteredJobIdSet.isEmpty()) {
jobInfoList = prefilteredJobIdSet.stream().map((jId) -> jobManager.getJobInfoForNonTerminalJob(jId)).filter((jInfoOp) -> jInfoOp.isPresent()).map((jInfoOp) -> jInfoOp.get()).collect(Collectors.toList());
} else {
// no prefiltering applied start with complete set of non terminal jobs
jobInfoList = jobManager.getAllNonTerminalJobsList();
}
List<JobInfo> shortenedList = jobInfoList.subList(0, Math.min(jobInfoList.size(), request.getLimit().orElse(DEFAULT_ACTIVE_JOB_LIMIT)));
if (logger.isDebugEnabled()) {
logger.debug("List of non terminal jobs {}", jobInfoList);
}
return Observable.from(shortenedList).flatMap((jInfo) -> {
GetJobDetailsRequest req = new GetJobDetailsRequest("system", jInfo.jobId);
CompletionStage<GetJobDetailsResponse> respCS = ask(jInfo.jobActor, req, timeout).thenApply(GetJobDetailsResponse.class::cast);
return Observable.from(respCS.toCompletableFuture(), Schedulers.io()).onErrorResumeNext(ex -> {
logger.warn("caught exception {}", ex.getMessage(), ex);
return Observable.empty();
});
}).filter((resp) -> resp != null && resp.getJobMetadata().isPresent()).map((resp) -> resp.getJobMetadata().get()).map((metaData) -> new MantisJobMetadataView(metaData, request.getStageNumberList(), request.getWorkerIndexList(), request.getWorkerNumberList(), request.getWorkerStateList(), false));
}
Aggregations